From fe268109bb1ceac9e97159d1c37b6815168a5365 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:35:03 +0800 Subject: [PATCH 01/17] Smoothing --- .../PipeConfigNodeRemainingTimeOperator.java | 7 ++- ...DataNodeRemainingEventAndTimeOperator.java | 12 +++-- .../iotdb/commons/conf/CommonConfig.java | 14 +++++- .../iotdb/commons/conf/CommonDescriptor.java | 8 +++ .../PipeRemainingTimeRateAverageTime.java | 49 +++++++++++++++++++ .../iotdb/commons/pipe/config/PipeConfig.java | 7 +++ 6 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/PipeRemainingTimeRateAverageTime.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index 201aa775de5e2..9388865d12483 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.manager.pipe.metric; +import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.confignode.manager.pipe.extractor.IoTDBConfigRegionExtractor; @@ -63,6 +64,8 @@ long getCreationTime() { double getRemainingTime() { final double pipeRemainingTimeCommitRateSmoothingFactor = PipeConfig.getInstance().getPipeRemainingTimeCommitRateSmoothingFactor(); + final PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = + PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime(); // Do not calculate heartbeat event final long totalConfigRegionWriteEventCount = @@ -73,9 +76,9 @@ long getCreationTime() { lastConfigRegionCommitSmoothingValue = lastConfigRegionCommitSmoothingValue == Long.MIN_VALUE - ? configRegionCommitMeter.getOneMinuteRate() + ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(configRegionCommitMeter) : pipeRemainingTimeCommitRateSmoothingFactor - * configRegionCommitMeter.getOneMinuteRate() + * pipeRemainingTimeCommitRateAverageTime.getMeterRate(configRegionCommitMeter) + (1 - pipeRemainingTimeCommitRateSmoothingFactor) * lastConfigRegionCommitSmoothingValue; final double configRegionRemainingTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index f7a08295f408d..a37dff3b69e38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.metric; +import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor; import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor; @@ -89,6 +90,8 @@ long getRemainingEvents() { double getRemainingTime() { final double pipeRemainingTimeCommitRateSmoothingFactor = PipeConfig.getInstance().getPipeRemainingTimeCommitRateSmoothingFactor(); + final PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = + PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime(); // Do not take heartbeat event into account final int totalDataRegionWriteEventCount = @@ -111,8 +114,9 @@ long getRemainingEvents() { lastDataRegionCommitSmoothingValue = lastDataRegionCommitSmoothingValue == Long.MIN_VALUE - ? dataRegionCommitMeter.getOneMinuteRate() - : pipeRemainingTimeCommitRateSmoothingFactor * dataRegionCommitMeter.getOneMinuteRate() + ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(dataRegionCommitMeter) + : pipeRemainingTimeCommitRateSmoothingFactor + * pipeRemainingTimeCommitRateAverageTime.getMeterRate(dataRegionCommitMeter) + (1 - pipeRemainingTimeCommitRateSmoothingFactor) * lastDataRegionCommitSmoothingValue; final double dataRegionRemainingTime; @@ -133,9 +137,9 @@ long getRemainingEvents() { lastSchemaRegionCommitSmoothingValue = lastSchemaRegionCommitSmoothingValue == Long.MIN_VALUE - ? schemaRegionCommitMeter.getOneMinuteRate() + ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(schemaRegionCommitMeter) : pipeRemainingTimeCommitRateSmoothingFactor - * schemaRegionCommitMeter.getOneMinuteRate() + * pipeRemainingTimeCommitRateAverageTime.getMeterRate(schemaRegionCommitMeter) + (1 - pipeRemainingTimeCommitRateSmoothingFactor) * lastSchemaRegionCommitSmoothingValue; final double schemaRegionRemainingTime; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 1de7f33df3a83..117fe0e5d4902 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.enums.HandleSystemErrorStrategy; +import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.KillPoint.KillPoint; @@ -235,7 +236,9 @@ public class CommonConfig { private float pipeLeaderCacheMemoryUsagePercentage = 0.1F; private long pipeListeningQueueTransferSnapshotThreshold = 1000; private int pipeSnapshotExecutionMaxBatchSize = 1000; - private double pipeRemainingTimeCommitRateSmoothingFactor = 0.5; + private double pipeRemainingTimeCommitRateSmoothingFactor = 0.4; + private PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = + PipeRemainingTimeRateAverageTime.FIFTEEN_MINUTES; private long twoStageAggregateMaxCombinerLiveTimeInMs = 8 * 60 * 1000L; // 8 minutes private long twoStageAggregateDataRegionInfoCacheTimeInMs = 3 * 60 * 1000L; // 3 minutes @@ -1029,6 +1032,15 @@ public void setPipeRemainingTimeCommitRateSmoothingFactor( this.pipeRemainingTimeCommitRateSmoothingFactor = pipeRemainingTimeCommitRateSmoothingFactor; } + public PipeRemainingTimeRateAverageTime getPipeRemainingTimeCommitRateAverageTime() { + return pipeRemainingTimeCommitRateAverageTime; + } + + public void setPipeRemainingTimeCommitRateAverageTime( + PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime) { + this.pipeRemainingTimeCommitRateAverageTime = pipeRemainingTimeCommitRateAverageTime; + } + public double getPipeAllSinksRateLimitBytesPerSecond() { return pipeAllSinksRateLimitBytesPerSecond; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index f72ea5e528d15..bf2b3af0301e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.conf; import org.apache.iotdb.commons.enums.HandleSystemErrorStrategy; +import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; @@ -546,6 +547,13 @@ private void loadPipeProps(Properties properties) { properties.getProperty( "pipe_remaining_time_commit_rate_smoothing_factor", String.valueOf(config.getPipeRemainingTimeCommitRateSmoothingFactor())))); + config.setPipeRemainingTimeCommitRateAverageTime( + PipeRemainingTimeRateAverageTime.valueOf( + properties + .getProperty( + "pipe_remaining_time_commit_rate_average_time", + String.valueOf(config.getPipeRemainingTimeCommitRateAverageTime())) + .trim())); config.setTwoStageAggregateMaxCombinerLiveTimeInMs( Long.parseLong( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/PipeRemainingTimeRateAverageTime.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/PipeRemainingTimeRateAverageTime.java new file mode 100644 index 0000000000000..25f799d09f691 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/PipeRemainingTimeRateAverageTime.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.enums; + +import org.apache.iotdb.commons.pipe.config.PipeConfig; + +import com.codahale.metrics.Meter; + +public enum PipeRemainingTimeRateAverageTime { + ONE_MINUTE, + FIVE_MINUTES, + FIFTEEN_MINUTES, + MEAN; + + public double getMeterRate(final Meter meter) { + switch (this) { + case ONE_MINUTE: + return meter.getOneMinuteRate(); + case FIVE_MINUTES: + return meter.getFiveMinuteRate(); + case FIFTEEN_MINUTES: + return meter.getFifteenMinuteRate(); + case MEAN: + return meter.getMeanRate(); + default: + throw new UnsupportedOperationException( + String.format( + "The type %s is not supported in average time of pipe remaining time.", + PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime())); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java index 48debcce40816..9ac3c19d9b617 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,6 +144,10 @@ public double getPipeRemainingTimeCommitRateSmoothingFactor() { return COMMON_CONFIG.getPipeRemainingTimeCommitRateSmoothingFactor(); } + public PipeRemainingTimeRateAverageTime getPipeRemainingTimeCommitRateAverageTime() { + return COMMON_CONFIG.getPipeRemainingTimeCommitRateAverageTime(); + } + /////////////////////////////// Meta Consistency /////////////////////////////// public boolean isSeperatedPipeHeartbeatEnabled() { @@ -330,6 +335,8 @@ public void printAllConfigs() { LOGGER.info( "PipeRemainingTimeCommitRateSmoothingFactor: {}", getPipeRemainingTimeCommitRateSmoothingFactor()); + LOGGER.info( + "PipeRemainingTimeCommitRateAverageTime: {}", getPipeRemainingTimeCommitRateAverageTime()); LOGGER.info("PipeAsyncConnectorSelectorNumber: {}", getPipeAsyncConnectorSelectorNumber()); LOGGER.info("PipeAsyncConnectorMaxClientNumber: {}", getPipeAsyncConnectorMaxClientNumber()); From 9cc01052c0d5267f371dfb99a3890b7dfcfbf96c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:47:21 +0800 Subject: [PATCH 02/17] Update CommonConfig.java --- .../main/java/org/apache/iotdb/commons/conf/CommonConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 117fe0e5d4902..1b2d44d9ca16e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -236,9 +236,9 @@ public class CommonConfig { private float pipeLeaderCacheMemoryUsagePercentage = 0.1F; private long pipeListeningQueueTransferSnapshotThreshold = 1000; private int pipeSnapshotExecutionMaxBatchSize = 1000; - private double pipeRemainingTimeCommitRateSmoothingFactor = 0.4; + private double pipeRemainingTimeCommitRateSmoothingFactor = 1.0; private PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = - PipeRemainingTimeRateAverageTime.FIFTEEN_MINUTES; + PipeRemainingTimeRateAverageTime.MEAN; private long twoStageAggregateMaxCombinerLiveTimeInMs = 8 * 60 * 1000L; // 8 minutes private long twoStageAggregateDataRegionInfoCacheTimeInMs = 3 * 60 * 1000L; // 3 minutes From ace892d5abab15f9cf72d8b04f3a7ae34c8b9d30 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:41:36 +0800 Subject: [PATCH 03/17] Adjusted calculation --- .../PipeConfigNodeRemainingTimeMetrics.java | 20 +++ .../PipeConfigNodeRemainingTimeOperator.java | 88 ++++++----- .../pipe/task/PipeConfigNodeTaskStage.java | 17 +- .../agent/task/PipeDataNodeTaskAgent.java | 17 +- ...eDataNodeRemainingEventAndTimeMetrics.java | 20 +++ ...DataNodeRemainingEventAndTimeOperator.java | 148 ++++++++++-------- .../iotdb/commons/conf/CommonConfig.java | 13 +- .../iotdb/commons/conf/CommonDescriptor.java | 8 +- .../pipe/agent/task/PipeTaskAgent.java | 4 +- .../iotdb/commons/pipe/config/PipeConfig.java | 8 +- .../pipe/metric/PipeRemainingOperator.java | 78 +++++++++ 11 files changed, 291 insertions(+), 130 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java index 378251eb45c2e..ef57b7d5ac2fe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java @@ -110,6 +110,26 @@ public void register(final IoTDBConfigRegionExtractor extractor) { } } + public void thawRate(final String pipeID) { + if (!remainingTimeOperatorMap.containsKey(pipeID)) { + LOGGER.warn( + "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + pipeID); + return; + } + remainingTimeOperatorMap.get(pipeID).thawRate(); + } + + public void freezeRate(final String pipeID) { + if (!remainingTimeOperatorMap.containsKey(pipeID)) { + LOGGER.warn( + "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + pipeID); + return; + } + remainingTimeOperatorMap.get(pipeID).freezeRate(); + } + public void deregister(final String pipeID) { if (!remainingTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index 9388865d12483..51483260f6ba2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -21,78 +21,70 @@ import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.metric.PipeRemainingOperator; +import org.apache.iotdb.confignode.manager.pipe.execution.PipeConfigNodeSubtask; import org.apache.iotdb.confignode.manager.pipe.extractor.IoTDBConfigRegionExtractor; import com.codahale.metrics.Clock; import com.codahale.metrics.ExponentialMovingAverages; import com.codahale.metrics.Meter; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; -class PipeConfigNodeRemainingTimeOperator { +class PipeConfigNodeRemainingTimeOperator extends PipeRemainingOperator { - private static final long CONFIG_NODE_REMAINING_MAX_SECONDS = 365 * 24 * 60 * 60L; // 1 year - - private String pipeName; - private long creationTime = 0; - - private final ConcurrentMap - configRegionExtractors = new ConcurrentHashMap<>(); - private final Meter configRegionCommitMeter = - new Meter(new ExponentialMovingAverages(), Clock.defaultClock()); + private final Set configRegionExtractors = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final AtomicReference configRegionCommitMeter = new AtomicReference<>(null); private double lastConfigRegionCommitSmoothingValue = Long.MIN_VALUE; - //////////////////////////// Tags //////////////////////////// - - String getPipeName() { - return pipeName; - } - - long getCreationTime() { - return creationTime; - } - //////////////////////////// Remaining time calculation //////////////////////////// /** - * This will calculate the estimated remaining time of the given pipe's config region subTask. + * This will calculate the estimated remaining time of the given pipe's {@link + * PipeConfigNodeSubtask}. * * @return The estimated remaining time */ double getRemainingTime() { - final double pipeRemainingTimeCommitRateSmoothingFactor = - PipeConfig.getInstance().getPipeRemainingTimeCommitRateSmoothingFactor(); final PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime(); // Do not calculate heartbeat event final long totalConfigRegionWriteEventCount = - configRegionExtractors.keySet().stream() + configRegionExtractors.stream() .map(IoTDBConfigRegionExtractor::getUnTransferredEventCount) .reduce(Long::sum) .orElse(0L); - lastConfigRegionCommitSmoothingValue = - lastConfigRegionCommitSmoothingValue == Long.MIN_VALUE - ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(configRegionCommitMeter) - : pipeRemainingTimeCommitRateSmoothingFactor - * pipeRemainingTimeCommitRateAverageTime.getMeterRate(configRegionCommitMeter) - + (1 - pipeRemainingTimeCommitRateSmoothingFactor) - * lastConfigRegionCommitSmoothingValue; + configRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + lastConfigRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); + } + return meter; + }); + final double configRegionRemainingTime; if (totalConfigRegionWriteEventCount <= 0) { + notifyEmpty(); configRegionRemainingTime = 0; } else { + notifyNonEmpty(); configRegionRemainingTime = lastConfigRegionCommitSmoothingValue <= 0 ? Double.MAX_VALUE : totalConfigRegionWriteEventCount / lastConfigRegionCommitSmoothingValue; } - return configRegionRemainingTime >= CONFIG_NODE_REMAINING_MAX_SECONDS - ? CONFIG_NODE_REMAINING_MAX_SECONDS + return configRegionRemainingTime >= REMAINING_MAX_SECONDS + ? REMAINING_MAX_SECONDS : configRegionRemainingTime; } @@ -100,17 +92,31 @@ long getCreationTime() { void register(final IoTDBConfigRegionExtractor extractor) { setNameAndCreationTime(extractor.getPipeName(), extractor.getCreationTime()); - configRegionExtractors.put(extractor, extractor); - } - - private void setNameAndCreationTime(final String pipeName, final long creationTime) { - this.pipeName = pipeName; - this.creationTime = creationTime; + configRegionExtractors.add(extractor); } //////////////////////////// Rate //////////////////////////// void markConfigRegionCommit() { - configRegionCommitMeter.mark(); + configRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + meter.mark(); + } + return meter; + }); + } + + //////////////////////////// Switch //////////////////////////// + + @Override + public void thawRate() { + configRegionCommitMeter.compareAndSet( + null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + } + + @Override + public void freezeRate() { + configRegionCommitMeter.set(null); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java index 36eec65b69e82..59e9ade4d40f2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.task.stage.PipeTaskStage; import org.apache.iotdb.confignode.manager.pipe.execution.PipeConfigNodeSubtask; import org.apache.iotdb.confignode.manager.pipe.execution.PipeConfigNodeSubtaskExecutor; +import org.apache.iotdb.confignode.manager.pipe.metric.PipeConfigNodeRemainingTimeMetrics; import org.apache.iotdb.pipe.api.exception.PipeException; import java.util.Map; @@ -32,12 +33,12 @@ public class PipeConfigNodeTaskStage extends PipeTaskStage { private final PipeConfigNodeSubtask subtask; public PipeConfigNodeTaskStage( - String pipeName, - long creationTime, - Map extractorAttributes, - Map processorAttributes, - Map connectorAttributes, - PipeTaskMeta pipeTaskMeta) { + final String pipeName, + final long creationTime, + final Map extractorAttributes, + final Map processorAttributes, + final Map connectorAttributes, + final PipeTaskMeta pipeTaskMeta) { try { subtask = @@ -48,7 +49,7 @@ public PipeConfigNodeTaskStage( processorAttributes, connectorAttributes, pipeTaskMeta); - } catch (Exception e) { + } catch (final Exception e) { throw new PipeException( String.format( "Failed to create subtask for pipe %s, creation time %d", pipeName, creationTime), @@ -66,11 +67,13 @@ public void startSubtask() throws PipeException { // IoTDBConfigRegionExtractor is started by executor because starting // here may cause deadlock when triggering snapshot PipeConfigNodeSubtaskExecutor.getInstance().start(subtask.getTaskID()); + PipeConfigNodeRemainingTimeMetrics.getInstance().thawRate(subtask.getTaskID()); } @Override public void stopSubtask() throws PipeException { PipeConfigNodeSubtaskExecutor.getInstance().stop(subtask.getTaskID()); + PipeConfigNodeRemainingTimeMetrics.getInstance().freezeRate(subtask.getTaskID()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 37c04e26f5085..27e968b1b3457 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -237,6 +237,21 @@ private void closeSchemaRegionListeningQueueIfNecessary( }); } + @Override + protected void startPipe(final String pipeName, final long creationTime) { + super.startPipe(pipeName, creationTime); + + PipeDataNodeRemainingEventAndTimeMetrics.getInstance().thawRate(pipeName + "_" + creationTime); + } + + @Override + protected void stopPipe(final String pipeName, final long creationTime) { + super.stopPipe(pipeName, creationTime); + + PipeDataNodeRemainingEventAndTimeMetrics.getInstance() + .freezeRate(pipeName + "_" + creationTime); + } + @Override protected void dropPipe(final String pipeName, final long creationTime) { super.dropPipe(pipeName, creationTime); @@ -612,7 +627,7 @@ public Set getPipeTaskRegionIdSet(final String pipeName, final long cre ///////////////////////// Pipe Consensus ///////////////////////// - public ProgressIndex getPipeTaskProgressIndex(String pipeName, int consensusGroupId) { + public ProgressIndex getPipeTaskProgressIndex(final String pipeName, final int consensusGroupId) { if (!tryReadLockWithTimeOut(10)) { throw new PipeException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java index c1ecb8460b4d0..79cb1447f7f62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java @@ -151,6 +151,26 @@ public void register(final IoTDBSchemaRegionExtractor extractor) { } } + public void thawRate(final String pipeID) { + if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { + LOGGER.warn( + "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + pipeID); + return; + } + remainingEventAndTimeOperatorMap.get(pipeID).thawRate(); + } + + public void freezeRate(final String pipeID) { + if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { + LOGGER.warn( + "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + pipeID); + return; + } + remainingEventAndTimeOperatorMap.get(pipeID).freezeRate(); + } + public void deregister(final String pipeID) { if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index a37dff3b69e38..cc07363240343 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -21,60 +21,47 @@ import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.metric.PipeRemainingOperator; import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor; import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor; import org.apache.iotdb.db.pipe.task.subtask.connector.PipeConnectorSubtask; +import org.apache.iotdb.pipe.api.event.Event; import com.codahale.metrics.Clock; import com.codahale.metrics.ExponentialMovingAverages; import com.codahale.metrics.Meter; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -class PipeDataNodeRemainingEventAndTimeOperator { - - private static final long DATA_NODE_REMAINING_MAX_SECONDS = 365 * 24 * 60 * 60L; // 1 year - - private String pipeName; - private long creationTime = 0; - - private final ConcurrentMap - dataRegionExtractors = new ConcurrentHashMap<>(); - private final ConcurrentMap dataRegionConnectors = - new ConcurrentHashMap<>(); - private final ConcurrentMap - schemaRegionExtractors = new ConcurrentHashMap<>(); - private final Meter dataRegionCommitMeter = - new Meter(new ExponentialMovingAverages(), Clock.defaultClock()); - private final Meter schemaRegionCommitMeter = - new Meter(new ExponentialMovingAverages(), Clock.defaultClock()); +import java.util.concurrent.atomic.AtomicReference; + +class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator { + private final Set dataRegionExtractors = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set dataRegionConnectors = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set schemaRegionExtractors = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final AtomicReference dataRegionCommitMeter = new AtomicReference<>(null); + private final AtomicReference schemaRegionCommitMeter = new AtomicReference<>(null); private double lastDataRegionCommitSmoothingValue = Long.MIN_VALUE; private double lastSchemaRegionCommitSmoothingValue = Long.MIN_VALUE; - //////////////////////////// Tags //////////////////////////// - - String getPipeName() { - return pipeName; - } - - long getCreationTime() { - return creationTime; - } - //////////////////////////// Remaining event & time calculation //////////////////////////// long getRemainingEvents() { - return dataRegionExtractors.keySet().stream() + return dataRegionExtractors.stream() .map(IoTDBDataRegionExtractor::getEventCount) .reduce(Integer::sum) .orElse(0) - + dataRegionConnectors.keySet().stream() + + dataRegionConnectors.stream() .map(connectorSubtask -> connectorSubtask.getEventCount(pipeName)) .reduce(Integer::sum) .orElse(0) - + schemaRegionExtractors.keySet().stream() + + schemaRegionExtractors.stream() .map(IoTDBSchemaRegionExtractor::getUnTransferredEventCount) .reduce(Long::sum) .orElse(0L); @@ -83,42 +70,41 @@ long getRemainingEvents() { /** * This will calculate the estimated remaining time of pipe. * - *

Note: The events in pipe assigner are omitted. + *

Note: The {@link Event}s in pipe assigner are omitted. * * @return The estimated remaining time */ double getRemainingTime() { - final double pipeRemainingTimeCommitRateSmoothingFactor = - PipeConfig.getInstance().getPipeRemainingTimeCommitRateSmoothingFactor(); final PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime(); // Do not take heartbeat event into account final int totalDataRegionWriteEventCount = - dataRegionExtractors.keySet().stream() + dataRegionExtractors.stream() .map(IoTDBDataRegionExtractor::getEventCount) .reduce(Integer::sum) .orElse(0) - + dataRegionConnectors.keySet().stream() + + dataRegionConnectors.stream() .map(connectorSubtask -> connectorSubtask.getEventCount(pipeName)) .reduce(Integer::sum) .orElse(0) - - dataRegionExtractors.keySet().stream() + - dataRegionExtractors.stream() .map(IoTDBDataRegionExtractor::getPipeHeartbeatEventCount) .reduce(Integer::sum) .orElse(0) - - dataRegionConnectors.keySet().stream() + - dataRegionConnectors.stream() .map(PipeConnectorSubtask::getPipeHeartbeatEventCount) .reduce(Integer::sum) .orElse(0); - lastDataRegionCommitSmoothingValue = - lastDataRegionCommitSmoothingValue == Long.MIN_VALUE - ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(dataRegionCommitMeter) - : pipeRemainingTimeCommitRateSmoothingFactor - * pipeRemainingTimeCommitRateAverageTime.getMeterRate(dataRegionCommitMeter) - + (1 - pipeRemainingTimeCommitRateSmoothingFactor) - * lastDataRegionCommitSmoothingValue; + dataRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + lastDataRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); + } + return meter; + }); final double dataRegionRemainingTime; if (totalDataRegionWriteEventCount <= 0) { dataRegionRemainingTime = 0; @@ -130,18 +116,19 @@ long getRemainingEvents() { } final long totalSchemaRegionWriteEventCount = - schemaRegionExtractors.keySet().stream() + schemaRegionExtractors.stream() .map(IoTDBSchemaRegionExtractor::getUnTransferredEventCount) .reduce(Long::sum) .orElse(0L); - lastSchemaRegionCommitSmoothingValue = - lastSchemaRegionCommitSmoothingValue == Long.MIN_VALUE - ? pipeRemainingTimeCommitRateAverageTime.getMeterRate(schemaRegionCommitMeter) - : pipeRemainingTimeCommitRateSmoothingFactor - * pipeRemainingTimeCommitRateAverageTime.getMeterRate(schemaRegionCommitMeter) - + (1 - pipeRemainingTimeCommitRateSmoothingFactor) - * lastSchemaRegionCommitSmoothingValue; + schemaRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + lastSchemaRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); + } + return meter; + }); final double schemaRegionRemainingTime; if (totalSchemaRegionWriteEventCount <= 0) { schemaRegionRemainingTime = 0; @@ -152,40 +139,71 @@ long getRemainingEvents() { : totalSchemaRegionWriteEventCount / lastSchemaRegionCommitSmoothingValue; } + if (totalDataRegionWriteEventCount + totalSchemaRegionWriteEventCount == 0) { + notifyEmpty(); + } else { + notifyNonEmpty(); + } + final double result = Math.max(dataRegionRemainingTime, schemaRegionRemainingTime); - return result >= DATA_NODE_REMAINING_MAX_SECONDS ? DATA_NODE_REMAINING_MAX_SECONDS : result; + return result >= REMAINING_MAX_SECONDS ? REMAINING_MAX_SECONDS : result; } //////////////////////////// Register & deregister (pipe integration) //////////////////////////// void register(final IoTDBDataRegionExtractor extractor) { setNameAndCreationTime(extractor.getPipeName(), extractor.getCreationTime()); - dataRegionExtractors.put(extractor, extractor); + dataRegionExtractors.add(extractor); } void register( final PipeConnectorSubtask connectorSubtask, final String pipeName, final long creationTime) { setNameAndCreationTime(pipeName, creationTime); - dataRegionConnectors.put(connectorSubtask, connectorSubtask); + dataRegionConnectors.add(connectorSubtask); } void register(final IoTDBSchemaRegionExtractor extractor) { setNameAndCreationTime(extractor.getPipeName(), extractor.getCreationTime()); - schemaRegionExtractors.put(extractor, extractor); - } - - private void setNameAndCreationTime(final String pipeName, final long creationTime) { - this.pipeName = pipeName; - this.creationTime = creationTime; + schemaRegionExtractors.add(extractor); } //////////////////////////// Rate //////////////////////////// void markDataRegionCommit() { - dataRegionCommitMeter.mark(); + dataRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + meter.mark(); + } + return meter; + }); } void markSchemaRegionCommit() { - schemaRegionCommitMeter.mark(); + schemaRegionCommitMeter.updateAndGet( + meter -> { + if (Objects.nonNull(meter)) { + meter.mark(); + } + return meter; + }); + } + + //////////////////////////// Switch //////////////////////////// + + // Thread-safe & Idempotent + @Override + public void thawRate() { + dataRegionCommitMeter.compareAndSet( + null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + schemaRegionCommitMeter.compareAndSet( + null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + } + + // Thread-safe & Idempotent + @Override + public void freezeRate() { + dataRegionCommitMeter.set(null); + schemaRegionCommitMeter.set(null); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 1b2d44d9ca16e..c21f853b8a722 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -236,7 +236,7 @@ public class CommonConfig { private float pipeLeaderCacheMemoryUsagePercentage = 0.1F; private long pipeListeningQueueTransferSnapshotThreshold = 1000; private int pipeSnapshotExecutionMaxBatchSize = 1000; - private double pipeRemainingTimeCommitRateSmoothingFactor = 1.0; + private long pipeRemainingTimeCommitRateAutoSwitchSeconds = 30; private PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime = PipeRemainingTimeRateAverageTime.MEAN; @@ -1023,13 +1023,14 @@ public void setPipeSnapshotExecutionMaxBatchSize(int pipeSnapshotExecutionMaxBat this.pipeSnapshotExecutionMaxBatchSize = pipeSnapshotExecutionMaxBatchSize; } - public double getPipeRemainingTimeCommitRateSmoothingFactor() { - return pipeRemainingTimeCommitRateSmoothingFactor; + public long getPipeRemainingTimeCommitRateAutoSwitchSeconds() { + return pipeRemainingTimeCommitRateAutoSwitchSeconds; } - public void setPipeRemainingTimeCommitRateSmoothingFactor( - double pipeRemainingTimeCommitRateSmoothingFactor) { - this.pipeRemainingTimeCommitRateSmoothingFactor = pipeRemainingTimeCommitRateSmoothingFactor; + public void setPipeRemainingTimeCommitRateAutoSwitchSeconds( + long pipeRemainingTimeCommitRateAutoSwitchSeconds) { + this.pipeRemainingTimeCommitRateAutoSwitchSeconds = + pipeRemainingTimeCommitRateAutoSwitchSeconds; } public PipeRemainingTimeRateAverageTime getPipeRemainingTimeCommitRateAverageTime() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index bf2b3af0301e7..67718e1224c9d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -542,11 +542,11 @@ private void loadPipeProps(Properties properties) { properties.getProperty( "pipe_snapshot_execution_max_batch_size", String.valueOf(config.getPipeSnapshotExecutionMaxBatchSize())))); - config.setPipeRemainingTimeCommitRateSmoothingFactor( - Double.parseDouble( + config.setPipeRemainingTimeCommitRateAutoSwitchSeconds( + Long.parseLong( properties.getProperty( - "pipe_remaining_time_commit_rate_smoothing_factor", - String.valueOf(config.getPipeRemainingTimeCommitRateSmoothingFactor())))); + "pipe_remaining_time_commit_rate_auto_switch_seconds", + String.valueOf(config.getPipeRemainingTimeCommitRateAutoSwitchSeconds())))); config.setPipeRemainingTimeCommitRateAverageTime( PipeRemainingTimeRateAverageTime.valueOf( properties diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index a1844bef5722e..aa0d777d921c5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -103,7 +103,7 @@ protected void acquireWriteLock() { pipeMetaKeeper.acquireWriteLock(); } - protected boolean tryWriteLockWithTimeOut(long timeOutInSeconds) { + protected boolean tryWriteLockWithTimeOut(final long timeOutInSeconds) { try { return pipeMetaKeeper.tryWriteLock(timeOutInSeconds); } catch (final InterruptedException e) { @@ -531,7 +531,7 @@ protected void dropPipe(final String pipeName) { pipeMetaKeeper.removePipeMeta(pipeName); } - private void startPipe(final String pipeName, final long creationTime) { + protected void startPipe(final String pipeName, final long creationTime) { final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (!checkBeforeStartPipe(existedPipeMeta, pipeName, creationTime)) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java index 9ac3c19d9b617..5fa4aa8a812a3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java @@ -140,8 +140,8 @@ public int getPipeSnapshotExecutionMaxBatchSize() { return COMMON_CONFIG.getPipeSnapshotExecutionMaxBatchSize(); } - public double getPipeRemainingTimeCommitRateSmoothingFactor() { - return COMMON_CONFIG.getPipeRemainingTimeCommitRateSmoothingFactor(); + public long getPipeRemainingTimeCommitAutoSwitchSeconds() { + return COMMON_CONFIG.getPipeRemainingTimeCommitRateAutoSwitchSeconds(); } public PipeRemainingTimeRateAverageTime getPipeRemainingTimeCommitRateAverageTime() { @@ -333,8 +333,8 @@ public void printAllConfigs() { getPipeListeningQueueTransferSnapshotThreshold()); LOGGER.info("PipeSnapshotExecutionMaxBatchSize: {}", getPipeSnapshotExecutionMaxBatchSize()); LOGGER.info( - "PipeRemainingTimeCommitRateSmoothingFactor: {}", - getPipeRemainingTimeCommitRateSmoothingFactor()); + "PipeRemainingTimeCommitAutoSwitchSeconds: {}", + getPipeRemainingTimeCommitAutoSwitchSeconds()); LOGGER.info( "PipeRemainingTimeCommitRateAverageTime: {}", getPipeRemainingTimeCommitRateAverageTime()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java new file mode 100644 index 0000000000000..0b731dfd4e5d1 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.metric; + +import org.apache.iotdb.commons.pipe.config.PipeConfig; + +public abstract class PipeRemainingOperator { + + protected static final long REMAINING_MAX_SECONDS = 365 * 24 * 60 * 60L; // 1 year + + protected String pipeName; + protected long creationTime = 0; + + private long lastEmptyTimeStamp = System.currentTimeMillis(); + private long lastNonEmptyTimeStamp = System.currentTimeMillis(); + + //////////////////////////// Tags //////////////////////////// + + public String getPipeName() { + return pipeName; + } + + public long getCreationTime() { + return creationTime; + } + + //////////////////////////// Register & deregister (pipe integration) //////////////////////////// + + protected void setNameAndCreationTime(final String pipeName, final long creationTime) { + this.pipeName = pipeName; + this.creationTime = creationTime; + } + + //////////////////////////// Switch //////////////////////////// + + protected void notifyEmpty() { + final long pipeRemainingTimeCommitAutoSwitchSeconds = + PipeConfig.getInstance().getPipeRemainingTimeCommitAutoSwitchSeconds(); + + lastEmptyTimeStamp = System.currentTimeMillis(); + if (lastEmptyTimeStamp - lastNonEmptyTimeStamp + >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { + freezeRate(); + } + } + + protected void notifyNonEmpty() { + final long pipeRemainingTimeCommitAutoSwitchSeconds = + PipeConfig.getInstance().getPipeRemainingTimeCommitAutoSwitchSeconds(); + + lastNonEmptyTimeStamp = System.currentTimeMillis(); + if (lastNonEmptyTimeStamp - lastEmptyTimeStamp + >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { + thawRate(); + } + } + + public abstract void thawRate(); + + public abstract void freezeRate(); +} From 3178cf90697afdb4285e60faaa2867af16333397 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:39:06 +0800 Subject: [PATCH 04/17] Update pom.xml --- iotdb-core/node-commons/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 5e5a1d251dc65..e75e207d030e7 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -95,6 +95,10 @@ metrics-core 1.3.3-SNAPSHOT + + io.dropwizard.metrics + metrics-core + org.slf4j slf4j-api From 17e24813a768784ce1c2c1517e65a729f5fc0b81 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:02:21 +0800 Subject: [PATCH 05/17] Fix log --- .../pipe/metric/PipeConfigNodeRemainingTimeMetrics.java | 4 ++-- .../pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java index ef57b7d5ac2fe..9bab5e4571541 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java @@ -113,7 +113,7 @@ public void register(final IoTDBConfigRegionExtractor extractor) { public void thawRate(final String pipeID) { if (!remainingTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( - "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + "Failed to thaw pipe remaining time rate, RemainingTimeOperator({}) does not exist", pipeID); return; } @@ -123,7 +123,7 @@ public void thawRate(final String pipeID) { public void freezeRate(final String pipeID) { if (!remainingTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( - "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + "Failed to freeze pipe remaining time rate, RemainingTimeOperator({}) does not exist", pipeID); return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java index 79cb1447f7f62..fc92136e6fee0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java @@ -154,7 +154,7 @@ public void register(final IoTDBSchemaRegionExtractor extractor) { public void thawRate(final String pipeID) { if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( - "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + "Failed to thaw pipe remaining time rate, RemainingEventAndTimeOperator({}) does not exist", pipeID); return; } @@ -164,7 +164,7 @@ public void thawRate(final String pipeID) { public void freezeRate(final String pipeID) { if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { LOGGER.warn( - "Failed to deregister pipe remaining event and time metrics, RemainingEventAndTimeOperator({}) does not exist", + "Failed to freeze pipe remaining time rate, RemainingEventAndTimeOperator({}) does not exist", pipeID); return; } From c58286a51d795304c23b7abe1e720d442323383f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:32:13 +0800 Subject: [PATCH 06/17] improvement --- .../metric/PipeConfigNodeRemainingTimeOperator.java | 5 +++-- .../manager/pipe/task/PipeConfigNodeTaskStage.java | 3 ++- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 5 +++-- .../PipeDataNodeRemainingEventAndTimeOperator.java | 10 ++++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index 51483260f6ba2..7a0bd1472364b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -39,9 +39,10 @@ class PipeConfigNodeRemainingTimeOperator extends PipeRemainingOperator { private final Set configRegionExtractors = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final AtomicReference configRegionCommitMeter = new AtomicReference<>(null); + private final AtomicReference configRegionCommitMeter = + new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); - private double lastConfigRegionCommitSmoothingValue = Long.MIN_VALUE; + private double lastConfigRegionCommitSmoothingValue = Long.MAX_VALUE; //////////////////////////// Remaining time calculation //////////////////////////// diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java index 59e9ade4d40f2..8a647f649a951 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java @@ -72,8 +72,9 @@ public void startSubtask() throws PipeException { @Override public void stopSubtask() throws PipeException { - PipeConfigNodeSubtaskExecutor.getInstance().stop(subtask.getTaskID()); + // Freeze first to guarantee more precise result PipeConfigNodeRemainingTimeMetrics.getInstance().freezeRate(subtask.getTaskID()); + PipeConfigNodeSubtaskExecutor.getInstance().stop(subtask.getTaskID()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 27e968b1b3457..0e7a2452d0987 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -246,10 +246,11 @@ protected void startPipe(final String pipeName, final long creationTime) { @Override protected void stopPipe(final String pipeName, final long creationTime) { - super.stopPipe(pipeName, creationTime); - + // Freeze first to guarantee more precise result PipeDataNodeRemainingEventAndTimeMetrics.getInstance() .freezeRate(pipeName + "_" + creationTime); + + super.stopPipe(pipeName, creationTime); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index cc07363240343..3b0de29e0ffb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -44,11 +44,13 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator { Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set schemaRegionExtractors = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final AtomicReference dataRegionCommitMeter = new AtomicReference<>(null); - private final AtomicReference schemaRegionCommitMeter = new AtomicReference<>(null); + private final AtomicReference dataRegionCommitMeter = + new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + private final AtomicReference schemaRegionCommitMeter = + new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); - private double lastDataRegionCommitSmoothingValue = Long.MIN_VALUE; - private double lastSchemaRegionCommitSmoothingValue = Long.MIN_VALUE; + private double lastDataRegionCommitSmoothingValue = Long.MAX_VALUE; + private double lastSchemaRegionCommitSmoothingValue = Long.MAX_VALUE; //////////////////////////// Remaining event & time calculation //////////////////////////// From b74278606cdf39b905b1072429c5d189bdaead54 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:00:02 +0800 Subject: [PATCH 07/17] adjusting --- .../pipe/metric/PipeConfigNodeRemainingTimeOperator.java | 3 +-- .../metric/PipeDataNodeRemainingEventAndTimeOperator.java | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index 7a0bd1472364b..e5b36860a76d8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -39,8 +39,7 @@ class PipeConfigNodeRemainingTimeOperator extends PipeRemainingOperator { private final Set configRegionExtractors = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final AtomicReference configRegionCommitMeter = - new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + private final AtomicReference configRegionCommitMeter = new AtomicReference<>(null); private double lastConfigRegionCommitSmoothingValue = Long.MAX_VALUE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index 3b0de29e0ffb4..f25bbbe023bf8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -44,10 +44,8 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator { Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set schemaRegionExtractors = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final AtomicReference dataRegionCommitMeter = - new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); - private final AtomicReference schemaRegionCommitMeter = - new AtomicReference<>(new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); + private final AtomicReference dataRegionCommitMeter = new AtomicReference<>(null); + private final AtomicReference schemaRegionCommitMeter = new AtomicReference<>(null); private double lastDataRegionCommitSmoothingValue = Long.MAX_VALUE; private double lastSchemaRegionCommitSmoothingValue = Long.MAX_VALUE; From 610e8ec25ccef7671e922ca821188754a7af57a8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:41:32 +0800 Subject: [PATCH 08/17] bug fix --- .../PipeConfigNodeRemainingTimeMetrics.java | 4 ++-- .../PipeConfigNodeRemainingTimeOperator.java | 6 ++++-- ...eDataNodeRemainingEventAndTimeMetrics.java | 4 ++-- ...DataNodeRemainingEventAndTimeOperator.java | 6 ++++-- .../pipe/metric/PipeRemainingOperator.java | 21 ++++++++++++++----- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java index 9bab5e4571541..003098df15931 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java @@ -117,7 +117,7 @@ public void thawRate(final String pipeID) { pipeID); return; } - remainingTimeOperatorMap.get(pipeID).thawRate(); + remainingTimeOperatorMap.get(pipeID).thawRate(true); } public void freezeRate(final String pipeID) { @@ -127,7 +127,7 @@ public void freezeRate(final String pipeID) { pipeID); return; } - remainingTimeOperatorMap.get(pipeID).freezeRate(); + remainingTimeOperatorMap.get(pipeID).freezeRate(true); } public void deregister(final String pipeID) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index e5b36860a76d8..fc9f50474e1d9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -110,13 +110,15 @@ void markConfigRegionCommit() { //////////////////////////// Switch //////////////////////////// @Override - public void thawRate() { + public void thawRate(final boolean isStartPipe) { + super.thawRate(isStartPipe); configRegionCommitMeter.compareAndSet( null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); } @Override - public void freezeRate() { + public void freezeRate(final boolean isStopPipe) { + super.freezeRate(isStopPipe); configRegionCommitMeter.set(null); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java index fc92136e6fee0..fba9194680508 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java @@ -158,7 +158,7 @@ public void thawRate(final String pipeID) { pipeID); return; } - remainingEventAndTimeOperatorMap.get(pipeID).thawRate(); + remainingEventAndTimeOperatorMap.get(pipeID).thawRate(true); } public void freezeRate(final String pipeID) { @@ -168,7 +168,7 @@ public void freezeRate(final String pipeID) { pipeID); return; } - remainingEventAndTimeOperatorMap.get(pipeID).freezeRate(); + remainingEventAndTimeOperatorMap.get(pipeID).freezeRate(true); } public void deregister(final String pipeID) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index f25bbbe023bf8..129d9866a4d15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -193,7 +193,8 @@ void markSchemaRegionCommit() { // Thread-safe & Idempotent @Override - public void thawRate() { + public void thawRate(final boolean isStartPipe) { + super.thawRate(isStartPipe); dataRegionCommitMeter.compareAndSet( null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); schemaRegionCommitMeter.compareAndSet( @@ -202,7 +203,8 @@ public void thawRate() { // Thread-safe & Idempotent @Override - public void freezeRate() { + public void freezeRate(final boolean isStopPipe) { + super.freezeRate(isStopPipe); dataRegionCommitMeter.set(null); schemaRegionCommitMeter.set(null); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java index 0b731dfd4e5d1..1d00abe096359 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java @@ -30,6 +30,7 @@ public abstract class PipeRemainingOperator { private long lastEmptyTimeStamp = System.currentTimeMillis(); private long lastNonEmptyTimeStamp = System.currentTimeMillis(); + protected volatile boolean isStopped = true; //////////////////////////// Tags //////////////////////////// @@ -57,7 +58,7 @@ protected void notifyEmpty() { lastEmptyTimeStamp = System.currentTimeMillis(); if (lastEmptyTimeStamp - lastNonEmptyTimeStamp >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { - freezeRate(); + freezeRate(false); } } @@ -67,12 +68,22 @@ protected void notifyNonEmpty() { lastNonEmptyTimeStamp = System.currentTimeMillis(); if (lastNonEmptyTimeStamp - lastEmptyTimeStamp - >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { - thawRate(); + >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000 + && !isStopped) { + // The stopped pipe's rate should only be thawed by "startPipe" command + thawRate(false); } } - public abstract void thawRate(); + public void thawRate(final boolean isStartPipe) { + if (isStartPipe) { + isStopped = false; + } + } - public abstract void freezeRate(); + public void freezeRate(final boolean isStopPipe) { + if (isStopPipe) { + isStopped = true; + } + } } From 8f25a1a68f7d996e0aa5d00a59b0796a65137aab Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:56:12 +0800 Subject: [PATCH 09/17] adjusting --- .../PipeConfigNodeRemainingTimeOperator.java | 8 ++++-- ...DataNodeRemainingEventAndTimeOperator.java | 8 ++++-- .../pipe/metric/PipeRemainingOperator.java | 26 +++++++++---------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java index fc9f50474e1d9..7a5044b0c6162 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java @@ -110,14 +110,18 @@ void markConfigRegionCommit() { //////////////////////////// Switch //////////////////////////// @Override - public void thawRate(final boolean isStartPipe) { + public synchronized void thawRate(final boolean isStartPipe) { super.thawRate(isStartPipe); + // The stopped pipe's rate should only be thawed by "startPipe" command + if (isStopped) { + return; + } configRegionCommitMeter.compareAndSet( null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); } @Override - public void freezeRate(final boolean isStopPipe) { + public synchronized void freezeRate(final boolean isStopPipe) { super.freezeRate(isStopPipe); configRegionCommitMeter.set(null); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java index 129d9866a4d15..dcab93d535c0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java @@ -193,8 +193,12 @@ void markSchemaRegionCommit() { // Thread-safe & Idempotent @Override - public void thawRate(final boolean isStartPipe) { + public synchronized void thawRate(final boolean isStartPipe) { super.thawRate(isStartPipe); + // The stopped pipe's rate should only be thawed by "startPipe" command + if (isStopped) { + return; + } dataRegionCommitMeter.compareAndSet( null, new Meter(new ExponentialMovingAverages(), Clock.defaultClock())); schemaRegionCommitMeter.compareAndSet( @@ -203,7 +207,7 @@ public void thawRate(final boolean isStartPipe) { // Thread-safe & Idempotent @Override - public void freezeRate(final boolean isStopPipe) { + public synchronized void freezeRate(final boolean isStopPipe) { super.freezeRate(isStopPipe); dataRegionCommitMeter.set(null); schemaRegionCommitMeter.set(null); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java index 1d00abe096359..97f80e5d98533 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/metric/PipeRemainingOperator.java @@ -30,7 +30,7 @@ public abstract class PipeRemainingOperator { private long lastEmptyTimeStamp = System.currentTimeMillis(); private long lastNonEmptyTimeStamp = System.currentTimeMillis(); - protected volatile boolean isStopped = true; + protected boolean isStopped = true; //////////////////////////// Tags //////////////////////////// @@ -51,37 +51,35 @@ protected void setNameAndCreationTime(final String pipeName, final long creation //////////////////////////// Switch //////////////////////////// - protected void notifyEmpty() { + protected void notifyNonEmpty() { final long pipeRemainingTimeCommitAutoSwitchSeconds = PipeConfig.getInstance().getPipeRemainingTimeCommitAutoSwitchSeconds(); - lastEmptyTimeStamp = System.currentTimeMillis(); - if (lastEmptyTimeStamp - lastNonEmptyTimeStamp + lastNonEmptyTimeStamp = System.currentTimeMillis(); + if (lastNonEmptyTimeStamp - lastEmptyTimeStamp >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { - freezeRate(false); + thawRate(false); } } - protected void notifyNonEmpty() { + protected void notifyEmpty() { final long pipeRemainingTimeCommitAutoSwitchSeconds = PipeConfig.getInstance().getPipeRemainingTimeCommitAutoSwitchSeconds(); - lastNonEmptyTimeStamp = System.currentTimeMillis(); - if (lastNonEmptyTimeStamp - lastEmptyTimeStamp - >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000 - && !isStopped) { - // The stopped pipe's rate should only be thawed by "startPipe" command - thawRate(false); + lastEmptyTimeStamp = System.currentTimeMillis(); + if (lastEmptyTimeStamp - lastNonEmptyTimeStamp + >= pipeRemainingTimeCommitAutoSwitchSeconds * 1000) { + freezeRate(false); } } - public void thawRate(final boolean isStartPipe) { + public synchronized void thawRate(final boolean isStartPipe) { if (isStartPipe) { isStopped = false; } } - public void freezeRate(final boolean isStopPipe) { + public synchronized void freezeRate(final boolean isStopPipe) { if (isStopPipe) { isStopped = true; } From cbd997a407bda0fe6089695dc37f5fae781c5665 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:15:38 +0800 Subject: [PATCH 10/17] bug fix --- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 10 ++++++++++ .../manager/pipe/task/PipeConfigNodeTaskStage.java | 6 ------ .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 9 ++------- .../PipeDataNodeRemainingEventAndTimeMetrics.java | 12 ++++++------ .../iotdb/commons/pipe/agent/task/PipeTaskAgent.java | 12 +++++++++++- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java index fb8507be9d401..abfea097dbffa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java @@ -64,6 +64,16 @@ protected boolean isShutdown() { return PipeConfigNodeAgent.runtime().isShutdown(); } + @Override + protected void thawRate(final String pipeName, final long creationTime) { + PipeConfigNodeRemainingTimeMetrics.getInstance().thawRate(pipeName + "_" + creationTime); + } + + @Override + protected void freezeRate(final String pipeName, final long creationTime) { + PipeConfigNodeRemainingTimeMetrics.getInstance().freezeRate(pipeName + "_" + creationTime); + } + @Override protected Map buildPipeTasks(final PipeMeta pipeMetaFromConfigNode) throws IllegalPathException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java index 8a647f649a951..7e676903857ea 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeConfigNodeTaskStage.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.pipe.task.stage.PipeTaskStage; import org.apache.iotdb.confignode.manager.pipe.execution.PipeConfigNodeSubtask; import org.apache.iotdb.confignode.manager.pipe.execution.PipeConfigNodeSubtaskExecutor; -import org.apache.iotdb.confignode.manager.pipe.metric.PipeConfigNodeRemainingTimeMetrics; import org.apache.iotdb.pipe.api.exception.PipeException; import java.util.Map; @@ -64,16 +63,11 @@ public void createSubtask() throws PipeException { @Override public void startSubtask() throws PipeException { - // IoTDBConfigRegionExtractor is started by executor because starting - // here may cause deadlock when triggering snapshot PipeConfigNodeSubtaskExecutor.getInstance().start(subtask.getTaskID()); - PipeConfigNodeRemainingTimeMetrics.getInstance().thawRate(subtask.getTaskID()); } @Override public void stopSubtask() throws PipeException { - // Freeze first to guarantee more precise result - PipeConfigNodeRemainingTimeMetrics.getInstance().freezeRate(subtask.getTaskID()); PipeConfigNodeSubtaskExecutor.getInstance().stop(subtask.getTaskID()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 0e7a2452d0987..427f798d93c96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -238,19 +238,14 @@ private void closeSchemaRegionListeningQueueIfNecessary( } @Override - protected void startPipe(final String pipeName, final long creationTime) { - super.startPipe(pipeName, creationTime); - + protected void thawRate(final String pipeName, final long creationTime) { PipeDataNodeRemainingEventAndTimeMetrics.getInstance().thawRate(pipeName + "_" + creationTime); } @Override - protected void stopPipe(final String pipeName, final long creationTime) { - // Freeze first to guarantee more precise result + protected void freezeRate(final String pipeName, final long creationTime) { PipeDataNodeRemainingEventAndTimeMetrics.getInstance() .freezeRate(pipeName + "_" + creationTime); - - super.stopPipe(pipeName, creationTime); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java index fba9194680508..fc17993304ebd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java @@ -153,9 +153,9 @@ public void register(final IoTDBSchemaRegionExtractor extractor) { public void thawRate(final String pipeID) { if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { - LOGGER.warn( - "Failed to thaw pipe remaining time rate, RemainingEventAndTimeOperator({}) does not exist", - pipeID); + // In dataNode, the "thawRate" may be called when there are no subtasks, and we call + // "startPipe". + // We thaw it later in "startPipeTask". return; } remainingEventAndTimeOperatorMap.get(pipeID).thawRate(true); @@ -163,9 +163,9 @@ public void thawRate(final String pipeID) { public void freezeRate(final String pipeID) { if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) { - LOGGER.warn( - "Failed to freeze pipe remaining time rate, RemainingEventAndTimeOperator({}) does not exist", - pipeID); + // In dataNode, the "freezeRate" may be called when there are no subtasks, and we call + // "stopPipe" after calling "startPipe". + // We do nothing because in that case the rate is not thawed initially return; } remainingEventAndTimeOperatorMap.get(pipeID).freezeRate(true); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index aa0d777d921c5..cc4177da750af 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -272,6 +272,9 @@ private void executeSinglePipeRuntimeMetaChanges( break; case STOPPED: if (Objects.requireNonNull(statusInAgent) == PipeStatus.RUNNING) { + // Only freeze rate for user stopped pipes + // Freeze first to get better results in calculation + freezeRate(pipeStaticMeta.getPipeName(), pipeStaticMeta.getCreationTime()); stopPipe(pipeStaticMeta.getPipeName(), pipeStaticMeta.getCreationTime()); } else { throw new IllegalStateException( @@ -290,6 +293,10 @@ private void executeSinglePipeRuntimeMetaChanges( } } + protected abstract void thawRate(final String pipeName, final long creationTime); + + protected abstract void freezeRate(final String pipeName, final long creationTime); + public TPushPipeMetaRespExceptionMessage handleDropPipe(final String pipeName) { acquireWriteLock(); try { @@ -566,9 +573,11 @@ protected void startPipe(final String pipeName, final long creationTime) { .getConsensusGroupId2TaskMetaMap() .values() .forEach(PipeTaskMeta::clearExceptionMessages); + + thawRate(pipeName, creationTime); } - protected void stopPipe(final String pipeName, final long creationTime) { + private void stopPipe(final String pipeName, final long creationTime) { final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (!checkBeforeStopPipe(existedPipeMeta, pipeName, creationTime)) { @@ -846,6 +855,7 @@ private void startPipeTask(final int consensusGroupId, final PipeStaticMeta pipe final PipeTask pipeTask = pipeTaskManager.getPipeTask(pipeStaticMeta, consensusGroupId); if (pipeTask != null) { pipeTask.start(); + thawRate(pipeStaticMeta.getPipeName(), pipeStaticMeta.getCreationTime()); } } From d305eca6e311197e0d35269ac4c6ab98a4182622 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:16:31 +0800 Subject: [PATCH 11/17] Update PipeTaskAgent.java --- .../org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index cc4177da750af..67bad18710f49 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -538,7 +538,7 @@ protected void dropPipe(final String pipeName) { pipeMetaKeeper.removePipeMeta(pipeName); } - protected void startPipe(final String pipeName, final long creationTime) { + private void startPipe(final String pipeName, final long creationTime) { final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); if (!checkBeforeStartPipe(existedPipeMeta, pipeName, creationTime)) { From 4d120f2febb284bfef4cff5af90dbc02889f6616 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:36:36 +0800 Subject: [PATCH 12/17] avoid always retrying dropped events --- .../event/PipeConfigRegionSnapshotEvent.java | 13 +++- .../event/PipeConfigRegionWritePlanEvent.java | 8 +- .../agent/task/PipeDataNodeTaskAgent.java | 15 ++++ .../async/IoTDBDataRegionAsyncConnector.java | 64 +++++++++------- .../pipe/event/UserDefinedEnrichedEvent.java | 23 +++--- .../common/heartbeat/PipeHeartbeatEvent.java | 15 ++-- .../schema/PipeSchemaRegionSnapshotEvent.java | 14 +++- .../PipeSchemaRegionWritePlanEvent.java | 8 +- .../PipeInsertNodeTabletInsertionEvent.java | 64 +++++++++------- .../tablet/PipeRawTabletInsertionEvent.java | 75 +++++++++++-------- .../common/terminate/PipeTerminateEvent.java | 10 ++- .../tsfile/PipeTsFileInsertionEvent.java | 52 +++++++------ .../event/realtime/PipeRealtimeEvent.java | 11 ++- ...peHistoricalDataRegionTsFileExtractor.java | 5 +- .../PipeRealtimeDataRegionExtractor.java | 7 +- .../assigner/PipeDataRegionAssigner.java | 2 +- .../commons/pipe/event/EnrichedEvent.java | 28 ++++--- .../commons/pipe/event/PipeSnapshotEvent.java | 3 +- .../pipe/event/PipeWritePlanEvent.java | 3 +- .../pipe/event/ProgressReportEvent.java | 7 +- .../IoTDBNonDataRegionExtractor.java | 11 ++- 21 files changed, 270 insertions(+), 168 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java index dcdc0f8f81d9c..56c9d012c3ca4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java @@ -90,7 +90,7 @@ public PipeConfigRegionSnapshotEvent() { public PipeConfigRegionSnapshotEvent( final String snapshotPath, final String templateFilePath, final CNSnapshotFileType type) { - this(snapshotPath, templateFilePath, type, null, null, null); + this(snapshotPath, templateFilePath, type, null, 0, null, null); } public PipeConfigRegionSnapshotEvent( @@ -98,9 +98,15 @@ public PipeConfigRegionSnapshotEvent( final String templateFilePath, final CNSnapshotFileType type, final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern) { - super(pipeName, pipeTaskMeta, pattern, PipeConfigNodeSnapshotResourceManager.getInstance()); + super( + pipeName, + creationTime, + pipeTaskMeta, + pattern, + PipeConfigNodeSnapshotResourceManager.getInstance()); this.snapshotPath = snapshotPath; this.templateFilePath = Objects.nonNull(templateFilePath) ? templateFilePath : ""; this.fileType = type; @@ -157,12 +163,13 @@ public boolean internallyDecreaseResourceReferenceCount(final String holderMessa @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { return new PipeConfigRegionSnapshotEvent( - snapshotPath, templateFilePath, fileType, pipeName, pipeTaskMeta, pattern); + snapshotPath, templateFilePath, fileType, pipeName, creationTime, pipeTaskMeta, pattern); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionWritePlanEvent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionWritePlanEvent.java index c699690746056..0d780bafca1e5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionWritePlanEvent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionWritePlanEvent.java @@ -41,16 +41,17 @@ public PipeConfigRegionWritePlanEvent() { public PipeConfigRegionWritePlanEvent( final ConfigPhysicalPlan configPhysicalPlan, final boolean isGeneratedByPipe) { - this(configPhysicalPlan, null, null, null, isGeneratedByPipe); + this(configPhysicalPlan, null, 0, null, null, isGeneratedByPipe); } public PipeConfigRegionWritePlanEvent( final ConfigPhysicalPlan configPhysicalPlan, final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final boolean isGeneratedByPipe) { - super(pipeName, pipeTaskMeta, pattern, isGeneratedByPipe); + super(pipeName, creationTime, pipeTaskMeta, pattern, isGeneratedByPipe); this.configPhysicalPlan = configPhysicalPlan; } @@ -61,12 +62,13 @@ public ConfigPhysicalPlan getConfigPhysicalPlan() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { return new PipeConfigRegionWritePlanEvent( - configPhysicalPlan, pipeName, pipeTaskMeta, pattern, false); + configPhysicalPlan, pipeName, creationTime, pipeTaskMeta, pattern, false); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 1c3862bb9917d..dec6af7ae3f72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -620,6 +620,21 @@ public void markCompleted(final String pipeName, final int regionId) { } } + ///////////////////////// Useless Retry Detection Logic ///////////////////////// + + public boolean isNonExistPipe(final String pipeName, final long creationTime) { + if (!tryReadLockWithTimeOut(1)) { + return false; + } + try { + return !(pipeMetaKeeper.containsPipeMeta(pipeName) + && pipeMetaKeeper.getPipeMetaByPipeName(pipeName).getStaticMeta().getCreationTime() + == creationTime); + } finally { + releaseReadLock(); + } + } + ///////////////////////// Utils ///////////////////////// public Set getPipeTaskRegionIdSet(final String pipeName, final long creationTime) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java index 3edcedc31d8b2..0c1d542d504c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient; import org.apache.iotdb.commons.pipe.connector.protocol.IoTDBConnector; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.pipe.connector.client.IoTDBDataNodeAsyncClientManager; import org.apache.iotdb.db.pipe.connector.payload.evolvable.builder.PipeEventBatch; import org.apache.iotdb.db.pipe.connector.payload.evolvable.builder.PipeTransferBatchReqBuilder; @@ -371,33 +372,46 @@ private void transferQueuedEventsIfNecessary() throws Exception { final Event peekedEvent = retryEventQueue.peek(); - if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) { - retryConnector.transfer((PipeInsertNodeTabletInsertionEvent) peekedEvent); - } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) { - retryConnector.transfer((PipeRawTabletInsertionEvent) peekedEvent); - } else if (peekedEvent instanceof PipeTsFileInsertionEvent) { - retryConnector.transfer((PipeTsFileInsertionEvent) peekedEvent); - } else { - LOGGER.warn( - "IoTDBThriftAsyncConnector does not support transfer generic event: {}.", - peekedEvent); - } + try { + if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) { + retryConnector.transfer((PipeInsertNodeTabletInsertionEvent) peekedEvent); + } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) { + retryConnector.transfer((PipeRawTabletInsertionEvent) peekedEvent); + } else if (peekedEvent instanceof PipeTsFileInsertionEvent) { + retryConnector.transfer((PipeTsFileInsertionEvent) peekedEvent); + } else { + LOGGER.warn( + "IoTDBThriftAsyncConnector does not support transfer generic event: {}.", + peekedEvent); + } - if (peekedEvent instanceof EnrichedEvent) { - ((EnrichedEvent) peekedEvent) - .decreaseReferenceCount(IoTDBDataRegionAsyncConnector.class.getName(), true); - } + if (peekedEvent instanceof EnrichedEvent) { + ((EnrichedEvent) peekedEvent) + .decreaseReferenceCount(IoTDBDataRegionAsyncConnector.class.getName(), true); + } - final Event polledEvent = retryEventQueue.poll(); - if (polledEvent != peekedEvent) { - LOGGER.error( - "The event polled from the queue is not the same as the event peeked from the queue. " - + "Peeked event: {}, polled event: {}.", - peekedEvent, - polledEvent); - } - if (polledEvent != null && LOGGER.isDebugEnabled()) { - LOGGER.debug("Polled event {} from retry queue.", polledEvent); + final Event polledEvent = retryEventQueue.poll(); + if (polledEvent != peekedEvent) { + LOGGER.error( + "The event polled from the queue is not the same as the event peeked from the queue. " + + "Peeked event: {}, polled event: {}.", + peekedEvent, + polledEvent); + } + if (polledEvent != null && LOGGER.isDebugEnabled()) { + LOGGER.debug("Polled event {} from retry queue.", polledEvent); + } + } catch (final Exception e) { + if (peekedEvent instanceof EnrichedEvent + && PipeAgent.task() + .isNonExistPipe( + ((EnrichedEvent) peekedEvent).getPipeName(), + ((EnrichedEvent) peekedEvent).getCreationTime())) { + ((EnrichedEvent) peekedEvent) + .clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName()); + } else { + throw e; + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/UserDefinedEnrichedEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/UserDefinedEnrichedEvent.java index ff6e34fffcf09..4803158db2e06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/UserDefinedEnrichedEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/UserDefinedEnrichedEvent.java @@ -31,7 +31,7 @@ public class UserDefinedEnrichedEvent extends EnrichedEvent { private final UserDefinedEvent userDefinedEvent; private final EnrichedEvent enrichedEvent; - public static Event maybeOf(Event event) { + public static Event maybeOf(final Event event) { return event instanceof UserDefinedEvent && ((UserDefinedEvent) event).getSourceEvent() instanceof EnrichedEvent ? new UserDefinedEnrichedEvent( @@ -39,9 +39,11 @@ public static Event maybeOf(Event event) { : event; } - private UserDefinedEnrichedEvent(UserDefinedEvent userDefinedEvent, EnrichedEvent enrichedEvent) { + private UserDefinedEnrichedEvent( + final UserDefinedEvent userDefinedEvent, final EnrichedEvent enrichedEvent) { super( enrichedEvent.getPipeName(), + enrichedEvent.getCreationTime(), enrichedEvent.getPipeTaskMeta(), enrichedEvent.getPipePattern(), enrichedEvent.getStartTime(), @@ -55,12 +57,12 @@ public UserDefinedEvent getUserDefinedEvent() { } @Override - public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { + public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { return enrichedEvent.internallyIncreaseResourceReferenceCount(holderMessage); } @Override - public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { + public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { return enrichedEvent.internallyDecreaseResourceReferenceCount(holderMessage); } @@ -71,13 +73,14 @@ public ProgressIndex getProgressIndex() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { return enrichedEvent.shallowCopySelfAndBindPipeTaskMetaForProgressReport( - pipeName, pipeTaskMeta, pattern, startTime, endTime); + pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java index 9738f86a5dd40..f82d799b7d3d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java @@ -38,7 +38,6 @@ public class PipeHeartbeatEvent extends EnrichedEvent { private static final Logger LOGGER = LoggerFactory.getLogger(PipeHeartbeatEvent.class); private final String dataRegionId; - private String pipeName; private long timePublished; private long timeAssigned; @@ -60,18 +59,19 @@ public class PipeHeartbeatEvent extends EnrichedEvent { private final boolean shouldPrintMessage; public PipeHeartbeatEvent(final String dataRegionId, final boolean shouldPrintMessage) { - super(null, null, null, Long.MIN_VALUE, Long.MAX_VALUE); + super(null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE); this.dataRegionId = dataRegionId; this.shouldPrintMessage = shouldPrintMessage; } public PipeHeartbeatEvent( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final String dataRegionId, final long timePublished, final boolean shouldPrintMessage) { - super(pipeName, pipeTaskMeta, null, Long.MIN_VALUE, Long.MAX_VALUE); + super(pipeName, creationTime, pipeTaskMeta, null, Long.MIN_VALUE, Long.MAX_VALUE); this.dataRegionId = dataRegionId; this.timePublished = timePublished; this.shouldPrintMessage = shouldPrintMessage; @@ -100,6 +100,7 @@ public ProgressIndex getProgressIndex() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, @@ -107,7 +108,7 @@ public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( // Should record PipeTaskMeta, for sometimes HeartbeatEvents should report exceptions. // Here we ignore parameters `pattern`, `startTime`, and `endTime`. return new PipeHeartbeatEvent( - pipeName, pipeTaskMeta, dataRegionId, timePublished, shouldPrintMessage); + pipeName, creationTime, pipeTaskMeta, dataRegionId, timePublished, shouldPrintMessage); } @Override @@ -128,12 +129,6 @@ public boolean isShouldPrintMessage() { /////////////////////////////// Delay Reporting /////////////////////////////// - public void bindPipeName(final String pipeName) { - if (shouldPrintMessage) { - this.pipeName = pipeName; - } - } - public void onPublished() { if (shouldPrintMessage) { timePublished = System.currentTimeMillis(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionSnapshotEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionSnapshotEvent.java index d647285bdc633..60bbf95feee66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionSnapshotEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionSnapshotEvent.java @@ -78,7 +78,7 @@ public PipeSchemaRegionSnapshotEvent() { public PipeSchemaRegionSnapshotEvent( final String mTreeSnapshotPath, final String tagLogSnapshotPath, final String databaseName) { - this(mTreeSnapshotPath, tagLogSnapshotPath, databaseName, null, null, null); + this(mTreeSnapshotPath, tagLogSnapshotPath, databaseName, null, 0, null, null); } public PipeSchemaRegionSnapshotEvent( @@ -86,9 +86,10 @@ public PipeSchemaRegionSnapshotEvent( final String tagLogSnapshotPath, final String databaseName, final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern) { - super(pipeName, pipeTaskMeta, pattern, PipeResourceManager.snapshot()); + super(pipeName, creationTime, pipeTaskMeta, pattern, PipeResourceManager.snapshot()); this.mTreeSnapshotPath = mTreeSnapshotPath; this.tagLogSnapshotPath = Objects.nonNull(tagLogSnapshotPath) ? tagLogSnapshotPath : ""; this.databaseName = databaseName; @@ -145,12 +146,19 @@ public boolean internallyDecreaseResourceReferenceCount(final String holderMessa @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { return new PipeSchemaRegionSnapshotEvent( - mTreeSnapshotPath, tagLogSnapshotPath, databaseName, pipeName, pipeTaskMeta, pattern); + mTreeSnapshotPath, + tagLogSnapshotPath, + databaseName, + pipeName, + creationTime, + pipeTaskMeta, + pattern); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionWritePlanEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionWritePlanEvent.java index 6e0b5cccf8660..d6eefe086e55c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionWritePlanEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/schema/PipeSchemaRegionWritePlanEvent.java @@ -40,16 +40,17 @@ public PipeSchemaRegionWritePlanEvent() { } public PipeSchemaRegionWritePlanEvent(final PlanNode planNode, final boolean isGeneratedByPipe) { - this(planNode, null, null, null, isGeneratedByPipe); + this(planNode, null, 0, null, null, isGeneratedByPipe); } public PipeSchemaRegionWritePlanEvent( final PlanNode planNode, final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final boolean isGeneratedByPipe) { - super(pipeName, pipeTaskMeta, pattern, isGeneratedByPipe); + super(pipeName, creationTime, pipeTaskMeta, pattern, isGeneratedByPipe); this.planNode = planNode; } @@ -60,12 +61,13 @@ public PlanNode getPlanNode() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { return new PipeSchemaRegionWritePlanEvent( - planNode, pipeName, pipeTaskMeta, pattern, isGeneratedByPipe); + planNode, pipeName, creationTime, pipeTaskMeta, pattern, isGeneratedByPipe); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java index 11711fa1d034e..ca0c1c23dcd97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java @@ -68,11 +68,11 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent private ProgressIndex progressIndex; public PipeInsertNodeTabletInsertionEvent( - WALEntryHandler walEntryHandler, - PartialPath devicePath, - ProgressIndex progressIndex, - boolean isAligned, - boolean isGeneratedByPipe) { + final WALEntryHandler walEntryHandler, + final PartialPath devicePath, + final ProgressIndex progressIndex, + final boolean isAligned, + final boolean isGeneratedByPipe) { this( walEntryHandler, devicePath, @@ -80,6 +80,7 @@ public PipeInsertNodeTabletInsertionEvent( isAligned, isGeneratedByPipe, null, + 0, null, null, Long.MIN_VALUE, @@ -87,17 +88,18 @@ public PipeInsertNodeTabletInsertionEvent( } private PipeInsertNodeTabletInsertionEvent( - WALEntryHandler walEntryHandler, - PartialPath devicePath, - ProgressIndex progressIndex, - boolean isAligned, - boolean isGeneratedByPipe, - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { - super(pipeName, pipeTaskMeta, pattern, startTime, endTime); + final WALEntryHandler walEntryHandler, + final PartialPath devicePath, + final ProgressIndex progressIndex, + final boolean isAligned, + final boolean isGeneratedByPipe, + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { + super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); this.walEntryHandler = walEntryHandler; // Record device path here so there's no need to get it from InsertNode cache later. this.devicePath = devicePath; @@ -129,11 +131,11 @@ public String getDeviceId() { /////////////////////////// EnrichedEvent /////////////////////////// @Override - public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { + public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { try { PipeResourceManager.wal().pin(walEntryHandler); return true; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( String.format( "Increase reference count for memtable %d error. Holder Message: %s", @@ -144,7 +146,7 @@ public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { } @Override - public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { + public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { try { PipeResourceManager.wal().unpin(walEntryHandler); // Release the containers' memory. @@ -153,7 +155,7 @@ public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { dataContainers = null; } return true; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( String.format( "Decrease reference count for memtable %d error. Holder Message: %s", @@ -164,7 +166,7 @@ public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { } @Override - public void bindProgressIndex(ProgressIndex progressIndex) { + public void bindProgressIndex(final ProgressIndex progressIndex) { this.progressIndex = progressIndex; } @@ -175,11 +177,12 @@ public ProgressIndex getProgressIndex() { @Override public PipeInsertNodeTabletInsertionEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { return new PipeInsertNodeTabletInsertionEvent( walEntryHandler, devicePath, @@ -187,6 +190,7 @@ public PipeInsertNodeTabletInsertionEvent shallowCopySelfAndBindPipeTaskMetaForP isAligned, isGeneratedByPipe, pipeName, + creationTime, pipeTaskMeta, pattern, startTime, @@ -224,7 +228,7 @@ public boolean mayEventTimeOverlappedWithTimeRange() { throw new UnSupportedDataTypeException( String.format("InsertNode type %s is not supported.", insertNode.getClass().getName())); } - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Exception occurred when determining the event time of PipeInsertNodeTabletInsertionEvent({}) overlaps with the time range: [{}, {}]. Returning true to ensure data integrity.", this, @@ -238,7 +242,8 @@ public boolean mayEventTimeOverlappedWithTimeRange() { /////////////////////////// TabletInsertionEvent /////////////////////////// @Override - public Iterable processRowByRow(BiConsumer consumer) { + public Iterable processRowByRow( + final BiConsumer consumer) { return initDataContainers().stream() .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processRowByRow(consumer)) .flatMap(Collection::stream) @@ -246,7 +251,8 @@ public Iterable processRowByRow(BiConsumer processTablet(BiConsumer consumer) { + public Iterable processTablet( + final BiConsumer consumer) { return initDataContainers().stream() .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processTablet(consumer)) .flatMap(Collection::stream) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java index e792f58098195..239f73b3c328d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java @@ -52,16 +52,17 @@ public class PipeRawTabletInsertionEvent extends EnrichedEvent implements Tablet private ProgressIndex overridingProgressIndex; private PipeRawTabletInsertionEvent( - Tablet tablet, - boolean isAligned, - EnrichedEvent sourceEvent, - boolean needToReport, - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { - super(pipeName, pipeTaskMeta, pattern, startTime, endTime); + final Tablet tablet, + final boolean isAligned, + final EnrichedEvent sourceEvent, + final boolean needToReport, + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { + super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); this.tablet = Objects.requireNonNull(tablet); this.isAligned = isAligned; this.sourceEvent = sourceEvent; @@ -69,18 +70,20 @@ private PipeRawTabletInsertionEvent( } public PipeRawTabletInsertionEvent( - Tablet tablet, - boolean isAligned, - String pipeName, - PipeTaskMeta pipeTaskMeta, - EnrichedEvent sourceEvent, - boolean needToReport) { + final Tablet tablet, + final boolean isAligned, + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final EnrichedEvent sourceEvent, + final boolean needToReport) { this( tablet, isAligned, sourceEvent, needToReport, pipeName, + creationTime, pipeTaskMeta, null, Long.MIN_VALUE, @@ -88,28 +91,30 @@ public PipeRawTabletInsertionEvent( } @TestOnly - public PipeRawTabletInsertionEvent(Tablet tablet, boolean isAligned) { - this(tablet, isAligned, null, false, null, null, null, Long.MIN_VALUE, Long.MAX_VALUE); + public PipeRawTabletInsertionEvent(final Tablet tablet, final boolean isAligned) { + this(tablet, isAligned, null, false, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE); } @TestOnly - public PipeRawTabletInsertionEvent(Tablet tablet, boolean isAligned, PipePattern pattern) { - this(tablet, isAligned, null, false, null, null, pattern, Long.MIN_VALUE, Long.MAX_VALUE); + public PipeRawTabletInsertionEvent( + final Tablet tablet, final boolean isAligned, final PipePattern pattern) { + this(tablet, isAligned, null, false, null, 0, null, pattern, Long.MIN_VALUE, Long.MAX_VALUE); } @TestOnly - public PipeRawTabletInsertionEvent(Tablet tablet, long startTime, long endTime) { - this(tablet, false, null, false, null, null, null, startTime, endTime); + public PipeRawTabletInsertionEvent( + final Tablet tablet, final long startTime, final long endTime) { + this(tablet, false, null, false, null, 0, null, null, startTime, endTime); } @Override - public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { + public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { allocatedMemoryBlock = PipeResourceManager.memory().forceAllocateWithRetry(tablet); return true; } @Override - public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { + public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { allocatedMemoryBlock.close(); // Record the deviceId before the memory is released, @@ -130,7 +135,7 @@ protected void reportProgress() { } @Override - public void bindProgressIndex(ProgressIndex overridingProgressIndex) { + public void bindProgressIndex(final ProgressIndex overridingProgressIndex) { // Normally not all events need to report progress, but if the overridingProgressIndex // is given, indicating that the progress needs to be reported. if (Objects.nonNull(overridingProgressIndex)) { @@ -152,17 +157,19 @@ public ProgressIndex getProgressIndex() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { return new PipeRawTabletInsertionEvent( tablet, isAligned, sourceEvent, needToReport, pipeName, + creationTime, pipeTaskMeta, pattern, startTime, @@ -196,7 +203,8 @@ public String getDeviceId() { /////////////////////////// TabletInsertionEvent /////////////////////////// @Override - public Iterable processRowByRow(BiConsumer consumer) { + public Iterable processRowByRow( + final BiConsumer consumer) { if (dataContainer == null) { dataContainer = new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, pipePattern); @@ -205,7 +213,8 @@ public Iterable processRowByRow(BiConsumer processTablet(BiConsumer consumer) { + public Iterable processTablet( + final BiConsumer consumer) { if (dataContainer == null) { dataContainer = new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, pipePattern); @@ -241,7 +250,7 @@ public long count() { public PipeRawTabletInsertionEvent parseEventWithPatternOrTime() { return new PipeRawTabletInsertionEvent( - convertToTablet(), isAligned, pipeName, pipeTaskMeta, this, needToReport); + convertToTablet(), isAligned, pipeName, creationTime, pipeTaskMeta, this, needToReport); } public boolean hasNoNeedParsingAndIsEmpty() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java index c1252fe668f96..cc423f1bf3d65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java @@ -38,8 +38,11 @@ public class PipeTerminateEvent extends EnrichedEvent { private final int dataRegionId; public PipeTerminateEvent( - final String pipeName, final PipeTaskMeta pipeTaskMeta, final int dataRegionId) { - super(pipeName, pipeTaskMeta, null, Long.MIN_VALUE, Long.MAX_VALUE); + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final int dataRegionId) { + super(pipeName, creationTime, pipeTaskMeta, null, Long.MIN_VALUE, Long.MAX_VALUE); this.dataRegionId = dataRegionId; } @@ -61,13 +64,14 @@ public ProgressIndex getProgressIndex() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { // Should record PipeTaskMeta, for the terminateEvent shall report progress to // notify the pipeTask it's completed. - return new PipeTerminateEvent(pipeName, pipeTaskMeta, dataRegionId); + return new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 55739bf8f3291..a7c0cc5c255d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -59,7 +59,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent implements TsFileIns private TsFileInsertionDataContainer dataContainer; public PipeTsFileInsertionEvent( - TsFileResource resource, boolean isLoaded, boolean isGeneratedByPipe) { + final TsFileResource resource, final boolean isLoaded, final boolean isGeneratedByPipe) { // The modFile must be copied before the event is assigned to the listening pipes this( resource, @@ -67,6 +67,7 @@ public PipeTsFileInsertionEvent( isLoaded, isGeneratedByPipe, null, + 0, null, null, Long.MIN_VALUE, @@ -74,16 +75,17 @@ public PipeTsFileInsertionEvent( } public PipeTsFileInsertionEvent( - TsFileResource resource, - boolean isWithMod, - boolean isLoaded, - boolean isGeneratedByPipe, - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { - super(pipeName, pipeTaskMeta, pattern, startTime, endTime); + final TsFileResource resource, + final boolean isWithMod, + final boolean isLoaded, + final boolean isGeneratedByPipe, + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { + super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); this.resource = resource; tsFile = resource.getTsFile(); @@ -186,14 +188,14 @@ public long getFileStartTime() { /////////////////////////// EnrichedEvent /////////////////////////// @Override - public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { + public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { try { tsFile = PipeResourceManager.tsfile().increaseFileReference(tsFile, true, resource); if (isWithMod) { modFile = PipeResourceManager.tsfile().increaseFileReference(modFile, false, null); } return true; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( String.format( "Increase reference count for TsFile %s or modFile %s error. Holder Message: %s", @@ -204,14 +206,14 @@ public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { } @Override - public boolean internallyDecreaseResourceReferenceCount(String holderMessage) { + public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { try { PipeResourceManager.tsfile().decreaseFileReference(tsFile); if (isWithMod) { PipeResourceManager.tsfile().decreaseFileReference(modFile); } return true; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( String.format( "Decrease reference count for TsFile %s error. Holder Message: %s", @@ -231,7 +233,7 @@ public ProgressIndex getProgressIndex() { return MinimumProgressIndex.INSTANCE; } return resource.getMaxProgressIndexAfterClose(); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { LOGGER.warn( String.format( "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath())); @@ -242,17 +244,19 @@ public ProgressIndex getProgressIndex() { @Override public PipeTsFileInsertionEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime) { + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { return new PipeTsFileInsertionEvent( resource, isWithMod, isLoaded, isGeneratedByPipe, pipeName, + creationTime, pipeTaskMeta, pattern, startTime, @@ -305,7 +309,7 @@ public Iterable toTabletInsertionEvents() { return Collections.emptyList(); } return initDataContainer().toTabletInsertionEvents(); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { Thread.currentThread().interrupt(); close(); @@ -325,7 +329,7 @@ private TsFileInsertionDataContainer initDataContainer() { tsFile, pipePattern, startTime, endTime, pipeTaskMeta, this); } return dataContainer; - } catch (IOException e) { + } catch (final IOException e) { close(); final String errorMsg = String.format("Read TsFile %s error.", resource.getTsFilePath()); @@ -334,7 +338,7 @@ private TsFileInsertionDataContainer initDataContainer() { } } - public long count(boolean skipReportOnCommit) throws IOException { + public long count(final boolean skipReportOnCommit) throws IOException { long count = 0; if (shouldParseTime()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java index 77df1f0292be4..ce8e7d9f0fa94 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java @@ -58,7 +58,13 @@ public PipeRealtimeEvent( // PipeTaskMeta is used to report the progress of the event, the PipeRealtimeEvent // is only used in the realtime event extractor, which does not need to report the progress // of the event, so the pipeTaskMeta is always null. - super(event != null ? event.getPipeName() : null, pipeTaskMeta, pattern, startTime, endTime); + super( + event != null ? event.getPipeName() : null, + event != null ? event.getCreationTime() : 0, + pipeTaskMeta, + pattern, + startTime, + endTime); this.event = event; this.tsFileEpoch = tsFileEpoch; @@ -140,13 +146,14 @@ public void skipParsingTime() { @Override public PipeRealtimeEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { return new PipeRealtimeEvent( event.shallowCopySelfAndBindPipeTaskMetaForProgressReport( - pipeName, pipeTaskMeta, pattern, startTime, endTime), + pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime), this.tsFileEpoch, this.device2Measurements, pipeTaskMeta, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index b1e2649e6db70..1ed78ea804596 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -94,6 +94,8 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa private static final long PIPE_MIN_FLUSH_INTERVAL_IN_MS = 2000; private String pipeName; + private long creationTime; + private PipeTaskMeta pipeTaskMeta; private ProgressIndex startIndex; @@ -250,6 +252,7 @@ public void customize( (PipeTaskExtractorRuntimeEnvironment) configuration.getRuntimeEnvironment(); pipeName = environment.getPipeName(); + creationTime = environment.getCreationTime(); pipeTaskMeta = environment.getPipeTaskMeta(); startIndex = environment.getPipeTaskMeta().getProgressIndex(); @@ -563,7 +566,7 @@ public synchronized Event supply() { if (resource == null) { isTerminateSignalSent = true; final PipeTerminateEvent terminateEvent = - new PipeTerminateEvent(pipeName, pipeTaskMeta, dataRegionId); + new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId); terminateEvent.increaseReferenceCount( PipeHistoricalDataRegionTsFileExtractor.class.getName()); return terminateEvent; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java index c4bec4246e483..c26db376da096 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java @@ -74,6 +74,7 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor { LoggerFactory.getLogger(PipeRealtimeDataRegionExtractor.class); protected String pipeName; + protected long creationTime; protected String dataRegionId; protected PipeTaskMeta pipeTaskMeta; @@ -167,7 +168,7 @@ public void customize( // indexed by the taskID of IoTDBDataRegionExtractor. To avoid PipeRealtimeDataRegionExtractor // holding a reference to IoTDBDataRegionExtractor, the taskID should be constructed to // match that of IoTDBDataRegionExtractor. - final long creationTime = environment.getCreationTime(); + creationTime = environment.getCreationTime(); taskID = pipeName + "_" + dataRegionId + "_" + creationTime; pipePattern = PipePattern.parsePipePatternFromSourceParameters(parameters); @@ -387,6 +388,10 @@ public final String getPipeName() { return pipeName; } + public final long getCreationTime() { + return creationTime; + } + public final PipeTaskMeta getPipeTaskMeta() { return pipeTaskMeta; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java index e85fd5b0a1d15..16f9ebf61d922 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java @@ -76,6 +76,7 @@ public void assignToExtractor(PipeRealtimeEvent event, long sequence, boolean en final PipeRealtimeEvent copiedEvent = event.shallowCopySelfAndBindPipeTaskMetaForProgressReport( extractor.getPipeName(), + extractor.getCreationTime(), extractor.getPipeTaskMeta(), extractor.getPipePattern(), extractor.getRealtimeDataExtractionStartTime(), @@ -90,7 +91,6 @@ public void assignToExtractor(PipeRealtimeEvent event, long sequence, boolean en extractor.extract(copiedEvent); if (innerEvent instanceof PipeHeartbeatEvent) { - ((PipeHeartbeatEvent) innerEvent).bindPipeName(extractor.getPipeName()); ((PipeHeartbeatEvent) innerEvent).onAssigned(); } }); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java index b43e37788cd46..e7e915aa09ce5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java @@ -48,6 +48,7 @@ public abstract class EnrichedEvent implements Event { protected final AtomicBoolean isReleased; protected final String pipeName; + protected final long creationTime; protected final PipeTaskMeta pipeTaskMeta; protected String committerKey; @@ -67,6 +68,7 @@ public abstract class EnrichedEvent implements Event { protected EnrichedEvent( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pipePattern, final long startTime, @@ -74,6 +76,7 @@ protected EnrichedEvent( referenceCount = new AtomicInteger(0); isReleased = new AtomicBoolean(false); this.pipeName = pipeName; + this.creationTime = creationTime; this.pipeTaskMeta = pipeTaskMeta; this.pipePattern = pipePattern; this.startTime = startTime; @@ -129,7 +132,7 @@ public boolean increaseReferenceCount(final String holderMessage) { * {@code false} if the {@link EnrichedEvent} is not controlled by the invoker, which means * the data stored in the event is not safe to use */ - public abstract boolean internallyIncreaseResourceReferenceCount(String holderMessage); + public abstract boolean internallyIncreaseResourceReferenceCount(final String holderMessage); /** * Decrease the {@link EnrichedEvent#referenceCount} of this {@link EnrichedEvent} by 1. If the @@ -208,7 +211,7 @@ public boolean clearReferenceCount(final String holderMessage) { * @return {@code true} if the {@link EnrichedEvent#referenceCount} is decreased successfully, * {@code true} otherwise */ - public abstract boolean internallyDecreaseResourceReferenceCount(String holderMessage); + public abstract boolean internallyDecreaseResourceReferenceCount(final String holderMessage); protected void reportProgress() { if (pipeTaskMeta != null) { @@ -245,6 +248,10 @@ public final String getPipeName() { return pipeName; } + public final long getCreationTime() { + return creationTime; + } + /** * Get the pattern string of this {@link EnrichedEvent}. * @@ -291,11 +298,12 @@ public boolean shouldParseTime() { } public abstract EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( - String pipeName, - PipeTaskMeta pipeTaskMeta, - PipePattern pattern, - long startTime, - long endTime); + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime); public PipeTaskMeta getPipeTaskMeta() { return pipeTaskMeta; @@ -315,7 +323,7 @@ public void setCommitterKeyAndCommitId(final String committerKey, final long com this.commitId = commitId; } - public void setRebootTimes(int rebootTimes) { + public void setRebootTimes(final int rebootTimes) { this.rebootTimes = rebootTimes; } @@ -345,14 +353,14 @@ public boolean isReleased() { * Used for pipeConsensus. In PipeConsensus, we only need committerKey, commitId and rebootTimes * to uniquely identify an event */ - public boolean equalsInPipeConsensus(Object o) { + public boolean equalsInPipeConsensus(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - EnrichedEvent otherEvent = (EnrichedEvent) o; + final EnrichedEvent otherEvent = (EnrichedEvent) o; return Objects.equals(committerKey, otherEvent.committerKey) && commitId == otherEvent.commitId && rebootTimes == otherEvent.rebootTimes; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeSnapshotEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeSnapshotEvent.java index fa26319e6dc00..f130a83c0ee16 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeSnapshotEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeSnapshotEvent.java @@ -35,10 +35,11 @@ public abstract class PipeSnapshotEvent extends EnrichedEvent implements Seriali protected PipeSnapshotEvent( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final PipeSnapshotResourceManager resourceManager) { - super(pipeName, pipeTaskMeta, pattern, Long.MIN_VALUE, Long.MAX_VALUE); + super(pipeName, creationTime, pipeTaskMeta, pattern, Long.MIN_VALUE, Long.MAX_VALUE); this.resourceManager = resourceManager; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeWritePlanEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeWritePlanEvent.java index d37aee82aaad8..c553e6ce99625 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeWritePlanEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeWritePlanEvent.java @@ -31,10 +31,11 @@ public abstract class PipeWritePlanEvent extends EnrichedEvent implements Serial protected PipeWritePlanEvent( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final boolean isGeneratedByPipe) { - super(pipeName, pipeTaskMeta, pattern, Long.MIN_VALUE, Long.MAX_VALUE); + super(pipeName, creationTime, pipeTaskMeta, pattern, Long.MIN_VALUE, Long.MAX_VALUE); this.isGeneratedByPipe = isGeneratedByPipe; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/ProgressReportEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/ProgressReportEvent.java index 2851b03593c15..4f00c89c5e61d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/ProgressReportEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/ProgressReportEvent.java @@ -34,11 +34,12 @@ public class ProgressReportEvent extends EnrichedEvent { public ProgressReportEvent( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pipePattern, final long startTime, final long endTime) { - super(pipeName, pipeTaskMeta, pipePattern, startTime, endTime); + super(pipeName, creationTime, pipeTaskMeta, pipePattern, startTime, endTime); } @Override @@ -64,11 +65,13 @@ public ProgressIndex getProgressIndex() { @Override public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( final String pipeName, + final long creationTime, final PipeTaskMeta pipeTaskMeta, final PipePattern pattern, final long startTime, final long endTime) { - return new ProgressReportEvent(pipeName, pipeTaskMeta, pattern, startTime, endTime); + return new ProgressReportEvent( + pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBNonDataRegionExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBNonDataRegionExtractor.java index 9d6ce0ffde813..a67e818be983a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBNonDataRegionExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBNonDataRegionExtractor.java @@ -154,7 +154,12 @@ public EnrichedEvent supply() throws Exception { historicalEvents .remove(0) .shallowCopySelfAndBindPipeTaskMetaForProgressReport( - pipeName, pipeTaskMeta, pipePattern, Long.MIN_VALUE, Long.MAX_VALUE); + pipeName, + creationTime, + pipeTaskMeta, + pipePattern, + Long.MIN_VALUE, + Long.MAX_VALUE); if (historicalEvents.isEmpty()) { // We only report progress for the last snapshot event. @@ -180,7 +185,7 @@ public EnrichedEvent supply() throws Exception { || (!isForwardingPipeRequests && realtimeEvent.isGeneratedByPipe())) { final ProgressReportEvent event = new ProgressReportEvent( - pipeName, pipeTaskMeta, pipePattern, Long.MIN_VALUE, Long.MAX_VALUE); + pipeName, creationTime, pipeTaskMeta, pipePattern, Long.MIN_VALUE, Long.MAX_VALUE); event.bindProgressIndex(new MetaProgressIndex(iterator.getNextIndex() - 1)); event.increaseReferenceCount(IoTDBNonDataRegionExtractor.class.getName()); return event; @@ -189,7 +194,7 @@ public EnrichedEvent supply() throws Exception { realtimeEvent = (PipeWritePlanEvent) realtimeEvent.shallowCopySelfAndBindPipeTaskMetaForProgressReport( - pipeName, pipeTaskMeta, pipePattern, Long.MIN_VALUE, Long.MAX_VALUE); + pipeName, creationTime, pipeTaskMeta, pipePattern, Long.MIN_VALUE, Long.MAX_VALUE); realtimeEvent.bindProgressIndex(new MetaProgressIndex(iterator.getNextIndex() - 1)); realtimeEvent.increaseReferenceCount(IoTDBNonDataRegionExtractor.class.getName()); return realtimeEvent; From eb3645c99eab73be8fcd6a696f548e3e9f1720bb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:04:52 +0800 Subject: [PATCH 13/17] partial revert --- .../agent/task/PipeDataNodeTaskAgent.java | 15 ----- .../async/IoTDBDataRegionAsyncConnector.java | 64 ++++++++----------- 2 files changed, 25 insertions(+), 54 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index dec6af7ae3f72..1c3862bb9917d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -620,21 +620,6 @@ public void markCompleted(final String pipeName, final int regionId) { } } - ///////////////////////// Useless Retry Detection Logic ///////////////////////// - - public boolean isNonExistPipe(final String pipeName, final long creationTime) { - if (!tryReadLockWithTimeOut(1)) { - return false; - } - try { - return !(pipeMetaKeeper.containsPipeMeta(pipeName) - && pipeMetaKeeper.getPipeMetaByPipeName(pipeName).getStaticMeta().getCreationTime() - == creationTime); - } finally { - releaseReadLock(); - } - } - ///////////////////////// Utils ///////////////////////// public Set getPipeTaskRegionIdSet(final String pipeName, final long creationTime) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java index 0c1d542d504c3..3edcedc31d8b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient; import org.apache.iotdb.commons.pipe.connector.protocol.IoTDBConnector; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; -import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.pipe.connector.client.IoTDBDataNodeAsyncClientManager; import org.apache.iotdb.db.pipe.connector.payload.evolvable.builder.PipeEventBatch; import org.apache.iotdb.db.pipe.connector.payload.evolvable.builder.PipeTransferBatchReqBuilder; @@ -372,46 +371,33 @@ private void transferQueuedEventsIfNecessary() throws Exception { final Event peekedEvent = retryEventQueue.peek(); - try { - if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) { - retryConnector.transfer((PipeInsertNodeTabletInsertionEvent) peekedEvent); - } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) { - retryConnector.transfer((PipeRawTabletInsertionEvent) peekedEvent); - } else if (peekedEvent instanceof PipeTsFileInsertionEvent) { - retryConnector.transfer((PipeTsFileInsertionEvent) peekedEvent); - } else { - LOGGER.warn( - "IoTDBThriftAsyncConnector does not support transfer generic event: {}.", - peekedEvent); - } + if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) { + retryConnector.transfer((PipeInsertNodeTabletInsertionEvent) peekedEvent); + } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) { + retryConnector.transfer((PipeRawTabletInsertionEvent) peekedEvent); + } else if (peekedEvent instanceof PipeTsFileInsertionEvent) { + retryConnector.transfer((PipeTsFileInsertionEvent) peekedEvent); + } else { + LOGGER.warn( + "IoTDBThriftAsyncConnector does not support transfer generic event: {}.", + peekedEvent); + } - if (peekedEvent instanceof EnrichedEvent) { - ((EnrichedEvent) peekedEvent) - .decreaseReferenceCount(IoTDBDataRegionAsyncConnector.class.getName(), true); - } + if (peekedEvent instanceof EnrichedEvent) { + ((EnrichedEvent) peekedEvent) + .decreaseReferenceCount(IoTDBDataRegionAsyncConnector.class.getName(), true); + } - final Event polledEvent = retryEventQueue.poll(); - if (polledEvent != peekedEvent) { - LOGGER.error( - "The event polled from the queue is not the same as the event peeked from the queue. " - + "Peeked event: {}, polled event: {}.", - peekedEvent, - polledEvent); - } - if (polledEvent != null && LOGGER.isDebugEnabled()) { - LOGGER.debug("Polled event {} from retry queue.", polledEvent); - } - } catch (final Exception e) { - if (peekedEvent instanceof EnrichedEvent - && PipeAgent.task() - .isNonExistPipe( - ((EnrichedEvent) peekedEvent).getPipeName(), - ((EnrichedEvent) peekedEvent).getCreationTime())) { - ((EnrichedEvent) peekedEvent) - .clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName()); - } else { - throw e; - } + final Event polledEvent = retryEventQueue.poll(); + if (polledEvent != peekedEvent) { + LOGGER.error( + "The event polled from the queue is not the same as the event peeked from the queue. " + + "Peeked event: {}, polled event: {}.", + peekedEvent, + polledEvent); + } + if (polledEvent != null && LOGGER.isDebugEnabled()) { + LOGGER.debug("Polled event {} from retry queue.", polledEvent); } } } From 570a1ad165e1be36a26876a6784d8e3177844143 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:30:52 +0800 Subject: [PATCH 14/17] bug fix --- .../PipeConfigNodeRemainingTimeMetrics.java | 14 +----- ...eDataNodeRemainingEventAndTimeMetrics.java | 26 ++--------- .../commons/pipe/event/EnrichedEvent.java | 4 ++ .../pipe/progress/PipeEventCommitManager.java | 43 ++++++++++--------- 4 files changed, 30 insertions(+), 57 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java index 003098df15931..557e355596e98 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.manager.pipe.metric; -import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskRuntimeEnvironment; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -142,12 +141,7 @@ public void deregister(final String pipeID) { } } - public void markRegionCommit(final PipeTaskRuntimeEnvironment pipeTaskRuntimeEnvironment) { - // Filter commit attempt from assigner - final String pipeName = pipeTaskRuntimeEnvironment.getPipeName(); - final long creationTime = pipeTaskRuntimeEnvironment.getCreationTime(); - final String pipeID = pipeName + "_" + creationTime; - + public void markRegionCommit(final String pipeID, final boolean isDataRegion) { if (Objects.isNull(metricService)) { return; } @@ -157,12 +151,6 @@ public void markRegionCommit(final PipeTaskRuntimeEnvironment pipeTaskRuntimeEnv "Failed to mark pipe region commit, RemainingTimeOperator({}) does not exist", pipeID); return; } - // Prevent not set pipeName / creation times & potential differences between pipeNames and - // creation times - if (!Objects.equals(pipeName, operator.getPipeName()) - || !Objects.equals(creationTime, operator.getCreationTime())) { - return; - } operator.markConfigRegionCommit(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java index fc17993304ebd..8f9008f5d9a2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java @@ -19,17 +19,12 @@ package org.apache.iotdb.db.pipe.metric; -import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskRuntimeEnvironment; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor; import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor; import org.apache.iotdb.db.pipe.task.subtask.connector.PipeConnectorSubtask; -import org.apache.iotdb.db.schemaengine.SchemaEngine; -import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.metrics.AbstractMetricService; import org.apache.iotdb.metrics.metricsets.IMetricSet; import org.apache.iotdb.metrics.utils.MetricLevel; @@ -183,13 +178,7 @@ public void deregister(final String pipeID) { } } - public void markRegionCommit(final PipeTaskRuntimeEnvironment pipeTaskRuntimeEnvironment) { - // Filter commit attempt from assigner - final String pipeName = pipeTaskRuntimeEnvironment.getPipeName(); - final int regionId = pipeTaskRuntimeEnvironment.getRegionId(); - final long creationTime = pipeTaskRuntimeEnvironment.getCreationTime(); - final String pipeID = pipeName + "_" + creationTime; - + public void markRegionCommit(final String pipeID, final boolean isDataRegion) { if (Objects.isNull(metricService)) { return; } @@ -201,19 +190,10 @@ public void markRegionCommit(final PipeTaskRuntimeEnvironment pipeTaskRuntimeEnv pipeID); return; } - // Prevent not set pipeName / creation times & potential differences between pipeNames and - // creation times - if (!Objects.equals(pipeName, operator.getPipeName()) - || !Objects.equals(creationTime, operator.getCreationTime())) { - return; - } - // Prevent empty region-ids - if (StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId))) { + if (isDataRegion) { operator.markDataRegionCommit(); - } - - if (SchemaEngine.getInstance().getAllSchemaRegionIds().contains(new SchemaRegionId(regionId))) { + } else { operator.markSchemaRegionCommit(); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java index e7e915aa09ce5..8de596a5447e1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java @@ -252,6 +252,10 @@ public final long getCreationTime() { return creationTime; } + public final boolean isDataRegionEvent() { + return !(this instanceof PipeWritePlanEvent) && !(this instanceof PipeSnapshotEvent); + } + /** * Get the pattern string of this {@link EnrichedEvent}. * diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java index 0b740f7e12a78..d51f012661261 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.commons.pipe.progress; -import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskRuntimeEnvironment; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.metric.PipeEventCommitMetrics; @@ -29,7 +28,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; +import java.util.function.BiConsumer; public class PipeEventCommitManager { @@ -38,7 +37,7 @@ public class PipeEventCommitManager { // key: pipeName_regionId private final Map eventCommitterMap = new ConcurrentHashMap<>(); - private Consumer commitRateMarker; + private BiConsumer commitRateMarker; public void register( final String pipeName, @@ -88,10 +87,26 @@ public void enrichWithCommitterKeyAndCommitId( } public void commit(final EnrichedEvent event, final String committerKey) { - if (committerKey == null - || event == null + if (event == null || !event.needToCommit() - || event.getCommitId() <= EnrichedEvent.NO_COMMIT_ID) { + || Objects.isNull(event.getPipeName()) + || event.getCreationTime() == 0) { + return; + } + if (Objects.nonNull(commitRateMarker)) { + try { + commitRateMarker.accept( + event.getPipeName() + '_' + event.getCreationTime(), event.isDataRegionEvent()); + } catch (final Exception e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Failed to mark commit rate for pipe task: {}, stack trace: {}", + committerKey, + Thread.currentThread().getStackTrace()); + } + } + } + if (committerKey == null || event.getCommitId() <= EnrichedEvent.NO_COMMIT_ID) { return; } final PipeEventCommitter committer = eventCommitterMap.get(committerKey); @@ -108,20 +123,6 @@ public void commit(final EnrichedEvent event, final String committerKey) { } committer.commit(event); - if (Objects.nonNull(commitRateMarker)) { - try { - commitRateMarker.accept( - new PipeTaskRuntimeEnvironment( - committer.getPipeName(), committer.getCreationTime(), committer.getRegionId())); - } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Failed to mark commit rate for pipe: {}, stack trace: {}", - committerKey, - Thread.currentThread().getStackTrace()); - } - } - } } private static String generateCommitterKey( @@ -129,7 +130,7 @@ private static String generateCommitterKey( return String.format("%s_%s_%s", pipeName, regionId, creationTime); } - public void setCommitRateMarker(final Consumer commitRateMarker) { + public void setCommitRateMarker(final BiConsumer commitRateMarker) { this.commitRateMarker = commitRateMarker; } From 79bafcde0100254d6c761ce4464b6b96e443fa1e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:06:57 +0800 Subject: [PATCH 15/17] fix --- .../apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java | 1 + .../event/common/tablet/PipeInsertNodeTabletInsertionEvent.java | 2 +- .../pipe/event/common/tsfile/TsFileInsertionDataContainer.java | 2 ++ .../historical/PipeHistoricalDataRegionTsFileExtractor.java | 1 + .../pipe/processor/twostage/plugin/TwoStageCountProcessor.java | 2 +- .../iotdb/db/pipe/task/connection/PipeEventCollector.java | 1 + 6 files changed, 7 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java index 1b747652e1cd7..cb2183b200056 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java @@ -105,6 +105,7 @@ private void collectTabletInsertionEvent() { tablet, isAligned, sourceEvent == null ? null : sourceEvent.getPipeName(), + sourceEvent == null ? 0 : sourceEvent.getCreationTime(), pipeTaskMeta, sourceEvent, false)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java index ca0c1c23dcd97..b9ddfd0d55d90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java @@ -340,7 +340,7 @@ public List toRawTabletInsertionEvents() { .map( tablet -> new PipeRawTabletInsertionEvent( - tablet, isAligned, pipeName, pipeTaskMeta, this, false)) + tablet, isAligned, pipeName, creationTime, pipeTaskMeta, this, false)) .filter(event -> !event.hasNoNeedParsingAndIsEmpty()) .collect(Collectors.toList()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java index 36dfc3df45899..03e61cf280986 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java @@ -252,6 +252,7 @@ public TabletInsertionEvent next() { tablet, isAligned, sourceEvent != null ? sourceEvent.getPipeName() : null, + sourceEvent != null ? sourceEvent.getCreationTime() : 0, pipeTaskMeta, sourceEvent, true); @@ -262,6 +263,7 @@ public TabletInsertionEvent next() { tablet, isAligned, sourceEvent != null ? sourceEvent.getPipeName() : null, + sourceEvent != null ? sourceEvent.getCreationTime() : 0, pipeTaskMeta, sourceEvent, false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 1ed78ea804596..237dace7c02f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -579,6 +579,7 @@ public synchronized Event supply() { false, false, pipeName, + creationTime, pipeTaskMeta, pipePattern, historicalDataExtractionStartTime, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java index 2017051866fce..07f952277aadb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java @@ -253,7 +253,7 @@ private void collectGlobalCountIfNecessary(EventCollector eventCollector) throws tablet.addValue(outputSeries.getMeasurement(), 0, timestampCountPair.right); eventCollector.collect( - new PipeRawTabletInsertionEvent(tablet, false, null, null, null, false)); + new PipeRawTabletInsertionEvent(tablet, false, null, 0, null, null, false)); PipeCombineHandlerManager.getInstance() .updateLastCombinedValue(pipeName, creationTime, timestampCountPair); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java index 3055568aadf93..fd673fe9b95e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java @@ -142,6 +142,7 @@ private void parseAndCollectEvent(final PipeSchemaRegionWritePlanEvent deleteDat new PipeSchemaRegionWritePlanEvent( planNode, deleteDataEvent.getPipeName(), + deleteDataEvent.getCreationTime(), deleteDataEvent.getPipeTaskMeta(), deleteDataEvent.getPipePattern(), deleteDataEvent.isGeneratedByPipe())) From 8ef174859ea6053182ba613a1ac01a76808034ae Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:52:40 +0800 Subject: [PATCH 16/17] Update CountPointProcessor.java --- .../src/main/java/org/apache/iotdb/CountPointProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java index 838090e7a3ad4..ed67b9abae63e 100644 --- a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java +++ b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java @@ -73,7 +73,7 @@ public void process(Event event, EventCollector eventCollector) throws Exception tablet.addTimestamp(0, System.currentTimeMillis()); tablet.addValue(aggregateSeries.getMeasurement(), 0, writePointCount.get()); eventCollector.collect( - new PipeRawTabletInsertionEvent(tablet, false, null, null, null, false)); + new PipeRawTabletInsertionEvent(tablet, false, null, 0, null, null, false)); } } From 0da5af69622f63950250c9ee8e14f225a07a28d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:54:03 +0800 Subject: [PATCH 17/17] Update CountPointProcessor.java --- .../java/org/apache/iotdb/CountPointProcessor.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java index ed67b9abae63e..1e13b27b3a3eb 100644 --- a/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java +++ b/example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java @@ -44,26 +44,28 @@ public class CountPointProcessor implements PipeProcessor { private PartialPath aggregateSeries; @Override - public void validate(PipeParameterValidator validator) { + public void validate(final PipeParameterValidator validator) { validator.validateRequiredAttribute(AGGREGATE_SERIES_KEY); } @Override - public void customize(PipeParameters parameters, PipeProcessorRuntimeConfiguration configuration) + public void customize( + final PipeParameters parameters, final PipeProcessorRuntimeConfiguration configuration) throws Exception { this.aggregateSeries = new PartialPath(parameters.getString(AGGREGATE_SERIES_KEY)); } @Override - public void process(TabletInsertionEvent tabletInsertionEvent, EventCollector eventCollector) { + public void process( + final TabletInsertionEvent tabletInsertionEvent, final EventCollector eventCollector) { tabletInsertionEvent.processTablet( (tablet, rowCollector) -> writePointCount.addAndGet(tablet.rowSize)); } @Override - public void process(Event event, EventCollector eventCollector) throws Exception { + public void process(final Event event, final EventCollector eventCollector) throws Exception { if (event instanceof PipeHeartbeatEvent) { - Tablet tablet = + final Tablet tablet = new Tablet( aggregateSeries.getDevice(), Collections.singletonList(