From 5789d10e549300373cf0f9a11fbc0693cb63d172 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Wed, 12 Jun 2024 16:52:14 +0800 Subject: [PATCH 01/22] pipeConsensus metrics --- .../iotdb/consensus/ConsensusFactory.java | 3 + .../pipe/PipeConsensusServerImpl.java | 44 +- .../consensuspipe/ConsensusPipeConnector.java | 25 ++ .../consensuspipe/ConsensusPipeManager.java | 2 + .../metric/PipeConsensusServerMetrics.java | 162 ++++++++ .../metric/PipeConsensusSyncLagManager.java | 109 +++++ .../agent/task/PipeDataNodeTaskAgent.java | 13 +- .../PipeConsensusAsyncConnector.java | 83 +++- .../PipeConsensusSyncConnector.java | 25 +- ...ConsensusTabletInsertNodeEventHandler.java | 10 +- ...eConsensusTabletInsertionEventHandler.java | 14 +- .../PipeConsensusTabletRawEventHandler.java | 45 -- ...eConsensusTsFileInsertionEventHandler.java | 22 +- .../PipeConsensusConnectorMetric.java | 290 +++++++++++++ .../PipeConsensusReceiverMetric.java | 384 ++++++++++++++++++ .../pipeconsensus/PipeConsensusReceiver.java | 188 +++++++-- .../PipeConsensusClientMgrContainer.java | 2 +- .../constant/PipeConnectorConstant.java | 1 + .../commons/pipe/event/EnrichedEvent.java | 2 +- .../pipe/progress/PipeEventCommitManager.java | 8 + .../pipe/progress/PipeEventCommitter.java | 4 + .../commons/service/metric/enums/Metric.java | 4 + 22 files changed, 1324 insertions(+), 116 deletions(-) create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletRawEventHandler.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetric.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetric.java diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ConsensusFactory.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ConsensusFactory.java index 7742baf3bdccf..37ca108a93401 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ConsensusFactory.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ConsensusFactory.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.client.container.PipeConsensusClientMgrContainer; import org.apache.iotdb.consensus.config.ConsensusConfig; import org.apache.iotdb.consensus.config.PipeConsensusConfig.ReplicateMode; +import org.apache.iotdb.consensus.pipe.metric.PipeConsensusSyncLagManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +67,8 @@ public static Optional getConsensusImpl( className = REAL_PIPE_CONSENSUS; // initialize pipeConsensus' thrift component PipeConsensusClientMgrContainer.build(); + // initialize pipeConsensus's metric component + PipeConsensusSyncLagManager.build(); } Class executor = Class.forName(className); Constructor executorConstructor = diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index f4a64fc691bfe..79b83c3fe1410 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -28,15 +28,18 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.pipe.task.meta.PipeStatus; +import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.consensus.IStateMachine; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.consensus.config.PipeConsensusConfig; +import org.apache.iotdb.consensus.config.PipeConsensusConfig.ReplicateMode; import org.apache.iotdb.consensus.exception.ConsensusGroupModifyPeerException; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeManager; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName; import org.apache.iotdb.consensus.pipe.consensuspipe.ProgressIndexManager; +import org.apache.iotdb.consensus.pipe.metric.PipeConsensusServerMetrics; import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompletedReq; import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompletedResp; import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToCreateConsensusPipeReq; @@ -77,6 +80,8 @@ public class PipeConsensusServerImpl { private final ConsensusPipeManager consensusPipeManager; private final ProgressIndexManager progressIndexManager; private final IClientManager syncClientManager; + private final PipeConsensusServerMetrics pipeConsensusServerMetrics; + private final ReplicateMode replicateMode; private ProgressIndex cachedProgressIndex = MinimumProgressIndex.INSTANCE; @@ -98,6 +103,8 @@ public PipeConsensusServerImpl( this.consensusPipeManager = consensusPipeManager; this.progressIndexManager = config.getPipe().getProgressIndexManager(); this.syncClientManager = syncClientManager; + this.pipeConsensusServerMetrics = new PipeConsensusServerMetrics(this); + this.replicateMode = config.getReplicateMode(); if (configuration.isEmpty()) { peerManager.recover(); @@ -129,6 +136,7 @@ public PipeConsensusServerImpl( public synchronized void start(boolean startConsensusPipes) throws IOException { stateMachine.start(); + MetricService.getInstance().addMetricSet(this.pipeConsensusServerMetrics); if (startConsensusPipes) { // start all consensus pipes @@ -274,12 +282,24 @@ public synchronized void checkConsensusPipe(Map e public TSStatus write(IConsensusRequest request) { try { + long consensusWriteStartTime = System.nanoTime(); stateMachineLock.lock(); + long getStateMachineLockTime = System.nanoTime(); + // statistic the time of acquiring stateMachine lock + pipeConsensusServerMetrics.recordGetStateMachineLockTime( + getStateMachineLockTime - consensusWriteStartTime); if (request instanceof ComparableConsensusRequest) { ((ComparableConsensusRequest) request) .setProgressIndex(progressIndexManager.assignProgressIndex(thisNode.getGroupId())); } - return stateMachine.write(request); + + long writeToStateMachineStartTime = System.nanoTime(); + TSStatus result = stateMachine.write(request); + long writeToStateMachineEndTime = System.nanoTime(); + // statistic the time of writing request into stateMachine + pipeConsensusServerMetrics.recordWriteStateMachineTime( + writeToStateMachineEndTime - writeToStateMachineStartTime); + return result; } finally { stateMachineLock.unlock(); } @@ -287,8 +307,20 @@ public TSStatus write(IConsensusRequest request) { public TSStatus writeOnFollowerReplica(IConsensusRequest request) { try { + long consensusWriteStartTime = System.nanoTime(); stateMachineLock.lock(); - return stateMachine.write(request); + long getStateMachineLockTime = System.nanoTime(); + // statistic the time of acquiring stateMachine lock + pipeConsensusServerMetrics.recordGetStateMachineLockTime( + getStateMachineLockTime - consensusWriteStartTime); + + long writeToStateMachineStartTime = System.nanoTime(); + TSStatus result = stateMachine.write(request); + long writeToStateMachineEndTime = System.nanoTime(); + // statistic the time of writing request into stateMachine + pipeConsensusServerMetrics.recordWriteStateMachineTime( + writeToStateMachineEndTime - writeToStateMachineStartTime); + return result; } finally { stateMachineLock.unlock(); } @@ -540,4 +572,12 @@ public boolean containsPeer(Peer peer) { public List getPeers() { return peerManager.getPeers(); } + + public String getConsensusGroupId() { + return consensusGroupId; + } + + public long getReplicateMode() { + return (replicateMode == ReplicateMode.BATCH) ? 0 : 1; + } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java new file mode 100644 index 0000000000000..3b00e5d265394 --- /dev/null +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.consensus.pipe.consensuspipe; + +public interface ConsensusPipeConnector { + long getLocalUserWriteProgress(); + + long getCurrentPipeReplicateProgress(); +} diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeManager.java index bb1bf2a176942..a6402a8610d2b 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeManager.java @@ -29,6 +29,7 @@ import java.util.Map; import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME; import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY; @@ -78,6 +79,7 @@ public void createConsensusPipe(Peer senderPeer, Peer receiverPeer) throws Excep .put( CONNECTOR_CONSENSUS_GROUP_ID_KEY, String.valueOf(consensusPipeName.getConsensusGroupId().getId())) + .put(CONNECTOR_CONSENSUS_PIPE_NAME, consensusPipeName.toString()) .put(CONNECTOR_IOTDB_IP_KEY, receiverPeer.getEndpoint().ip) .put(CONNECTOR_IOTDB_PORT_KEY, String.valueOf(receiverPeer.getEndpoint().port)) .put(CONNECTOR_IOTDB_PARALLEL_TASKS_KEY, String.valueOf(1)) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java new file mode 100644 index 0000000000000..47cd626dd16ca --- /dev/null +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.pipe.metric; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.consensus.pipe.PipeConsensusServerImpl; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Timer; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class PipeConsensusServerMetrics implements IMetricSet { + private final PipeConsensusServerImpl impl; + private final PipeConsensusSyncLagManager syncLagManager = + PipeConsensusSyncLagManager.getInstance(); + + private Timer getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + public PipeConsensusServerMetrics(PipeConsensusServerImpl impl) { + this.impl = impl; + } + + private static final String IMPL = "PipeConsensusServerImpl"; + + public void recordGetStateMachineLockTime(long costTimeInNanos) { + getStateMachineLockTimer.updateNanos(costTimeInNanos); + } + + public void recordWriteStateMachineTime(long costTimeInNanos) { + writeStateMachineTimer.updateNanos(costTimeInNanos); + } + + @Override + public void bindTo(AbstractMetricService metricService) { + bindTimer(metricService); + bindAutoGauge(metricService); + bindGauge(metricService); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + unbindTimer(metricService); + unbindAutoGauge(metricService); + unbindGauge(metricService); + } + + public void bindGauge(AbstractMetricService metricService) { + metricService + .getOrCreateGauge( + Metric.PIPE_CONSENSUS.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + IMPL, + Tag.TYPE.toString(), + "replicateMode") + .set(impl.getReplicateMode()); + } + + public void unbindGauge(AbstractMetricService metricService) { + metricService.remove( + MetricType.GAUGE, + Metric.PIPE_CONSENSUS.toString(), + Tag.NAME.toString(), + IMPL, + Tag.TYPE.toString(), + "replicateMode"); + } + + public void bindAutoGauge(AbstractMetricService metricService) { + metricService.createAutoGauge( + Metric.PIPE_CONSENSUS.toString(), + MetricLevel.IMPORTANT, + syncLagManager, + PipeConsensusSyncLagManager::calculateSyncLag, + Tag.NAME.toString(), + IMPL, + Tag.REGION.toString(), + impl.getConsensusGroupId(), + Tag.TYPE.toString(), + "syncLag"); + } + + public void unbindAutoGauge(AbstractMetricService metricService) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_CONSENSUS.toString(), + Tag.NAME.toString(), + IMPL, + Tag.REGION.toString(), + impl.getConsensusGroupId(), + Tag.TYPE.toString(), + "syncLag"); + } + + public void bindTimer(AbstractMetricService metricService) { + getStateMachineLockTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "getStateMachineLock", + Tag.REGION.toString(), + impl.getConsensusGroupId()); + writeStateMachineTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "writeStateMachine", + Tag.REGION.toString(), + impl.getConsensusGroupId()); + } + + public void unbindTimer(AbstractMetricService metricService) { + getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "getStateMachineLock", + Tag.REGION.toString(), + impl.getConsensusGroupId()); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "writeStateMachine", + Tag.REGION.toString(), + impl.getConsensusGroupId()); + } +} diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java new file mode 100644 index 0000000000000..b01b5406c47c7 --- /dev/null +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.pipe.metric; + +import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeConnector; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * This class is used to aggregate the write progress of all Connectors to calculate the minimum + * synchronization progress of all follower copies, thereby calculating syncLag. + */ +public class PipeConsensusSyncLagManager { + long userWriteProgress = 0; + long minReplicateProgress = Long.MAX_VALUE; + List consensusPipeConnectorList = new CopyOnWriteArrayList<>(); + + private void updateReplicateProgress() { + // if there isn't a consensus pipe task, replicate progress is 0. + if (consensusPipeConnectorList.isEmpty()) { + minReplicateProgress = 0; + } + // else we find the minimum progress in all consensus pipe task. + consensusPipeConnectorList.forEach( + consensusPipeConnector -> { + minReplicateProgress = + Math.min( + minReplicateProgress, consensusPipeConnector.getCurrentPipeReplicateProgress()); + }); + } + + private void updateUserWriteProgress() { + // if there isn't a consensus pipe task, user write progress is Long.MAX_VALUE. + if (consensusPipeConnectorList.isEmpty()) { + userWriteProgress = Long.MAX_VALUE; + } + // since the user write progress of different consensus pipes on the same DataRegion is the + // same, we only need to take out one Connector to calculate + ConsensusPipeConnector connector = consensusPipeConnectorList.get(0); + userWriteProgress = connector.getLocalUserWriteProgress(); + } + + public void addConsensusPipeConnector(ConsensusPipeConnector consensusPipeConnector) { + consensusPipeConnectorList.add(consensusPipeConnector); + } + + /** + * SyncLag represents the difference between the current replica users' write progress and the + * minimum synchronization progress of all other replicas. The semantics is how much data the + * leader has left to synchronize. + */ + public long calculateSyncLag() { + updateUserWriteProgress(); + updateReplicateProgress(); + // if there isn't a consensus pipe task, the syncLag is userWriteProgress - 0 + if (minReplicateProgress == Long.MAX_VALUE) { + return userWriteProgress; + } else { + return userWriteProgress - minReplicateProgress; + } + } + + private PipeConsensusSyncLagManager() { + // do nothing + } + + /** singleton */ + private static class PipeConsensusSyncLagManagerHolder { + private static PipeConsensusSyncLagManager INSTANCE; + + private PipeConsensusSyncLagManagerHolder() { + // empty constructor + } + + public static void build() { + if (INSTANCE == null) { + INSTANCE = new PipeConsensusSyncLagManager(); + } + } + } + + public static PipeConsensusSyncLagManager getInstance() { + return PipeConsensusSyncLagManagerHolder.INSTANCE; + } + + // Only when consensus protocol is PipeConsensus, this method will be called once when construct + // consensus class. + public static void build() { + PipeConsensusSyncLagManagerHolder.build(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 37c04e26f5085..29972a0e69a5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -555,11 +555,15 @@ private void changePipeStatusBeforeRestart(final String pipeName) { dataRegionPipeTasks.parallelStream().forEach(PipeTask::drop); // Stop schema region tasks - pipeTaskManager.getPipeTasks(pipeMeta.getStaticMeta()).values().parallelStream() + pipeTaskManager + .getPipeTasks(pipeMeta.getStaticMeta()) + .values() + .parallelStream() .forEach(PipeTask::stop); // Re-create data region tasks - dataRegionPipeTasks.parallelStream() + dataRegionPipeTasks + .parallelStream() .forEach( pipeTask -> { final PipeTask newPipeTask = @@ -612,6 +616,11 @@ public Set getPipeTaskRegionIdSet(final String pipeName, final long cre ///////////////////////// Pipe Consensus ///////////////////////// + public PipeStaticMeta getPipeStaticMeta(final String pipeName) { + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + return pipeMeta == null ? null : pipeMeta.getStaticMeta(); + } + public ProgressIndex getPipeTaskProgressIndex(String pipeName, int consensusGroupId) { if (!tryReadLockWithTimeOut(10)) { throw new PipeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 50da4d56839f9..12e822679fa00 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -25,20 +25,28 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncPipeConsensusServiceClient; import org.apache.iotdb.commons.client.container.PipeConsensusClientMgrContainer; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.connector.protocol.IoTDBConnector; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; +import org.apache.iotdb.commons.pipe.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeConnector; +import org.apache.iotdb.consensus.pipe.metric.PipeConsensusSyncLagManager; import org.apache.iotdb.consensus.pipe.thrift.TCommitId; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTabletBatchEventHandler; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTabletInsertNodeEventHandler; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTsFileInsertionEventHandler; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.builder.PipeConsensusAsyncBatchReqBuilder; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTabletBinaryReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTabletInsertNodeReq; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; @@ -66,46 +74,35 @@ // TODO: Optimize the network and disk io for TsFile onComplete // TODO: support Tablet Batch -public class PipeConsensusAsyncConnector extends IoTDBConnector { - +public class PipeConsensusAsyncConnector extends IoTDBConnector implements ConsensusPipeConnector { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusAsyncConnector.class); - private static final String CUSTOMIZE_EXCEPTION_MSG = "Failed to customize pipeConsensusAsyncConnector because there isn't consensusGroupId passed in. Please check your construct parameters."; - private static final String ENQUEUE_EXCEPTION_MSG = "Timeout: PipeConsensusConnector offers an event into transferBuffer failed, because transferBuffer is full."; - private static final String THRIFT_ERROR_FORMATTER_WITHOUT_ENDPOINT = "Failed to borrow client from client pool or exception occurred " + "when sending to receiver."; - private static final String THRIFT_ERROR_FORMATTER_WITH_ENDPOINT = "Failed to borrow client from client pool or exception occurred " + "when sending to receiver %s:%s."; - private static final IoTDBConfig IOTDB_CONFIG = IoTDBDescriptor.getInstance().getConfig(); - private static final long PIPE_CONSENSUS_EVENT_ENQUEUE_TIMEOUT_IN_MS = IOTDB_CONFIG.getConnectionTimeoutInMS() / 6; private final BlockingQueue retryEventQueue = new LinkedBlockingQueue<>(); - // We use enrichedEvent here to make use of EnrichedEvent.equalsInPipeConsensus private final BlockingQueue transferBuffer = new LinkedBlockingDeque<>(IOTDB_CONFIG.getPipeConsensusPipelineSize()); - private final AtomicBoolean isClosed = new AtomicBoolean(false); - private final int thisDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); - + private PipeConsensusConnectorMetric pipeConsensusConnectorMetric; + private String consensusPipeName; private int consensusGroupId; - private PipeConsensusSyncConnector retryConnector; - private IClientManager asyncTransferClientManager; - private PipeConsensusAsyncBatchReqBuilder tabletBatchBuilder; + private long currentReplicateProgress = 0; @Override public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) @@ -118,10 +115,18 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati } consensusGroupId = parameters.getInt(PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY); + // Get consensusPipeName from parameters passed by PipeConsensusImpl + if (!parameters.hasAttribute(PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME)) { + throw new PipeException(CUSTOMIZE_EXCEPTION_MSG); + } + consensusPipeName = parameters.getString(PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME); + // In PipeConsensus, one pipeConsensusTask corresponds to a pipeConsensusConnector. Thus, // `nodeUrls` here actually is a singletonList that contains one peer's TEndPoint. But here we // retain the implementation of list to cope with possible future expansion - retryConnector = new PipeConsensusSyncConnector(nodeUrls, consensusGroupId, thisDataNodeId); + retryConnector = + new PipeConsensusSyncConnector( + nodeUrls, consensusGroupId, thisDataNodeId, pipeConsensusConnectorMetric); retryConnector.customize(parameters, configuration); asyncTransferClientManager = PipeConsensusClientMgrContainer.getInstance().getAsyncClientManager(); @@ -136,6 +141,11 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati // currently, tablet batch is false by default in PipeConsensus; isTabletBatchModeEnabled = false; + + // initialize metric components + pipeConsensusConnectorMetric = new PipeConsensusConnectorMetric(this); + PipeConsensusSyncLagManager.getInstance().addConsensusPipeConnector(this); + MetricService.getInstance().addMetricSet(this.pipeConsensusConnectorMetric); } /** Add an event to transferBuffer, whose events will be asynchronizedly transfer to receiver. */ @@ -148,9 +158,12 @@ private boolean addEvent2Buffer(EnrichedEvent event) { event.getCommitId(), event); } + long currentTime = System.nanoTime(); boolean result = transferBuffer.offer( event, PIPE_CONSENSUS_EVENT_ENQUEUE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); + long duration = System.nanoTime() - currentTime; + pipeConsensusConnectorMetric.recordConnectorEnqueueTimer(duration); // add reference if (result) { event.increaseReferenceCount(PipeConsensusAsyncConnector.class.getName()); @@ -182,6 +195,8 @@ public synchronized void removeEventFromBuffer(EnrichedEvent event) { current = iterator.next(); } iterator.remove(); + // update replicate progress + currentReplicateProgress = Math.max(currentReplicateProgress, event.getCommitId()); // decrease reference count event.decreaseReferenceCount(PipeConsensusAsyncConnector.class.getName(), true); } @@ -252,7 +267,10 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception insertNode, tCommitId, tConsensusGroupId, progressIndex, thisDataNodeId); final PipeConsensusTabletInsertNodeEventHandler pipeConsensusInsertNodeReqHandler = new PipeConsensusTabletInsertNodeEventHandler( - pipeInsertNodeTabletInsertionEvent, pipeConsensusTransferReq, this); + pipeInsertNodeTabletInsertionEvent, + pipeConsensusTransferReq, + this, + pipeConsensusConnectorMetric); transfer(pipeConsensusInsertNodeReqHandler); } @@ -322,7 +340,12 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception final PipeConsensusTsFileInsertionEventHandler pipeConsensusTsFileInsertionEventHandler = new PipeConsensusTsFileInsertionEventHandler( - pipeTsFileInsertionEvent, this, tCommitId, tConsensusGroupId, thisDataNodeId); + pipeTsFileInsertionEvent, + this, + tCommitId, + tConsensusGroupId, + thisDataNodeId, + pipeConsensusConnectorMetric); transfer(pipeConsensusTsFileInsertionEventHandler); } catch (Exception e) { @@ -466,6 +489,7 @@ public void addFailureEventToRetryQueue(final Event event) { public void addFailureEventsToRetryQueue(final Iterable events) { for (final Event event : events) { addFailureEventToRetryQueue(event); + pipeConsensusConnectorMetric.recordRetryCounter(); } } @@ -512,7 +536,7 @@ public synchronized void close() { } } - //////////////////////////// TODO: APIs provided for metric framework //////////////////////////// + //////////////////////////// APIs provided for metric framework //////////////////////////// public int getTransferBufferSize() { return transferBuffer.size(); @@ -521,4 +545,25 @@ public int getTransferBufferSize() { public int getRetryBufferSize() { return retryEventQueue.size(); } + + @Override + public long getLocalUserWriteProgress() { + PipeStaticMeta staticMeta = PipeAgent.task().getPipeStaticMeta(consensusPipeName); + String committerKey = + String.format( + "%s_%s_%s", staticMeta.getPipeName(), consensusGroupId, staticMeta.getCreationTime()); + return PipeEventCommitManager.getInstance().getGivenConsensusPipeCommitId(committerKey); + } + + @Override + public long getCurrentPipeReplicateProgress() { + return currentReplicateProgress; + } + + public String getConsensusGroupIdStr() { + ConsensusGroupId groupId = + ConsensusGroupId.Factory.create( + TConsensusGroupType.DataRegion.getValue(), consensusGroupId); + return groupId.toString(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java index e29d5f49398c5..ed7e09a045b10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFilePieceWithModReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealWithModReq; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -65,30 +66,24 @@ /** This connector is used for PipeConsensus to transfer queued event. */ public class PipeConsensusSyncConnector extends IoTDBConnector { - private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusSyncConnector.class); - private static final String PIPE_CONSENSUS_SYNC_CONNECTION_FAILED_FORMAT = "PipeConsensus: syncClient connection to %s:%s failed when %s, because: %s"; - private static final String TABLET_INSERTION_NODE_SCENARIO = "transfer insertionNode tablet"; - private static final String TSFILE_SCENARIO = "transfer tsfile"; - private static final String TABLET_BATCH_SCENARIO = "transfer tablet batch"; - private final IClientManager syncRetryClientManager; - private final List peers; - private final int thisDataNodeId; - private final int consensusGroupId; - + private final PipeConsensusConnectorMetric pipeConsensusConnectorMetric; private PipeConsensusSyncBatchReqBuilder tabletBatchBuilder; public PipeConsensusSyncConnector( - List peers, int consensusGroupId, int thisDataNodeId) { + List peers, + int consensusGroupId, + int thisDataNodeId, + PipeConsensusConnectorMetric pipeConsensusConnectorMetric) { // In PipeConsensus, one pipeConsensusTask corresponds to a pipeConsensusConnector. Thus, // `peers` here actually is a singletonList that contains one peer's TEndPoint. But here we // retain the implementation of list to cope with possible future expansion @@ -97,6 +92,7 @@ public PipeConsensusSyncConnector( this.thisDataNodeId = thisDataNodeId; this.syncRetryClientManager = PipeConsensusClientMgrContainer.getInstance().getSyncClientManager(); + this.pipeConsensusConnectorMetric = pipeConsensusConnectorMetric; } @Override @@ -137,7 +133,10 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception doTransfer(); } } else { + long startTime = System.nanoTime(); doTransferWrapper((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent); + long duration = System.nanoTime() - startTime; + pipeConsensusConnectorMetric.recordRetryWALTransferTimer(duration); } } catch (Exception e) { throw new PipeConnectionException( @@ -154,12 +153,14 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception // processor and will not change the event type like // org.apache.iotdb.db.pipe.connector.protocol.thrift.sync.IoTDBDataRegionSyncConnector try { + long startTime = System.nanoTime(); // In order to commit in order if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { doTransfer(); } - doTransfer((PipeTsFileInsertionEvent) tsFileInsertionEvent); + long duration = System.nanoTime() - startTime; + pipeConsensusConnectorMetric.recordRetryTsFileTransferTimer(duration); } catch (Exception e) { throw new PipeConnectionException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java index 7b05fe297f279..0f0cb91091c5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java @@ -23,6 +23,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.thrift.TException; @@ -31,10 +32,11 @@ public class PipeConsensusTabletInsertNodeEventHandler extends PipeConsensusTabletInsertionEventHandler { public PipeConsensusTabletInsertNodeEventHandler( - PipeInsertNodeTabletInsertionEvent event, - TPipeConsensusTransferReq req, - PipeConsensusAsyncConnector connector) { - super(event, req, connector); + PipeInsertNodeTabletInsertionEvent event, + TPipeConsensusTransferReq req, + PipeConsensusAsyncConnector connector, + PipeConsensusConnectorMetric metric) { + super(event, req, connector, metric); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java index 3e12b23f4633c..fbb5a2c316be6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java @@ -26,6 +26,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.handler.PipeTransferTabletInsertionEventHandler; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -46,13 +47,20 @@ public abstract class PipeConsensusTabletInsertionEventHandler { - - public PipeConsensusTabletRawEventHandler( - PipeRawTabletInsertionEvent event, - TPipeConsensusTransferReq req, - PipeConsensusAsyncConnector connector) { - super(event, req, connector); - } - - @Override - protected void doTransfer(AsyncPipeConsensusServiceClient client, TPipeConsensusTransferReq req) - throws TException { - client.pipeConsensusTransfer(req, this); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java index 4ed42afc63c4f..42e2077fecf26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFilePieceWithModReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealWithModReq; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.rpc.TSStatusCode; @@ -74,12 +75,19 @@ public class PipeConsensusTsFileInsertionEventHandler private AsyncPipeConsensusServiceClient client; + private final PipeConsensusConnectorMetric metric; + + private long createTime; + + private long startTransferPieceTime; + public PipeConsensusTsFileInsertionEventHandler( final PipeTsFileInsertionEvent event, final PipeConsensusAsyncConnector connector, final TCommitId commitId, final TConsensusGroupId consensusGroupId, - final int thisDataNodeId) + final int thisDataNodeId, + final PipeConsensusConnectorMetric metric) throws FileNotFoundException { this.event = event; this.connector = connector; @@ -102,15 +110,19 @@ public PipeConsensusTsFileInsertionEventHandler( : new RandomAccessFile(tsFile, "r"); isSealSignalSent = new AtomicBoolean(false); + + this.metric = metric; + this.createTime = System.nanoTime(); } public void transfer(final AsyncPipeConsensusServiceClient client) throws TException, IOException { + startTransferPieceTime = System.nanoTime(); + this.client = client; client.setShouldReturnSelf(false); final int readLength = reader.read(readBuffer); - if (readLength == -1) { if (currentFile == modFile) { currentFile = tsFile; @@ -217,6 +229,9 @@ public void onComplete(final TPipeConsensusTransferResp response) { client.setShouldReturnSelf(true); client.returnSelf(); } + + long duration = System.nanoTime() - createTime; + metric.recordConnectorTsFileTransferTimer(duration); } return; } @@ -245,6 +260,8 @@ public void onComplete(final TPipeConsensusTransferResp response) { .handle(status, response.getStatus().getMessage(), tsFile.getName()); } } + long duration = System.nanoTime() - startTransferPieceTime; + metric.recordConnectorTsFilePieceTransferTimer(duration); transfer(client); } catch (final Exception e) { @@ -269,6 +286,7 @@ public void onError(final Exception exception) { LOGGER.warn("Failed to close file reader when failed to transfer file.", e); } finally { connector.addFailureEventToRetryQueue(event); + metric.recordRetryCounter(); if (client != null) { client.setShouldReturnSelf(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetric.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetric.java new file mode 100644 index 0000000000000..8fb80d681a6ba --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetric.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.consensus; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.Timer; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class PipeConsensusConnectorMetric implements IMetricSet { + private final PipeConsensusAsyncConnector pipeConsensusAsyncConnector; + + private Timer connectorEnqueueTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer connectorWALTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer connectorTsFileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer connectorTsFilePieceTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer retryWALTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer retryTsFileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Counter retryCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + + private static final String CONNECTOR = "pipeConsensusAsyncConnector"; + + public PipeConsensusConnectorMetric(PipeConsensusAsyncConnector pipeConsensusAsyncConnector) { + this.pipeConsensusAsyncConnector = pipeConsensusAsyncConnector; + } + + public void recordConnectorEnqueueTimer(long costTimeInNanos) { + connectorEnqueueTimer.updateNanos(costTimeInNanos); + } + + public void recordConnectorWalTransferTimer(long costTimeInNanos) { + connectorWALTransferTimer.updateNanos(costTimeInNanos); + } + + public void recordConnectorTsFileTransferTimer(long costTimeInNanos) { + connectorTsFileTransferTimer.updateNanos(costTimeInNanos); + } + + public void recordConnectorTsFilePieceTransferTimer(long costTimeInNanos) { + connectorTsFilePieceTransferTimer.updateNanos(costTimeInNanos); + } + + public void recordRetryWALTransferTimer(long costTimeInNanos) { + retryWALTransferTimer.updateNanos(costTimeInNanos); + } + + public void recordRetryTsFileTransferTimer(long costTimeInNanos) { + retryTsFileTransferTimer.updateNanos(costTimeInNanos); + } + + public void recordRetryCounter() { + retryCounter.inc(); + } + + @Override + public void bindTo(AbstractMetricService metricService) { + bindCounter(metricService); + bindAutoGauge(metricService); + bindTimer(metricService); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + unbindCounter(metricService); + unbindAutoGauge(metricService); + unbindTimer(metricService); + } + + private void bindCounter(AbstractMetricService metricService) { + metricService.getOrCreateCounter( + Metric.PIPE_RETRY_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "pipeConsensusRetryCount"); + } + + private void bindAutoGauge(AbstractMetricService metricService) { + metricService.createAutoGauge( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + pipeConsensusAsyncConnector, + PipeConsensusAsyncConnector::getTransferBufferSize, + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "transferBufferSize"); + metricService.createAutoGauge( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + pipeConsensusAsyncConnector, + PipeConsensusAsyncConnector::getRetryBufferSize, + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "retryBufferSize"); + } + + private void bindTimer(AbstractMetricService metricService) { + connectorEnqueueTimer = + metricService.getOrCreateTimer( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorEnqueueTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + connectorTsFilePieceTransferTimer = + metricService.getOrCreateTimer( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorTsFilePieceTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + connectorTsFileTransferTimer = + metricService.getOrCreateTimer( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorTsFileTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + connectorWALTransferTimer = + metricService.getOrCreateTimer( + Metric.PIPE_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorWALTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + retryWALTransferTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RETRY_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "retryWALTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + retryTsFileTransferTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RETRY_SEND_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "retryTsFileTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + } + + private void unbindCounter(AbstractMetricService metricService) { + retryCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + + metricService.remove( + MetricType.COUNTER, + Metric.PIPE_RETRY_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "pipeConsensusRetryCount"); + } + + private void unbindAutoGauge(AbstractMetricService metricService) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "transferBufferSize"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "retryBufferSize"); + } + + private void unbindTimer(AbstractMetricService metricService) { + connectorEnqueueTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + connectorWALTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + connectorTsFileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + connectorTsFilePieceTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + retryWALTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + retryTsFileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + metricService.remove( + MetricType.TIMER, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorTsFileTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorTsFilePieceTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorTsFileTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "connectorWALTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RETRY_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "retryWALTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RETRY_SEND_EVENT.toString(), + Tag.NAME.toString(), + CONNECTOR, + Tag.TYPE.toString(), + "retryTsFileTransferTimer", + Tag.REGION.toString(), + pipeConsensusAsyncConnector.getConsensusGroupIdStr()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetric.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetric.java new file mode 100644 index 0000000000000..97c01c5c803de --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetric.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.consensus; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.pipe.receiver.protocol.pipeconsensus.PipeConsensusReceiver; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Timer; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +public class PipeConsensusReceiverMetric implements IMetricSet { + private final PipeConsensusReceiver pipeConsensusReceiver; + + private Timer tsFilePieceWriteTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer tsFilePiecePreCheckTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer tsFileSealLoadTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer tsFileSealPreCheckTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer borrowTsFileWriterTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer acquireExecutorLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer dispatchWaitingTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer receiveWALTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer receiveTsFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer receiveEventTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + private static final String RECEIVER = "pipeConsensusReceiver"; + + public PipeConsensusReceiverMetric(PipeConsensusReceiver pipeConsensusReceiver) { + this.pipeConsensusReceiver = pipeConsensusReceiver; + } + + public void recordTsFilePieceWriteTime(long costTimeInNanos) { + tsFilePieceWriteTimer.updateNanos(costTimeInNanos); + } + + public void recordTsFilePiecePreCheckTime(long costTimeInNanos) { + tsFilePiecePreCheckTimer.updateNanos(costTimeInNanos); + } + + public void recordTsFileSealLoadTimer(long costTimeInNanos) { + tsFileSealLoadTimer.updateNanos(costTimeInNanos); + } + + public void recordTsFileSealPreCheckTimer(long costTimeInNanos) { + tsFileSealPreCheckTimer.updateNanos(costTimeInNanos); + } + + public void recordBorrowTsFileWriterTimer(long costTimeInNanos) { + borrowTsFileWriterTimer.updateNanos(costTimeInNanos); + } + + public void recordAcquireExecutorLockTimer(long costTimeInNanos) { + acquireExecutorLockTimer.updateNanos(costTimeInNanos); + } + + public void recordDispatchWaitingTimer(long costTimeInNanos) { + dispatchWaitingTimer.updateNanos(costTimeInNanos); + } + + public void recordReceiveWALTimer(long costTimeInNanos) { + receiveWALTimer.updateNanos(costTimeInNanos); + } + + public void recordReceiveTsFileTimer(long costTimeInNanos) { + receiveTsFileTimer.updateNanos(costTimeInNanos); + } + + public void recordReceiveEventTimer(long costTimeInNanos) { + receiveEventTimer.updateNanos(costTimeInNanos); + } + + @Override + public void bindTo(AbstractMetricService metricService) { + bindAutoGauge(metricService); + bindStageTimer(metricService); + bindReceiveTimer(metricService); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + unbindAutoGauge(metricService); + unbindStageTimer(metricService); + unbindReceiveTimer(metricService); + } + + public void bindAutoGauge(AbstractMetricService metricService) { + metricService.createAutoGauge( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + pipeConsensusReceiver, + PipeConsensusReceiver::getReceiveBufferSize, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveBufferSize"); + metricService.createAutoGauge( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + pipeConsensusReceiver, + PipeConsensusReceiver::getWALEventCount, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "walEventCount"); + metricService.createAutoGauge( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + pipeConsensusReceiver, + PipeConsensusReceiver::getTsFileEventCount, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileEventCount"); + } + + public void bindStageTimer(AbstractMetricService metricService) { + tsFilePieceWriteTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsPieceWrite"); + tsFilePiecePreCheckTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFilePiecePreCheck"); + tsFileSealLoadTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileSealLoad"); + tsFileSealPreCheckTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileSealPreCheck"); + borrowTsFileWriterTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "borrowTsFileWriter"); + acquireExecutorLockTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "acquireExecutorLock"); + dispatchWaitingTimer = + metricService.getOrCreateTimer( + Metric.STAGE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "dispatchWaiting"); + } + + public void bindReceiveTimer(AbstractMetricService metricService) { + receiveEventTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveEvent"); + receiveWALTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.STAGE.toString(), + "receiveWALEvent"); + receiveTsFileTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.STAGE.toString(), + "receiveTsFileEvent"); + } + + public void unbindAutoGauge(AbstractMetricService metricService) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveBufferSize"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "walEventCount"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileEventCount"); + } + + public void unbindStageTimer(AbstractMetricService metricService) { + tsFilePieceWriteTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + tsFilePiecePreCheckTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + tsFileSealLoadTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + tsFileSealPreCheckTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + borrowTsFileWriterTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + acquireExecutorLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + dispatchWaitingTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsPieceWrite"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFilePiecePreCheck"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileSealLoad"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "tsFileSealPreCheck"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "borrowTsFileWriter"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "acquireExecutorLock"); + metricService.remove( + MetricType.TIMER, + Metric.STAGE.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "dispatchWaiting"); + } + + public void unbindReceiveTimer(AbstractMetricService metricService) { + receiveEventTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + receiveWALTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + receiveTsFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveEvent"); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveWALEvent"); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + RECEIVER, + Tag.REGION.toString(), + pipeConsensusReceiver.getConsensusGroupIdStr(), + Tag.TYPE.toString(), + "receiveTsFileEvent"); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 60a0b88beb902..54d87766aacb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFilePieceReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealWithModReq; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusReceiverMetric; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -89,7 +90,7 @@ public class PipeConsensusReceiver { * IOTDB_CONFIG.getPipeConsensusPipelineSize(); private static final long CLOSE_TSFILE_WRITER_MAX_WAIT_TIME_IN_MS = 5000; private static final long RETRY_WAIT_TIME = 500; - private final RequestExecutor requestExecutor = new RequestExecutor(); + private final RequestExecutor requestExecutor; private final PipeConsensus pipeConsensus; private final ConsensusGroupId consensusGroupId; // Used to buffer TsFile when transfer TsFile asynchronously. @@ -97,6 +98,7 @@ public class PipeConsensusReceiver { private final PipeConsensusTsFileWriterPool pipeConsensusTsFileWriterPool = new PipeConsensusTsFileWriterPool(); private final AtomicReference receiverFileDirWithIdSuffix = new AtomicReference<>(); + private final PipeConsensusReceiverMetric pipeConsensusReceiverMetric; private FolderManager folderManager; public PipeConsensusReceiver( @@ -105,6 +107,8 @@ public PipeConsensusReceiver( ConsensusPipeName consensusPipeName) { this.pipeConsensus = pipeConsensus; this.consensusGroupId = consensusGroupId; + this.pipeConsensusReceiverMetric = new PipeConsensusReceiverMetric(this); + this.requestExecutor = new RequestExecutor(pipeConsensusReceiverMetric); // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is // data/datanode/system/pipe/consensus/receiver/__consensus{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} @@ -130,6 +134,7 @@ public PipeConsensusReceiver( * load event must be synchronized. */ public TPipeConsensusTransferResp receive(final TPipeConsensusTransferReq req) { + long startNanos = System.nanoTime(); // PreCheck: if there are these cases: read-only; null impl; inactive impl, etc. The receiver // will reject synchronization. TPipeConsensusTransferResp resp = preCheckForReceiver(req); @@ -144,18 +149,26 @@ public TPipeConsensusTransferResp receive(final TPipeConsensusTransferReq req) { case TRANSFER_TS_FILE_PIECE_WITH_MOD: // Just take a place in requestExecutor's buffer, the further seal request will remove // its place from buffer. - requestExecutor.onRequest(req, true); - return loadEvent(req); + requestExecutor.onRequest(req, true, false); + resp = loadEvent(req); + break; case TRANSFER_TS_FILE_SEAL: case TRANSFER_TS_FILE_SEAL_WITH_MOD: // TODO: check memory when logging wal(in further version) + resp = requestExecutor.onRequest(req, false, true); + break; case TRANSFER_TABLET_BINARY: case TRANSFER_TABLET_INSERT_NODE: // TODO: support batch transfer(in further version) case TRANSFER_TABLET_BATCH: default: - return requestExecutor.onRequest(req, false); + resp = requestExecutor.onRequest(req, false, false); + break; } + // update receive an event's duration + long durationNanos = System.nanoTime() - startNanos; + pipeConsensusReceiverMetric.recordReceiveEventTimer(durationNanos); + return resp; } // Unknown request type, which means the request can not be handled by this receiver, // maybe the version of the receiver is not compatible with the sender @@ -287,15 +300,19 @@ private TPipeConsensusTransferResp handleTransferFilePiece( final PipeConsensusTransferFilePieceReq req, final boolean isSingleFile) { LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile pieces", consensusGroupId); - PipeConsensusTsFileWriter diskBuffer = + long startBorrowTsFileWriterNanos = System.nanoTime(); + PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); + long startPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + startPreCheckNanos - startBorrowTsFileWriterNanos); try { - updateWritingFileIfNeeded(diskBuffer, req.getFileName(), isSingleFile); - final File writingFile = diskBuffer.getWritingFile(); - final RandomAccessFile writingFileWriter = diskBuffer.getWritingFileWriter(); + updateWritingFileIfNeeded(tsFileWriter, req.getFileName(), isSingleFile); + final File writingFile = tsFileWriter.getWritingFile(); + final RandomAccessFile writingFileWriter = tsFileWriter.getWritingFileWriter(); - if (isWritingFileOffsetNonCorrect(diskBuffer, req.getStartWritingOffset())) { + if (isWritingFileOffsetNonCorrect(tsFileWriter, req.getStartWritingOffset())) { if (!writingFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { // If the file is a tsFile, then the content will not be changed for a specific // filename. However, for other files (mod, snapshot, etc.) the content varies for the @@ -318,7 +335,11 @@ private TPipeConsensusTransferResp handleTransferFilePiece( status, writingFileWriter.length()); } + long endPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordTsFilePiecePreCheckTime( + endPreCheckNanos - startPreCheckNanos); writingFileWriter.write(req.getFilePiece()); + pipeConsensusReceiverMetric.recordTsFilePieceWriteTime(System.nanoTime() - endPreCheckNanos); return PipeConsensusTransferFilePieceResp.toTPipeConsensusTransferResp( RpcUtils.SUCCESS_STATUS, writingFileWriter.length()); } catch (Exception e) { @@ -343,8 +364,12 @@ private TPipeConsensusTransferResp handleTransferFilePiece( private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsFileSealReq req) { LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile seal", consensusGroupId); + long startBorrowTsFileWriterNanos = System.nanoTime(); PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); + long startPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + startPreCheckNanos - startBorrowTsFileWriterNanos); File writingFile = tsFileWriter.getWritingFile(); RandomAccessFile writingFileWriter = tsFileWriter.getWritingFileWriter(); @@ -381,10 +406,14 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF // writingFile will be deleted after load if no exception occurs tsFileWriter.setWritingFile(null); + long endPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordTsFileSealPreCheckTimer( + endPreCheckNanos - startPreCheckNanos); final TSStatus status = loadFileToDataRegion( fileAbsolutePath, ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex()))); + pipeConsensusReceiverMetric.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); @@ -436,8 +465,12 @@ private TPipeConsensusTransferResp handleTransferFileSealWithMods( LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile seal with mods", consensusGroupId); + long startBorrowTsFileWriterNanos = System.nanoTime(); PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); + long startPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + startPreCheckNanos - startBorrowTsFileWriterNanos); File writingFile = tsFileWriter.getWritingFile(); RandomAccessFile writingFileWriter = tsFileWriter.getWritingFileWriter(); @@ -491,10 +524,14 @@ private TPipeConsensusTransferResp handleTransferFileSealWithMods( final List fileAbsolutePaths = files.stream().map(File::getAbsolutePath).collect(Collectors.toList()); + long endPreCheckNanos = System.nanoTime(); + pipeConsensusReceiverMetric.recordTsFileSealPreCheckTimer( + endPreCheckNanos - startPreCheckNanos); final TSStatus status = loadFileToDataRegion( fileAbsolutePaths.get(1), ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex()))); + pipeConsensusReceiverMetric.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); @@ -1076,35 +1113,54 @@ private class RequestExecutor { // An ordered set that buffers transfer requests' TCommitId, whose length is not larger than // PIPE_CONSENSUS_PIPELINE_SIZE. // Here we use set is to avoid duplicate events being received in some special cases - private final TreeSet reqExecutionOrderBuffer; + private final TreeSet reqExecutionOrderBuffer; private final Lock lock; private final Condition condition; + private final PipeConsensusReceiverMetric metric; private long onSyncedCommitIndex = -1; private int connectorRebootTimes = 0; + private int walEventCount = 0; + private int tsFileEventCount = 0; - public RequestExecutor() { - reqExecutionOrderBuffer = + public RequestExecutor(PipeConsensusReceiverMetric metric) { + this.reqExecutionOrderBuffer = new TreeSet<>( - Comparator.comparingInt(TCommitId::getRebootTimes) - .thenComparingLong(TCommitId::getCommitIndex)); - lock = new ReentrantLock(); - condition = lock.newCondition(); + Comparator.comparingInt(RequestMeta::getRebootTimes) + .thenComparingLong(RequestMeta::getCommitIndex)); + this.lock = new ReentrantLock(); + this.condition = lock.newCondition(); + this.metric = metric; } - private void onSuccess(long nextSyncedCommitIndex) { + private void onSuccess(long nextSyncedCommitIndex, boolean isTransferTsFileSeal) { LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: process no.{} event successfully!", consensusGroupId, nextSyncedCommitIndex); - reqExecutionOrderBuffer.pollFirst(); + RequestMeta curMeta = reqExecutionOrderBuffer.pollFirst(); onSyncedCommitIndex = nextSyncedCommitIndex; + // update metric, notice that curMeta is never null. + if (isTransferTsFileSeal) { + tsFileEventCount--; + metric.recordReceiveTsFileTimer(System.nanoTime() - curMeta.getStartApplyNanos()); + } else { + walEventCount--; + metric.recordReceiveWALTimer(System.nanoTime() - curMeta.getStartApplyNanos()); + } } private TPipeConsensusTransferResp onRequest( - final TPipeConsensusTransferReq req, final boolean isTransferTsFilePiece) { + final TPipeConsensusTransferReq req, + final boolean isTransferTsFilePiece, + final boolean isTransferTsFileSeal) { + long startAcquireLockNanos = System.nanoTime(); lock.lock(); try { + long startDispatchNanos = System.nanoTime(); + metric.recordAcquireExecutorLockTimer(startDispatchNanos - startAcquireLockNanos); + TCommitId tCommitId = req.getCommitId(); + RequestMeta requestMeta = new RequestMeta(tCommitId); LOGGER.info( "PipeConsensus-ConsensusGroup-{}: start to receive no.{} event", consensusGroupId, @@ -1134,16 +1190,26 @@ private TPipeConsensusTransferResp onRequest( if (tCommitId.getRebootTimes() > connectorRebootTimes) { resetWithNewestRebootTime(tCommitId.getRebootTimes()); } - reqExecutionOrderBuffer.add(tCommitId); + reqExecutionOrderBuffer.add(requestMeta); + // update metric + if (isTransferTsFilePiece) { + tsFileEventCount++; + } else if (!isTransferTsFileSeal) { + walEventCount++; + } + // TsFilePieceTransferEvent will not enter further procedure, it just holds a place in // buffer. Only after the corresponding sealing event is processed, this event can be // dequeued. if (isTransferTsFilePiece) { + long startApplyNanos = System.nanoTime(); + metric.recordDispatchWaitingTimer(startApplyNanos - startDispatchNanos); + requestMeta.setStartApplyNanos(startApplyNanos); return null; } if (reqExecutionOrderBuffer.size() >= IOTDB_CONFIG.getPipeConsensusPipelineSize() - && !reqExecutionOrderBuffer.first().equals(tCommitId)) { + && !reqExecutionOrderBuffer.first().equals(requestMeta)) { // If reqBuffer is full and current thread do not hold the reqBuffer's peek, this req // is not supposed to be processed. So current thread should notify the corresponding // threads to process the peek. @@ -1152,8 +1218,11 @@ private TPipeConsensusTransferResp onRequest( // Polling to process while (true) { - if (reqExecutionOrderBuffer.first().equals(tCommitId) + if (reqExecutionOrderBuffer.first().equals(requestMeta) && tCommitId.getCommitIndex() == onSyncedCommitIndex + 1) { + long startApplyNanos = System.nanoTime(); + metric.recordDispatchWaitingTimer(startApplyNanos - startDispatchNanos); + requestMeta.setStartApplyNanos(startApplyNanos); // If current req is supposed to be process, load this event through // DataRegionStateMachine. TPipeConsensusTransferResp resp = loadEvent(req); @@ -1164,19 +1233,22 @@ private TPipeConsensusTransferResp onRequest( // when the last seal req is applied, we can discard this event. if (resp != null && resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - onSuccess(onSyncedCommitIndex + 1); + onSuccess(onSyncedCommitIndex + 1, isTransferTsFilePiece); } return resp; } if (reqExecutionOrderBuffer.size() >= IOTDB_CONFIG.getPipeConsensusPipelineSize() - && reqExecutionOrderBuffer.first().equals(tCommitId)) { + && reqExecutionOrderBuffer.first().equals(requestMeta)) { + long startApplyNanos = System.nanoTime(); + metric.recordDispatchWaitingTimer(startApplyNanos - startDispatchNanos); + requestMeta.setStartApplyNanos(startApplyNanos); // If the reqBuffer is full and its peek is hold by current thread, load this event. TPipeConsensusTransferResp resp = loadEvent(req); if (resp != null && resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - onSuccess(tCommitId.getCommitIndex()); + onSuccess(tCommitId.getCommitIndex(), isTransferTsFileSeal); // signal all other reqs that may wait for this event condition.signalAll(); } @@ -1196,12 +1268,15 @@ private TPipeConsensusTransferResp onRequest( if (timeout && reqExecutionOrderBuffer.size() < IOTDB_CONFIG.getPipeConsensusPipelineSize() && reqExecutionOrderBuffer.first() != null - && reqExecutionOrderBuffer.first().equals(tCommitId)) { + && reqExecutionOrderBuffer.first().equals(requestMeta)) { + long startApplyNanos = System.nanoTime(); + metric.recordDispatchWaitingTimer(startApplyNanos - startDispatchNanos); + requestMeta.setStartApplyNanos(startApplyNanos); TPipeConsensusTransferResp resp = loadEvent(req); if (resp != null && resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - onSuccess(tCommitId.getCommitIndex()); + onSuccess(tCommitId.getCommitIndex(), isTransferTsFileSeal); // signal all other reqs that may wait for this event condition.signalAll(); } @@ -1241,4 +1316,63 @@ private void resetWithNewestRebootTime(int connectorRebootTimes) { this.connectorRebootTimes = connectorRebootTimes; } } + + private static class RequestMeta { + private final TCommitId commitId; + private long startApplyNanos = 0; + + public RequestMeta(TCommitId commitId) { + this.commitId = commitId; + } + + public int getRebootTimes() { + return commitId.getRebootTimes(); + } + + public long getCommitIndex() { + return commitId.getCommitIndex(); + } + + public void setStartApplyNanos(long startApplyNanos) { + // Notice that a tsFileInsertionEvent will enter RequestExecutor multiple times, we only need + // to record the time of the first apply + if (startApplyNanos == 0) { + this.startApplyNanos = startApplyNanos; + } + } + + public long getStartApplyNanos() { + return startApplyNanos; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RequestMeta that = (RequestMeta) o; + return commitId.equals(that.commitId); + } + } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + public int getReceiveBufferSize() { + return this.requestExecutor.reqExecutionOrderBuffer.size(); + } + + public int getWALEventCount() { + return this.requestExecutor.walEventCount; + } + + public int getTsFileEventCount() { + return this.requestExecutor.tsFileEventCount; + } + + public String getConsensusGroupIdStr() { + return consensusGroupId.toString(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/container/PipeConsensusClientMgrContainer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/container/PipeConsensusClientMgrContainer.java index 5a52eb3440932..6fc041634331e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/container/PipeConsensusClientMgrContainer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/container/PipeConsensusClientMgrContainer.java @@ -42,7 +42,7 @@ public class PipeConsensusClientMgrContainer { private final IClientManager asyncClientManager; private final IClientManager syncClientManager; - public PipeConsensusClientMgrContainer() { + private PipeConsensusClientMgrContainer() { // load rpc client config PipeConsensusClientProperty config = PipeConsensusClientProperty.newBuilder() diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java index 3d0bd4cab7468..2dbf8525fde30 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java @@ -185,6 +185,7 @@ public class PipeConnectorConstant { public static final String SINK_CONSUMER_GROUP_KEY = "sink.consumer-group"; public static final String CONNECTOR_CONSENSUS_GROUP_ID_KEY = "connector.consensus.group-id"; + public static final String CONNECTOR_CONSENSUS_PIPE_NAME = "connector.consensus.pipe-name"; private PipeConnectorConstant() { throw new IllegalStateException("Utility class"); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java index 3742c3c60288c..2e24117a7a40d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java @@ -342,7 +342,7 @@ public boolean isReleased() { } /** - * Used for pipeConsensus. In PipeConsensus, we only need commiterKey, commitId and rebootTimes to + * Used for pipeConsensus. In PipeConsensus, we only need committerKey, commitId and rebootTimes to * uniquely identify an event */ public boolean equalsInPipeConsensus(Object o) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java index 9f9369ae56b63..0b740f7e12a78 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitManager.java @@ -137,6 +137,14 @@ private PipeEventCommitManager() { // Do nothing but make it private. } + public long getGivenConsensusPipeCommitId(String committerKey) { + final PipeEventCommitter committer = eventCommitterMap.get(committerKey); + if (committer == null) { + return 0; + } + return committer.getCurrentCommitId(); + } + private static class PipeEventCommitManagerHolder { private static final PipeEventCommitManager INSTANCE = new PipeEventCommitManager(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitter.java index 04502441daf13..559e53ee15b9f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/progress/PipeEventCommitter.java @@ -136,4 +136,8 @@ public int getRegionId() { public long commitQueueSize() { return commitQueue.size(); } + + public long getCurrentCommitId() { + return commitIdGenerator.get(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index c207435d3f191..1281c9cf16b35 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -53,6 +53,10 @@ public enum Metric { // consensus related STAGE("stage"), IOT_CONSENSUS("iot_consensus"), + PIPE_CONSENSUS("pipe_consensus"), + PIPE_SEND_EVENT("pipe_send_event"), + PIPE_RETRY_SEND_EVENT("pipe_retry_send_event"), + PIPE_RECEIVE_EVENT("pipe_receive_event"), IOT_SEND_LOG("iot_send_log"), IOT_RECEIVE_LOG("iot_receive_log"), RATIS_CONSENSUS_WRITE("ratis_consensus_write"), From 86f8984668afcc8da1f0c5707672769083c6e43b Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Wed, 12 Jun 2024 17:14:23 +0800 Subject: [PATCH 02/22] fix reference bug and enqueue bug --- .../PipeConsensusAsyncConnector.java | 26 ++++++++++++------- .../pipeconsensus/PipeConsensusReceiver.java | 4 +-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 12e822679fa00..62965797961fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -148,7 +148,7 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati MetricService.getInstance().addMetricSet(this.pipeConsensusConnectorMetric); } - /** Add an event to transferBuffer, whose events will be asynchronizedly transfer to receiver. */ + /** Add an event to transferBuffer, whose events will be asynchronously transfer to receiver. */ private boolean addEvent2Buffer(EnrichedEvent event) { try { if (LOGGER.isDebugEnabled()) { @@ -215,13 +215,12 @@ public void heartbeat() throws Exception { @Override public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception { + syncTransferQueuedEventsIfNecessary(); + boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tabletInsertionEvent); if (!enqueueResult) { throw new PipeException(ENQUEUE_EXCEPTION_MSG); } - - syncTransferQueuedEventsIfNecessary(); - // batch transfer tablets. if (isTabletBatchModeEnabled) { if (tabletBatchBuilder.onEvent(tabletInsertionEvent)) { @@ -302,11 +301,6 @@ private void transfer( @Override public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception { - boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tsFileInsertionEvent); - if (!enqueueResult) { - throw new PipeException(ENQUEUE_EXCEPTION_MSG); - } - syncTransferQueuedEventsIfNecessary(); transferBatchedEventsIfNecessary(); @@ -317,6 +311,10 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception return; } + boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tsFileInsertionEvent); + if (!enqueueResult) { + throw new PipeException(ENQUEUE_EXCEPTION_MSG); + } final PipeTsFileInsertionEvent pipeTsFileInsertionEvent = (PipeTsFileInsertionEvent) tsFileInsertionEvent; TCommitId tCommitId = @@ -404,7 +402,7 @@ private void transferBatchedEventsIfNecessary() throws IOException { * retry the {@link Event} and mark the {@link Event} as failure and stop the pipe if the * retry times exceeds the threshold. */ - private synchronized void syncTransferQueuedEventsIfNecessary() throws Exception { + private void syncTransferQueuedEventsIfNecessary() throws Exception { while (!retryEventQueue.isEmpty()) { synchronized (this) { if (isClosed.get() || retryEventQueue.isEmpty()) { @@ -502,6 +500,13 @@ public synchronized void clearRetryEventsReferenceCount() { } } + public synchronized void clearTransferBufferReferenceCount() { + while (!transferBuffer.isEmpty()) { + final EnrichedEvent event = transferBuffer.poll(); + event.clearReferenceCount(PipeConsensusAsyncConnector.class.getName()); + } + } + private void logOnClientException( final AsyncPipeConsensusServiceClient client, final Exception e) { if (client == null) { @@ -530,6 +535,7 @@ public synchronized void close() { retryConnector.close(); clearRetryEventsReferenceCount(); + clearTransferBufferReferenceCount(); if (tabletBatchBuilder != null) { tabletBatchBuilder.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 54d87766aacb7..51b6f657f4da7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -99,7 +99,7 @@ public class PipeConsensusReceiver { new PipeConsensusTsFileWriterPool(); private final AtomicReference receiverFileDirWithIdSuffix = new AtomicReference<>(); private final PipeConsensusReceiverMetric pipeConsensusReceiverMetric; - private FolderManager folderManager; + private final FolderManager folderManager; public PipeConsensusReceiver( PipeConsensus pipeConsensus, @@ -1233,7 +1233,7 @@ private TPipeConsensusTransferResp onRequest( // when the last seal req is applied, we can discard this event. if (resp != null && resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - onSuccess(onSyncedCommitIndex + 1, isTransferTsFilePiece); + onSuccess(onSyncedCommitIndex + 1, isTransferTsFileSeal); } return resp; } From 06658d2220bb5356da0da4588a51848b907df191 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Wed, 12 Jun 2024 17:26:12 +0800 Subject: [PATCH 03/22] spotless --- .../iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java | 8 ++------ .../PipeConsensusTabletInsertNodeEventHandler.java | 8 ++++---- .../apache/iotdb/commons/pipe/event/EnrichedEvent.java | 4 ++-- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 29972a0e69a5a..b11158fcfdf68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -555,15 +555,11 @@ private void changePipeStatusBeforeRestart(final String pipeName) { dataRegionPipeTasks.parallelStream().forEach(PipeTask::drop); // Stop schema region tasks - pipeTaskManager - .getPipeTasks(pipeMeta.getStaticMeta()) - .values() - .parallelStream() + pipeTaskManager.getPipeTasks(pipeMeta.getStaticMeta()).values().parallelStream() .forEach(PipeTask::stop); // Re-create data region tasks - dataRegionPipeTasks - .parallelStream() + dataRegionPipeTasks.parallelStream() .forEach( pipeTask -> { final PipeTask newPipeTask = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java index 0f0cb91091c5a..981fe74f7a61b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java @@ -32,10 +32,10 @@ public class PipeConsensusTabletInsertNodeEventHandler extends PipeConsensusTabletInsertionEventHandler { public PipeConsensusTabletInsertNodeEventHandler( - PipeInsertNodeTabletInsertionEvent event, - TPipeConsensusTransferReq req, - PipeConsensusAsyncConnector connector, - PipeConsensusConnectorMetric metric) { + PipeInsertNodeTabletInsertionEvent event, + TPipeConsensusTransferReq req, + PipeConsensusAsyncConnector connector, + PipeConsensusConnectorMetric metric) { super(event, req, connector, metric); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java index 2e24117a7a40d..b43e37788cd46 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java @@ -342,8 +342,8 @@ public boolean isReleased() { } /** - * Used for pipeConsensus. In PipeConsensus, we only need committerKey, commitId and rebootTimes to - * uniquely identify an event + * Used for pipeConsensus. In PipeConsensus, we only need committerKey, commitId and rebootTimes + * to uniquely identify an event */ public boolean equalsInPipeConsensus(Object o) { if (this == o) { From 158f45e862bbd509c663780b4cc01a16243de384 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 00:01:37 +0800 Subject: [PATCH 04/22] fix metric bug --- .../pipe/PipeConsensusServerImpl.java | 2 +- .../metric/PipeConsensusServerMetrics.java | 4 +-- .../metric/PipeConsensusSyncLagManager.java | 25 +++++++++++-------- .../PipeConsensusAsyncConnector.java | 5 +++- .../pipeconsensus/PipeConsensusReceiver.java | 5 ++++ .../commons/service/metric/enums/Metric.java | 4 +-- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index 79b83c3fe1410..bed222d7ae40e 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -161,7 +161,7 @@ public synchronized void stop() { // do not roll back, because it will stop anyway LOGGER.warn("{} cannot stop all consensus pipes", thisNode); } - + MetricService.getInstance().removeMetricSet(this.pipeConsensusServerMetrics); stateMachine.stop(); isStarted.set(false); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java index 47cd626dd16ca..563a09af4f09b 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -31,14 +31,14 @@ public class PipeConsensusServerMetrics implements IMetricSet { private final PipeConsensusServerImpl impl; - private final PipeConsensusSyncLagManager syncLagManager = - PipeConsensusSyncLagManager.getInstance(); + private final PipeConsensusSyncLagManager syncLagManager; private Timer getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; public PipeConsensusServerMetrics(PipeConsensusServerImpl impl) { this.impl = impl; + this.syncLagManager = PipeConsensusSyncLagManager.getInstance(impl.getConsensusGroupId()); } private static final String IMPL = "PipeConsensusServerImpl"; diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index b01b5406c47c7..3390f5d232472 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -22,6 +22,8 @@ import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeConnector; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; /** @@ -34,9 +36,10 @@ public class PipeConsensusSyncLagManager { List consensusPipeConnectorList = new CopyOnWriteArrayList<>(); private void updateReplicateProgress() { - // if there isn't a consensus pipe task, replicate progress is 0. + // if there isn't a consensus pipe task, replicate progress is Long.MAX_VALUE. if (consensusPipeConnectorList.isEmpty()) { - minReplicateProgress = 0; + minReplicateProgress = Long.MAX_VALUE; + return; } // else we find the minimum progress in all consensus pipe task. consensusPipeConnectorList.forEach( @@ -48,9 +51,10 @@ private void updateReplicateProgress() { } private void updateUserWriteProgress() { - // if there isn't a consensus pipe task, user write progress is Long.MAX_VALUE. + // if there isn't a consensus pipe task, user write progress is 0. if (consensusPipeConnectorList.isEmpty()) { - userWriteProgress = Long.MAX_VALUE; + userWriteProgress = 0; + return; } // since the user write progress of different consensus pipes on the same DataRegion is the // same, we only need to take out one Connector to calculate @@ -84,21 +88,22 @@ private PipeConsensusSyncLagManager() { /** singleton */ private static class PipeConsensusSyncLagManagerHolder { - private static PipeConsensusSyncLagManager INSTANCE; + private static Map INSTANCE_MAP; private PipeConsensusSyncLagManagerHolder() { // empty constructor } - public static void build() { - if (INSTANCE == null) { - INSTANCE = new PipeConsensusSyncLagManager(); + private static void build() { + if (INSTANCE_MAP == null) { + INSTANCE_MAP = new ConcurrentHashMap<>(); } } } - public static PipeConsensusSyncLagManager getInstance() { - return PipeConsensusSyncLagManagerHolder.INSTANCE; + public static PipeConsensusSyncLagManager getInstance(String groupId) { + return PipeConsensusSyncLagManagerHolder.INSTANCE_MAP.computeIfAbsent( + groupId, key -> new PipeConsensusSyncLagManager()); } // Only when consensus protocol is PipeConsensus, this method will be called once when construct diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 62965797961fb..249ce9a7fc858 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -144,7 +144,8 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati // initialize metric components pipeConsensusConnectorMetric = new PipeConsensusConnectorMetric(this); - PipeConsensusSyncLagManager.getInstance().addConsensusPipeConnector(this); + PipeConsensusSyncLagManager.getInstance(getConsensusGroupIdStr()) + .addConsensusPipeConnector(this); MetricService.getInstance().addMetricSet(this.pipeConsensusConnectorMetric); } @@ -540,6 +541,8 @@ public synchronized void close() { if (tabletBatchBuilder != null) { tabletBatchBuilder.close(); } + + MetricService.getInstance().removeMetricSet(this.pipeConsensusConnectorMetric); } //////////////////////////// APIs provided for metric framework //////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 51b6f657f4da7..89e84739ba9da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.pipeconsensus.response.PipeConsensusTransferFilePieceResp; import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp; import org.apache.iotdb.commons.pipe.receiver.IoTDBReceiverAgent; +import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; import org.apache.iotdb.consensus.pipe.PipeConsensus; import org.apache.iotdb.consensus.pipe.PipeConsensusServerImpl; @@ -109,6 +110,7 @@ public PipeConsensusReceiver( this.consensusGroupId = consensusGroupId; this.pipeConsensusReceiverMetric = new PipeConsensusReceiverMetric(this); this.requestExecutor = new RequestExecutor(pipeConsensusReceiverMetric); + MetricService.getInstance().addMetricSet(pipeConsensusReceiverMetric); // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is // data/datanode/system/pipe/consensus/receiver/__consensus{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} @@ -936,6 +938,9 @@ public synchronized void handleExit() { } } + // remove metric + MetricService.getInstance().removeMetricSet(pipeConsensusReceiverMetric); + LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Receiver exited.", consensusGroupId.getId()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 1281c9cf16b35..fa3b36890a6c4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -53,12 +53,12 @@ public enum Metric { // consensus related STAGE("stage"), IOT_CONSENSUS("iot_consensus"), + IOT_SEND_LOG("iot_send_log"), + IOT_RECEIVE_LOG("iot_receive_log"), PIPE_CONSENSUS("pipe_consensus"), PIPE_SEND_EVENT("pipe_send_event"), PIPE_RETRY_SEND_EVENT("pipe_retry_send_event"), PIPE_RECEIVE_EVENT("pipe_receive_event"), - IOT_SEND_LOG("iot_send_log"), - IOT_RECEIVE_LOG("iot_receive_log"), RATIS_CONSENSUS_WRITE("ratis_consensus_write"), RATIS_CONSENSUS_READ("ratis_consensus_read"), // storage engine related From 894a3f00d1361d493f70d97383bedb430fe2d2f8 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 00:05:59 +0800 Subject: [PATCH 05/22] fix comment --- .../pipe/metric/PipeConsensusSyncLagManager.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index 3390f5d232472..843c8cbd9fd59 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -29,6 +29,8 @@ /** * This class is used to aggregate the write progress of all Connectors to calculate the minimum * synchronization progress of all follower copies, thereby calculating syncLag. + * + *

Note: every consensusGroup/dataRegion has and only has 1 instance of this class. */ public class PipeConsensusSyncLagManager { long userWriteProgress = 0; @@ -86,23 +88,22 @@ private PipeConsensusSyncLagManager() { // do nothing } - /** singleton */ private static class PipeConsensusSyncLagManagerHolder { - private static Map INSTANCE_MAP; + private static Map CONSENSU_GROUP_ID_2_INSTANCE_MAP; private PipeConsensusSyncLagManagerHolder() { // empty constructor } private static void build() { - if (INSTANCE_MAP == null) { - INSTANCE_MAP = new ConcurrentHashMap<>(); + if (CONSENSU_GROUP_ID_2_INSTANCE_MAP == null) { + CONSENSU_GROUP_ID_2_INSTANCE_MAP = new ConcurrentHashMap<>(); } } } public static PipeConsensusSyncLagManager getInstance(String groupId) { - return PipeConsensusSyncLagManagerHolder.INSTANCE_MAP.computeIfAbsent( + return PipeConsensusSyncLagManagerHolder.CONSENSU_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent( groupId, key -> new PipeConsensusSyncLagManager()); } From b764ebce6642e6b650fcb223f1ca875a086b777c Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 00:22:30 +0800 Subject: [PATCH 06/22] fix syncLag bug --- .../pipe/task/subtask/processor/PipeProcessorSubtask.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java index 2b48becaed2ad..32f37e208182e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.metric.PipeProcessorMetrics; +import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor; import org.apache.iotdb.db.pipe.task.connection.PipeEventCollector; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.utils.ErrorHandlingUtils; @@ -148,8 +149,12 @@ protected boolean executeOnce() throws Exception { outputEventCollector); } } + final boolean shouldReport = - !isClosed.get() && outputEventCollector.hasNoCollectInvocationAfterReset(); + !isClosed.get() + && outputEventCollector.hasNoCollectInvocationAfterReset() + // Events generated from consensusPipe's transferred data should never be reported. + && !(pipeProcessor instanceof PipeConsensusProcessor); if (shouldReport && event instanceof EnrichedEvent) { PipeEventCommitManager.getInstance() .enrichWithCommitterKeyAndCommitId((EnrichedEvent) event, creationTime, regionId); From 35e3362b254412d0cb0d3b7bcc3bb5051e98ce0e Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 10:31:03 +0800 Subject: [PATCH 07/22] fix gauge invalid bug --- .../consensus/pipe/metric/PipeConsensusServerMetrics.java | 4 ++-- .../org/apache/iotdb/commons/service/metric/enums/Metric.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java index 563a09af4f09b..672c8fdeb33e1 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -68,7 +68,7 @@ public void unbindFrom(AbstractMetricService metricService) { public void bindGauge(AbstractMetricService metricService) { metricService .getOrCreateGauge( - Metric.PIPE_CONSENSUS.toString(), + Metric.PIPE_CONSENSUS_MODE.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), IMPL, @@ -80,7 +80,7 @@ public void bindGauge(AbstractMetricService metricService) { public void unbindGauge(AbstractMetricService metricService) { metricService.remove( MetricType.GAUGE, - Metric.PIPE_CONSENSUS.toString(), + Metric.PIPE_CONSENSUS_MODE.toString(), Tag.NAME.toString(), IMPL, Tag.TYPE.toString(), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index fa3b36890a6c4..76a40ce2e3d9a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -56,6 +56,7 @@ public enum Metric { IOT_SEND_LOG("iot_send_log"), IOT_RECEIVE_LOG("iot_receive_log"), PIPE_CONSENSUS("pipe_consensus"), + PIPE_CONSENSUS_MODE("pipe_consensus_mode"), PIPE_SEND_EVENT("pipe_send_event"), PIPE_RETRY_SEND_EVENT("pipe_retry_send_event"), PIPE_RECEIVE_EVENT("pipe_receive_event"), From 6a44de80c23af14daea7dfc6348f34eebb550a3f Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 11:03:10 +0800 Subject: [PATCH 08/22] fix review --- .../pipe/PipeConsensusServerImpl.java | 12 ++- .../consensuspipe/ConsensusPipeConnector.java | 4 +- .../metric/PipeConsensusSyncLagManager.java | 4 +- .../agent/task/PipeDataNodeTaskAgent.java | 4 +- .../PipeConsensusAsyncConnector.java | 74 +++++++++++-------- .../PipeConsensusSyncConnector.java | 12 +-- ...ConsensusTabletInsertNodeEventHandler.java | 4 +- ...eConsensusTabletInsertionEventHandler.java | 6 +- ...eConsensusTsFileInsertionEventHandler.java | 6 +- ...ava => PipeConsensusConnectorMetrics.java} | 4 +- ...java => PipeConsensusReceiverMetrics.java} | 4 +- .../pipeconsensus/PipeConsensusReceiver.java | 36 ++++----- 12 files changed, 95 insertions(+), 75 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/{PipeConsensusConnectorMetric.java => PipeConsensusConnectorMetrics.java} (98%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/{PipeConsensusReceiverMetric.java => PipeConsensusReceiverMetrics.java} (98%) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index bed222d7ae40e..c7adf133d385d 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.pipe.task.meta.PipeStatus; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.consensus.IStateMachine; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.common.Peer; @@ -69,7 +70,8 @@ public class PipeConsensusServerImpl { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusServerImpl.class); private static final long CHECK_TRANSMISSION_COMPLETION_INTERVAL_IN_MILLISECONDS = 2_000L; - + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); private final Peer thisNode; private final IStateMachine stateMachine; private final Lock stateMachineLock = new ReentrantLock(); @@ -288,13 +290,16 @@ public TSStatus write(IConsensusRequest request) { // statistic the time of acquiring stateMachine lock pipeConsensusServerMetrics.recordGetStateMachineLockTime( getStateMachineLockTime - consensusWriteStartTime); + long writeToStateMachineStartTime = System.nanoTime(); if (request instanceof ComparableConsensusRequest) { ((ComparableConsensusRequest) request) .setProgressIndex(progressIndexManager.assignProgressIndex(thisNode.getGroupId())); } - long writeToStateMachineStartTime = System.nanoTime(); + long startWriteTime = System.nanoTime(); TSStatus result = stateMachine.write(request); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(System.nanoTime() - startWriteTime); + long writeToStateMachineEndTime = System.nanoTime(); // statistic the time of writing request into stateMachine pipeConsensusServerMetrics.recordWriteStateMachineTime( @@ -317,6 +322,9 @@ public TSStatus writeOnFollowerReplica(IConsensusRequest request) { long writeToStateMachineStartTime = System.nanoTime(); TSStatus result = stateMachine.write(request); long writeToStateMachineEndTime = System.nanoTime(); + + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost( + writeToStateMachineEndTime - writeToStateMachineStartTime); // statistic the time of writing request into stateMachine pipeConsensusServerMetrics.recordWriteStateMachineTime( writeToStateMachineEndTime - writeToStateMachineStartTime); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java index 3b00e5d265394..6f1396db97229 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/ConsensusPipeConnector.java @@ -19,7 +19,7 @@ package org.apache.iotdb.consensus.pipe.consensuspipe; public interface ConsensusPipeConnector { - long getLocalUserWriteProgress(); + long getConsensusPipeCommitProgress(); - long getCurrentPipeReplicateProgress(); + long getConsensusPipeReplicateProgress(); } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index 843c8cbd9fd59..ec5a86d49a53d 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -48,7 +48,7 @@ private void updateReplicateProgress() { consensusPipeConnector -> { minReplicateProgress = Math.min( - minReplicateProgress, consensusPipeConnector.getCurrentPipeReplicateProgress()); + minReplicateProgress, consensusPipeConnector.getConsensusPipeReplicateProgress()); }); } @@ -61,7 +61,7 @@ private void updateUserWriteProgress() { // since the user write progress of different consensus pipes on the same DataRegion is the // same, we only need to take out one Connector to calculate ConsensusPipeConnector connector = consensusPipeConnectorList.get(0); - userWriteProgress = connector.getLocalUserWriteProgress(); + userWriteProgress = connector.getConsensusPipeCommitProgress(); } public void addConsensusPipeConnector(ConsensusPipeConnector consensusPipeConnector) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index b11158fcfdf68..1a0df387e24c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -612,9 +612,9 @@ public Set getPipeTaskRegionIdSet(final String pipeName, final long cre ///////////////////////// Pipe Consensus ///////////////////////// - public PipeStaticMeta getPipeStaticMeta(final String pipeName) { + public long getPipeCreationTime(final String pipeName) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - return pipeMeta == null ? null : pipeMeta.getStaticMeta(); + return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); } public ProgressIndex getPipeTaskProgressIndex(String pipeName, int consensusGroupId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 249ce9a7fc858..445c03b6e64ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -27,11 +27,9 @@ import org.apache.iotdb.commons.client.container.PipeConsensusClientMgrContainer; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; -import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.connector.protocol.IoTDBConnector; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; -import org.apache.iotdb.commons.pipe.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeConnector; import org.apache.iotdb.consensus.pipe.metric.PipeConsensusSyncLagManager; @@ -46,13 +44,14 @@ import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.builder.PipeConsensusAsyncBatchReqBuilder; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTabletBinaryReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTabletInsertNodeReq; -import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetrics; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -72,12 +71,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME; + // TODO: Optimize the network and disk io for TsFile onComplete // TODO: support Tablet Batch public class PipeConsensusAsyncConnector extends IoTDBConnector implements ConsensusPipeConnector { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusAsyncConnector.class); - private static final String CUSTOMIZE_EXCEPTION_MSG = - "Failed to customize pipeConsensusAsyncConnector because there isn't consensusGroupId passed in. Please check your construct parameters."; private static final String ENQUEUE_EXCEPTION_MSG = "Timeout: PipeConsensusConnector offers an event into transferBuffer failed, because transferBuffer is full."; private static final String THRIFT_ERROR_FORMATTER_WITHOUT_ENDPOINT = @@ -89,20 +89,33 @@ public class PipeConsensusAsyncConnector extends IoTDBConnector implements Conse private static final IoTDBConfig IOTDB_CONFIG = IoTDBDescriptor.getInstance().getConfig(); private static final long PIPE_CONSENSUS_EVENT_ENQUEUE_TIMEOUT_IN_MS = IOTDB_CONFIG.getConnectionTimeoutInMS() / 6; - private final BlockingQueue retryEventQueue = new LinkedBlockingQueue<>(); // We use enrichedEvent here to make use of EnrichedEvent.equalsInPipeConsensus private final BlockingQueue transferBuffer = new LinkedBlockingDeque<>(IOTDB_CONFIG.getPipeConsensusPipelineSize()); private final AtomicBoolean isClosed = new AtomicBoolean(false); private final int thisDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); - private PipeConsensusConnectorMetric pipeConsensusConnectorMetric; + private PipeConsensusConnectorMetrics pipeConsensusConnectorMetrics; private String consensusPipeName; private int consensusGroupId; private PipeConsensusSyncConnector retryConnector; private IClientManager asyncTransferClientManager; private PipeConsensusAsyncBatchReqBuilder tabletBatchBuilder; - private long currentReplicateProgress = 0; + private volatile long currentReplicateProgress = 0; + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + super.validate(validator); + // validate consensus pipe's parameters + final PipeParameters parameters = validator.getParameters(); + validator.validate( + args -> (boolean) args[0] || (boolean) args[1], + String.format( + "One of %s, %s must be specified in consensus pipe", + CONNECTOR_CONSENSUS_GROUP_ID_KEY, CONNECTOR_CONSENSUS_PIPE_NAME), + parameters.hasAttribute(CONNECTOR_CONSENSUS_GROUP_ID_KEY), + parameters.hasAttribute(CONNECTOR_CONSENSUS_PIPE_NAME)); + } @Override public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) @@ -110,23 +123,16 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati super.customize(parameters, configuration); // Get consensusGroupId from parameters passed by PipeConsensusImpl - if (!parameters.hasAttribute(PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY)) { - throw new PipeException(CUSTOMIZE_EXCEPTION_MSG); - } - consensusGroupId = parameters.getInt(PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY); - + consensusGroupId = parameters.getInt(CONNECTOR_CONSENSUS_GROUP_ID_KEY); // Get consensusPipeName from parameters passed by PipeConsensusImpl - if (!parameters.hasAttribute(PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME)) { - throw new PipeException(CUSTOMIZE_EXCEPTION_MSG); - } - consensusPipeName = parameters.getString(PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME); + consensusPipeName = parameters.getString(CONNECTOR_CONSENSUS_PIPE_NAME); // In PipeConsensus, one pipeConsensusTask corresponds to a pipeConsensusConnector. Thus, // `nodeUrls` here actually is a singletonList that contains one peer's TEndPoint. But here we // retain the implementation of list to cope with possible future expansion retryConnector = new PipeConsensusSyncConnector( - nodeUrls, consensusGroupId, thisDataNodeId, pipeConsensusConnectorMetric); + nodeUrls, consensusGroupId, thisDataNodeId, pipeConsensusConnectorMetrics); retryConnector.customize(parameters, configuration); asyncTransferClientManager = PipeConsensusClientMgrContainer.getInstance().getAsyncClientManager(); @@ -143,13 +149,15 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati isTabletBatchModeEnabled = false; // initialize metric components - pipeConsensusConnectorMetric = new PipeConsensusConnectorMetric(this); + pipeConsensusConnectorMetrics = new PipeConsensusConnectorMetrics(this); PipeConsensusSyncLagManager.getInstance(getConsensusGroupIdStr()) .addConsensusPipeConnector(this); - MetricService.getInstance().addMetricSet(this.pipeConsensusConnectorMetric); + MetricService.getInstance().addMetricSet(this.pipeConsensusConnectorMetrics); } - /** Add an event to transferBuffer, whose events will be asynchronously transfer to receiver. */ + /** + * Add an event to transferBuffer, whose events will be asynchronously transferred to receiver. + */ private boolean addEvent2Buffer(EnrichedEvent event) { try { if (LOGGER.isDebugEnabled()) { @@ -164,11 +172,16 @@ private boolean addEvent2Buffer(EnrichedEvent event) { transferBuffer.offer( event, PIPE_CONSENSUS_EVENT_ENQUEUE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); long duration = System.nanoTime() - currentTime; - pipeConsensusConnectorMetric.recordConnectorEnqueueTimer(duration); + pipeConsensusConnectorMetrics.recordConnectorEnqueueTimer(duration); // add reference if (result) { event.increaseReferenceCount(PipeConsensusAsyncConnector.class.getName()); } + // if connector is closed when executing this method, need to clear this event's reference + // count to avoid unnecessarily pinning some resource such as WAL. + if (isClosed.get()) { + event.clearReferenceCount(PipeConsensusAsyncConnector.class.getName()); + } return result; } catch (InterruptedException e) { LOGGER.info("PipeConsensusConnector transferBuffer queue offer is interrupted.", e); @@ -270,7 +283,7 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception pipeInsertNodeTabletInsertionEvent, pipeConsensusTransferReq, this, - pipeConsensusConnectorMetric); + pipeConsensusConnectorMetrics); transfer(pipeConsensusInsertNodeReqHandler); } @@ -344,7 +357,7 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception tCommitId, tConsensusGroupId, thisDataNodeId, - pipeConsensusConnectorMetric); + pipeConsensusConnectorMetrics); transfer(pipeConsensusTsFileInsertionEventHandler); } catch (Exception e) { @@ -488,7 +501,7 @@ public void addFailureEventToRetryQueue(final Event event) { public void addFailureEventsToRetryQueue(final Iterable events) { for (final Event event : events) { addFailureEventToRetryQueue(event); - pipeConsensusConnectorMetric.recordRetryCounter(); + pipeConsensusConnectorMetrics.recordRetryCounter(); } } @@ -542,7 +555,7 @@ public synchronized void close() { tabletBatchBuilder.close(); } - MetricService.getInstance().removeMetricSet(this.pipeConsensusConnectorMetric); + MetricService.getInstance().removeMetricSet(this.pipeConsensusConnectorMetrics); } //////////////////////////// APIs provided for metric framework //////////////////////////// @@ -556,16 +569,15 @@ public int getRetryBufferSize() { } @Override - public long getLocalUserWriteProgress() { - PipeStaticMeta staticMeta = PipeAgent.task().getPipeStaticMeta(consensusPipeName); + public long getConsensusPipeCommitProgress() { + long creationTime = PipeAgent.task().getPipeCreationTime(consensusPipeName); String committerKey = - String.format( - "%s_%s_%s", staticMeta.getPipeName(), consensusGroupId, staticMeta.getCreationTime()); + String.format("%s_%s_%s", consensusPipeName, consensusGroupId, creationTime); return PipeEventCommitManager.getInstance().getGivenConsensusPipeCommitId(committerKey); } @Override - public long getCurrentPipeReplicateProgress() { + public long getConsensusPipeReplicateProgress() { return currentReplicateProgress; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java index ed7e09a045b10..c10c5bf93bfd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFilePieceWithModReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealReq; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.request.PipeConsensusTsFileSealWithModReq; -import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetrics; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -76,14 +76,14 @@ public class PipeConsensusSyncConnector extends IoTDBConnector { private final List peers; private final int thisDataNodeId; private final int consensusGroupId; - private final PipeConsensusConnectorMetric pipeConsensusConnectorMetric; + private final PipeConsensusConnectorMetrics pipeConsensusConnectorMetrics; private PipeConsensusSyncBatchReqBuilder tabletBatchBuilder; public PipeConsensusSyncConnector( List peers, int consensusGroupId, int thisDataNodeId, - PipeConsensusConnectorMetric pipeConsensusConnectorMetric) { + PipeConsensusConnectorMetrics pipeConsensusConnectorMetrics) { // In PipeConsensus, one pipeConsensusTask corresponds to a pipeConsensusConnector. Thus, // `peers` here actually is a singletonList that contains one peer's TEndPoint. But here we // retain the implementation of list to cope with possible future expansion @@ -92,7 +92,7 @@ public PipeConsensusSyncConnector( this.thisDataNodeId = thisDataNodeId; this.syncRetryClientManager = PipeConsensusClientMgrContainer.getInstance().getSyncClientManager(); - this.pipeConsensusConnectorMetric = pipeConsensusConnectorMetric; + this.pipeConsensusConnectorMetrics = pipeConsensusConnectorMetrics; } @Override @@ -136,7 +136,7 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception long startTime = System.nanoTime(); doTransferWrapper((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent); long duration = System.nanoTime() - startTime; - pipeConsensusConnectorMetric.recordRetryWALTransferTimer(duration); + pipeConsensusConnectorMetrics.recordRetryWALTransferTimer(duration); } } catch (Exception e) { throw new PipeConnectionException( @@ -160,7 +160,7 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception } doTransfer((PipeTsFileInsertionEvent) tsFileInsertionEvent); long duration = System.nanoTime() - startTime; - pipeConsensusConnectorMetric.recordRetryTsFileTransferTimer(duration); + pipeConsensusConnectorMetrics.recordRetryTsFileTransferTimer(duration); } catch (Exception e) { throw new PipeConnectionException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java index 981fe74f7a61b..cdd56d72cce5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertNodeEventHandler.java @@ -23,7 +23,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; -import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetrics; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.thrift.TException; @@ -35,7 +35,7 @@ public PipeConsensusTabletInsertNodeEventHandler( PipeInsertNodeTabletInsertionEvent event, TPipeConsensusTransferReq req, PipeConsensusAsyncConnector connector, - PipeConsensusConnectorMetric metric) { + PipeConsensusConnectorMetrics metric) { super(event, req, connector, metric); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java index fbb5a2c316be6..42b8190ec5904 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java @@ -26,7 +26,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.handler.PipeTransferTabletInsertionEventHandler; -import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetric; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetrics; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -47,7 +47,7 @@ public abstract class PipeConsensusTabletInsertionEventHandler receiverFileDirWithIdSuffix = new AtomicReference<>(); - private final PipeConsensusReceiverMetric pipeConsensusReceiverMetric; + private final PipeConsensusReceiverMetrics pipeConsensusReceiverMetrics; private final FolderManager folderManager; public PipeConsensusReceiver( @@ -108,9 +108,9 @@ public PipeConsensusReceiver( ConsensusPipeName consensusPipeName) { this.pipeConsensus = pipeConsensus; this.consensusGroupId = consensusGroupId; - this.pipeConsensusReceiverMetric = new PipeConsensusReceiverMetric(this); - this.requestExecutor = new RequestExecutor(pipeConsensusReceiverMetric); - MetricService.getInstance().addMetricSet(pipeConsensusReceiverMetric); + this.pipeConsensusReceiverMetrics = new PipeConsensusReceiverMetrics(this); + this.requestExecutor = new RequestExecutor(pipeConsensusReceiverMetrics); + MetricService.getInstance().addMetricSet(pipeConsensusReceiverMetrics); // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is // data/datanode/system/pipe/consensus/receiver/__consensus{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} @@ -169,7 +169,7 @@ public TPipeConsensusTransferResp receive(final TPipeConsensusTransferReq req) { } // update receive an event's duration long durationNanos = System.nanoTime() - startNanos; - pipeConsensusReceiverMetric.recordReceiveEventTimer(durationNanos); + pipeConsensusReceiverMetrics.recordReceiveEventTimer(durationNanos); return resp; } // Unknown request type, which means the request can not be handled by this receiver, @@ -306,7 +306,7 @@ private TPipeConsensusTransferResp handleTransferFilePiece( PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); long startPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + pipeConsensusReceiverMetrics.recordBorrowTsFileWriterTimer( startPreCheckNanos - startBorrowTsFileWriterNanos); try { @@ -338,10 +338,10 @@ private TPipeConsensusTransferResp handleTransferFilePiece( } long endPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordTsFilePiecePreCheckTime( + pipeConsensusReceiverMetrics.recordTsFilePiecePreCheckTime( endPreCheckNanos - startPreCheckNanos); writingFileWriter.write(req.getFilePiece()); - pipeConsensusReceiverMetric.recordTsFilePieceWriteTime(System.nanoTime() - endPreCheckNanos); + pipeConsensusReceiverMetrics.recordTsFilePieceWriteTime(System.nanoTime() - endPreCheckNanos); return PipeConsensusTransferFilePieceResp.toTPipeConsensusTransferResp( RpcUtils.SUCCESS_STATUS, writingFileWriter.length()); } catch (Exception e) { @@ -370,7 +370,7 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); long startPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + pipeConsensusReceiverMetrics.recordBorrowTsFileWriterTimer( startPreCheckNanos - startBorrowTsFileWriterNanos); File writingFile = tsFileWriter.getWritingFile(); RandomAccessFile writingFileWriter = tsFileWriter.getWritingFileWriter(); @@ -409,13 +409,13 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF tsFileWriter.setWritingFile(null); long endPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordTsFileSealPreCheckTimer( + pipeConsensusReceiverMetrics.recordTsFileSealPreCheckTimer( endPreCheckNanos - startPreCheckNanos); final TSStatus status = loadFileToDataRegion( fileAbsolutePath, ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex()))); - pipeConsensusReceiverMetric.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); + pipeConsensusReceiverMetrics.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); @@ -471,7 +471,7 @@ private TPipeConsensusTransferResp handleTransferFileSealWithMods( PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); long startPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordBorrowTsFileWriterTimer( + pipeConsensusReceiverMetrics.recordBorrowTsFileWriterTimer( startPreCheckNanos - startBorrowTsFileWriterNanos); File writingFile = tsFileWriter.getWritingFile(); RandomAccessFile writingFileWriter = tsFileWriter.getWritingFileWriter(); @@ -527,13 +527,13 @@ private TPipeConsensusTransferResp handleTransferFileSealWithMods( files.stream().map(File::getAbsolutePath).collect(Collectors.toList()); long endPreCheckNanos = System.nanoTime(); - pipeConsensusReceiverMetric.recordTsFileSealPreCheckTimer( + pipeConsensusReceiverMetrics.recordTsFileSealPreCheckTimer( endPreCheckNanos - startPreCheckNanos); final TSStatus status = loadFileToDataRegion( fileAbsolutePaths.get(1), ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex()))); - pipeConsensusReceiverMetric.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); + pipeConsensusReceiverMetrics.recordTsFileSealLoadTimer(System.nanoTime() - endPreCheckNanos); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); @@ -939,7 +939,7 @@ public synchronized void handleExit() { } // remove metric - MetricService.getInstance().removeMetricSet(pipeConsensusReceiverMetric); + MetricService.getInstance().removeMetricSet(pipeConsensusReceiverMetrics); LOGGER.info( "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Receiver exited.", @@ -1121,13 +1121,13 @@ private class RequestExecutor { private final TreeSet reqExecutionOrderBuffer; private final Lock lock; private final Condition condition; - private final PipeConsensusReceiverMetric metric; + private final PipeConsensusReceiverMetrics metric; private long onSyncedCommitIndex = -1; private int connectorRebootTimes = 0; private int walEventCount = 0; private int tsFileEventCount = 0; - public RequestExecutor(PipeConsensusReceiverMetric metric) { + public RequestExecutor(PipeConsensusReceiverMetrics metric) { this.reqExecutionOrderBuffer = new TreeSet<>( Comparator.comparingInt(RequestMeta::getRebootTimes) From 870cc5dff364487f12792156a2c504bace24ebdb Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 11:11:45 +0800 Subject: [PATCH 09/22] fix warning --- .../consensus/pipe/PipeConsensusServerImpl.java | 12 ++++++++---- .../pipe/metric/PipeConsensusSyncLagManager.java | 10 +++++----- .../pipeconsensus/PipeConsensusAsyncConnector.java | 2 +- .../PipeConsensusTabletInsertionEventHandler.java | 2 +- .../PipeConsensusTsFileInsertionEventHandler.java | 2 +- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index c7adf133d385d..87b56c1b8eea2 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -193,7 +193,11 @@ private List createConsensusPipes(List peers) { } return true; } catch (Exception e) { - LOGGER.warn("{} cannot create consensus pipe between {} and {}", thisNode, peer, e); + LOGGER.warn( + "{}: cannot create consensus pipe between {} and {}", + e.getMessage(), + thisNode, + peer); return false; } }) @@ -212,11 +216,11 @@ private List updateConsensusPipesStatus(List peers, PipeStatus statu return true; } catch (Exception e) { LOGGER.warn( - "{} cannot update consensus pipe between {} and {} to status {}", + "{}: cannot update consensus pipe between {} and {} to status {}", + e.getMessage(), thisNode, peer, - status, - e); + status); return false; } }) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index ec5a86d49a53d..a5be2524ef994 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -45,11 +45,11 @@ private void updateReplicateProgress() { } // else we find the minimum progress in all consensus pipe task. consensusPipeConnectorList.forEach( - consensusPipeConnector -> { - minReplicateProgress = - Math.min( - minReplicateProgress, consensusPipeConnector.getConsensusPipeReplicateProgress()); - }); + consensusPipeConnector -> + minReplicateProgress = + Math.min( + minReplicateProgress, + consensusPipeConnector.getConsensusPipeReplicateProgress())); } private void updateUserWriteProgress() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 445c03b6e64ae..611eaea2ad53a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -478,7 +478,7 @@ public void addFailureEventToRetryQueue(final Event event) { return; } - retryEventQueue.offer(event); + boolean ignore = retryEventQueue.offer(event); if (LOGGER.isDebugEnabled()) { LOGGER.debug( "PipeConsensus-ConsensusGroup-{}: Event {} transfer failed, will be added to retry queue.", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java index 42b8190ec5904..ae6a1e5334d20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java @@ -49,7 +49,7 @@ public abstract class PipeConsensusTabletInsertionEventHandler Date: Thu, 13 Jun 2024 11:35:30 +0800 Subject: [PATCH 10/22] fix bug --- .../receiver/protocol/pipeconsensus/PipeConsensusReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index fab27497006d9..6a25203fe9ba1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -1122,7 +1122,7 @@ private class RequestExecutor { private final Lock lock; private final Condition condition; private final PipeConsensusReceiverMetrics metric; - private long onSyncedCommitIndex = -1; + private long onSyncedCommitIndex = 0; private int connectorRebootTimes = 0; private int walEventCount = 0; private int tsFileEventCount = 0; From 3a3d5de095f805c0790124dd4c7d180a011f5c56 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 11:45:08 +0800 Subject: [PATCH 11/22] fix syncLag bug --- .../consensus/pipe/metric/PipeConsensusSyncLagManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index a5be2524ef994..672bb47e838e0 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -38,9 +38,9 @@ public class PipeConsensusSyncLagManager { List consensusPipeConnectorList = new CopyOnWriteArrayList<>(); private void updateReplicateProgress() { + minReplicateProgress = Long.MAX_VALUE; // if there isn't a consensus pipe task, replicate progress is Long.MAX_VALUE. if (consensusPipeConnectorList.isEmpty()) { - minReplicateProgress = Long.MAX_VALUE; return; } // else we find the minimum progress in all consensus pipe task. From ec594257548c32e65e2b1a348260def3807223ba Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 14:02:03 +0800 Subject: [PATCH 12/22] fix metric name --- .../pipe/PipeConsensusServerImpl.java | 2 +- .../PipeConsensusConnectorMetrics.java | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index 87b56c1b8eea2..7f085b66e55df 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -590,6 +590,6 @@ public String getConsensusGroupId() { } public long getReplicateMode() { - return (replicateMode == ReplicateMode.BATCH) ? 0 : 1; + return (replicateMode == ReplicateMode.BATCH) ? 2 : 1; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetrics.java index d9b795a34accb..7950fd76744b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusConnectorMetrics.java @@ -134,7 +134,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorEnqueueTimer", + "connectorEnqueue", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); connectorTsFilePieceTransferTimer = @@ -144,7 +144,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorTsFilePieceTransferTimer", + "connectorTsFilePieceTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); connectorTsFileTransferTimer = @@ -154,7 +154,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorTsFileTransferTimer", + "connectorTsFileTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); connectorWALTransferTimer = @@ -164,7 +164,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorWALTransferTimer", + "connectorWALTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); retryWALTransferTimer = @@ -174,7 +174,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "retryWALTransferTimer", + "retryWALTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); retryTsFileTransferTimer = @@ -184,7 +184,7 @@ private void bindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "retryTsFileTransferTimer", + "retryTsFileTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); } @@ -238,7 +238,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorTsFileTransferTimer", + "connectorTsFileTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); metricService.remove( @@ -247,7 +247,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorTsFilePieceTransferTimer", + "connectorTsFilePieceTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); metricService.remove( @@ -256,7 +256,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorTsFileTransferTimer", + "connectorTsFileTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); metricService.remove( @@ -265,7 +265,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "connectorWALTransferTimer", + "connectorWALTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); metricService.remove( @@ -274,7 +274,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "retryWALTransferTimer", + "retryWALTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); metricService.remove( @@ -283,7 +283,7 @@ private void unbindTimer(AbstractMetricService metricService) { Tag.NAME.toString(), CONNECTOR, Tag.TYPE.toString(), - "retryTsFileTransferTimer", + "retryTsFileTransfer", Tag.REGION.toString(), pipeConsensusAsyncConnector.getConsensusGroupIdStr()); } From b3f049b881890c1168a0087a15e99f908abb22e2 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 14:25:04 +0800 Subject: [PATCH 13/22] fix review --- .../pipe/metric/PipeConsensusServerMetrics.java | 8 ++++---- .../pipe/consensus/PipeConsensusReceiverMetrics.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java index 672c8fdeb33e1..862234fd333b1 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -53,16 +53,16 @@ public void recordWriteStateMachineTime(long costTimeInNanos) { @Override public void bindTo(AbstractMetricService metricService) { - bindTimer(metricService); bindAutoGauge(metricService); bindGauge(metricService); + bindStageTimer(metricService); } @Override public void unbindFrom(AbstractMetricService metricService) { - unbindTimer(metricService); unbindAutoGauge(metricService); unbindGauge(metricService); + unbindStageTimer(metricService); } public void bindGauge(AbstractMetricService metricService) { @@ -113,7 +113,7 @@ public void unbindAutoGauge(AbstractMetricService metricService) { "syncLag"); } - public void bindTimer(AbstractMetricService metricService) { + public void bindStageTimer(AbstractMetricService metricService) { getStateMachineLockTimer = metricService.getOrCreateTimer( Metric.STAGE.toString(), @@ -136,7 +136,7 @@ public void bindTimer(AbstractMetricService metricService) { impl.getConsensusGroupId()); } - public void unbindTimer(AbstractMetricService metricService) { + public void unbindStageTimer(AbstractMetricService metricService) { getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java index 810ffeabd80cb..d3bf0325cdbc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java @@ -149,7 +149,7 @@ public void bindStageTimer(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "tsPieceWrite"); + "tsFilePieceWrite"); tsFilePiecePreCheckTimer = metricService.getOrCreateTimer( Metric.STAGE.toString(), @@ -291,7 +291,7 @@ public void unbindStageTimer(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "tsPieceWrite"); + "tsFilePieceWrite"); metricService.remove( MetricType.TIMER, Metric.STAGE.toString(), @@ -349,9 +349,9 @@ public void unbindStageTimer(AbstractMetricService metricService) { } public void unbindReceiveTimer(AbstractMetricService metricService) { - receiveEventTimer = DoNothingMetricManager.DO_NOTHING_TIMER; receiveWALTimer = DoNothingMetricManager.DO_NOTHING_TIMER; receiveTsFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + receiveEventTimer = DoNothingMetricManager.DO_NOTHING_TIMER; metricService.remove( MetricType.TIMER, @@ -361,7 +361,7 @@ public void unbindReceiveTimer(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "receiveEvent"); + "receiveWALEvent"); metricService.remove( MetricType.TIMER, Metric.PIPE_RECEIVE_EVENT.toString(), @@ -370,7 +370,7 @@ public void unbindReceiveTimer(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "receiveWALEvent"); + "receiveTsFileEvent"); metricService.remove( MetricType.TIMER, Metric.PIPE_RECEIVE_EVENT.toString(), @@ -379,6 +379,6 @@ public void unbindReceiveTimer(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "receiveTsFileEvent"); + "receiveEvent"); } } From 7e0e9f443314bd4999719eeee8511ea90c0fb3af Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 15:39:12 +0800 Subject: [PATCH 14/22] fix review --- .../PipeConsensusAsyncConnector.java | 9 ++++---- .../PipeConsensusTabletBatchEventHandler.java | 7 +++++- .../PipeConsensusReceiverMetrics.java | 4 ++-- .../pipeconsensus/PipeConsensusReceiver.java | 23 ++++++++++++------- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 611eaea2ad53a..1a33a4bc3e447 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -239,7 +239,8 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception if (isTabletBatchModeEnabled) { if (tabletBatchBuilder.onEvent(tabletInsertionEvent)) { final PipeConsensusTabletBatchEventHandler pipeConsensusTabletBatchEventHandler = - new PipeConsensusTabletBatchEventHandler(tabletBatchBuilder, this); + new PipeConsensusTabletBatchEventHandler( + tabletBatchBuilder, this, pipeConsensusConnectorMetrics); transfer(pipeConsensusTabletBatchEventHandler); @@ -404,8 +405,9 @@ private void transferBatchedEventsIfNecessary() throws IOException { return; } - transfer(new PipeConsensusTabletBatchEventHandler(tabletBatchBuilder, this)); - + transfer( + new PipeConsensusTabletBatchEventHandler( + tabletBatchBuilder, this, pipeConsensusConnectorMetrics)); tabletBatchBuilder.onSuccess(); } @@ -501,7 +503,6 @@ public void addFailureEventToRetryQueue(final Event event) { public void addFailureEventsToRetryQueue(final Iterable events) { for (final Event event : events) { addFailureEventToRetryQueue(event); - pipeConsensusConnectorMetrics.recordRetryCounter(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java index 25816e68fa6ac..de9c7bec37d31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.payload.builder.PipeConsensusAsyncBatchReqBuilder; +import org.apache.iotdb.db.pipe.consensus.PipeConsensusConnectorMetrics; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -48,16 +49,19 @@ public class PipeConsensusTabletBatchEventHandler private final List events; private final TPipeConsensusBatchTransferReq req; private final PipeConsensusAsyncConnector connector; + private final PipeConsensusConnectorMetrics pipeConsensusConnectorMetrics; public PipeConsensusTabletBatchEventHandler( final PipeConsensusAsyncBatchReqBuilder batchBuilder, - final PipeConsensusAsyncConnector connector) + final PipeConsensusAsyncConnector connector, + final PipeConsensusConnectorMetrics pipeConsensusConnectorMetrics) throws IOException { // Deep copy to keep Ids' and events' reference requestCommitIds = batchBuilder.deepCopyRequestCommitIds(); events = batchBuilder.deepCopyEvents(); req = batchBuilder.toTPipeConsensusBatchTransferReq(); + this.pipeConsensusConnectorMetrics = pipeConsensusConnectorMetrics; this.connector = connector; } @@ -86,6 +90,7 @@ public void onComplete(final TPipeConsensusBatchTransferResp response) { .filter(tsStatus -> tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) .forEach( tsStatus -> { + pipeConsensusConnectorMetrics.recordRetryCounter(); connector .statusHandler() .handle(tsStatus, tsStatus.getMessage(), events.toString()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java index d3bf0325cdbc6..c94d14885e19e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java @@ -125,7 +125,7 @@ public void bindAutoGauge(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "walEventCount"); + "WALEventCount"); metricService.createAutoGauge( Metric.PIPE_RECEIVE_EVENT.toString(), MetricLevel.IMPORTANT, @@ -263,7 +263,7 @@ public void unbindAutoGauge(AbstractMetricService metricService) { Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), Tag.TYPE.toString(), - "walEventCount"); + "WALEventCount"); metricService.remove( MetricType.AUTO_GAUGE, Metric.PIPE_RECEIVE_EVENT.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 6a25203fe9ba1..810be40233f08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -156,7 +156,7 @@ public TPipeConsensusTransferResp receive(final TPipeConsensusTransferReq req) { break; case TRANSFER_TS_FILE_SEAL: case TRANSFER_TS_FILE_SEAL_WITH_MOD: - // TODO: check memory when logging wal(in further version) + // TODO: check memory when logging WAL(in further version) resp = requestExecutor.onRequest(req, false, true); break; case TRANSFER_TABLET_BINARY: @@ -1124,7 +1124,7 @@ private class RequestExecutor { private final PipeConsensusReceiverMetrics metric; private long onSyncedCommitIndex = 0; private int connectorRebootTimes = 0; - private int walEventCount = 0; + private int WALEventCount = 0; private int tsFileEventCount = 0; public RequestExecutor(PipeConsensusReceiverMetrics metric) { @@ -1149,7 +1149,7 @@ private void onSuccess(long nextSyncedCommitIndex, boolean isTransferTsFileSeal) tsFileEventCount--; metric.recordReceiveTsFileTimer(System.nanoTime() - curMeta.getStartApplyNanos()); } else { - walEventCount--; + WALEventCount--; metric.recordReceiveWALTimer(System.nanoTime() - curMeta.getStartApplyNanos()); } } @@ -1195,13 +1195,15 @@ private TPipeConsensusTransferResp onRequest( if (tCommitId.getRebootTimes() > connectorRebootTimes) { resetWithNewestRebootTime(tCommitId.getRebootTimes()); } - reqExecutionOrderBuffer.add(requestMeta); // update metric - if (isTransferTsFilePiece) { + if (isTransferTsFilePiece && !reqExecutionOrderBuffer.contains(requestMeta)) { + // only update tsFileEventCount when tsFileEvent is first enqueue. tsFileEventCount++; - } else if (!isTransferTsFileSeal) { - walEventCount++; } + if (!isTransferTsFileSeal && !isTransferTsFilePiece) { + WALEventCount++; + } + reqExecutionOrderBuffer.add(requestMeta); // TsFilePieceTransferEvent will not enter further procedure, it just holds a place in // buffer. Only after the corresponding sealing event is processed, this event can be @@ -1361,6 +1363,11 @@ public boolean equals(Object o) { RequestMeta that = (RequestMeta) o; return commitId.equals(that.commitId); } + + @Override + public int hashCode() { + return Objects.hash(commitId); + } } //////////////////////////// APIs provided for metric framework //////////////////////////// @@ -1370,7 +1377,7 @@ public int getReceiveBufferSize() { } public int getWALEventCount() { - return this.requestExecutor.walEventCount; + return this.requestExecutor.WALEventCount; } public int getTsFileEventCount() { From bb924e429c30396ee4a05ecc34950fc95c9b69fc Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Thu, 13 Jun 2024 16:32:18 +0800 Subject: [PATCH 15/22] fix review --- .../pipe/metric/PipeConsensusSyncLagManager.java | 9 +++++++++ .../pipeconsensus/PipeConsensusAsyncConnector.java | 2 ++ 2 files changed, 11 insertions(+) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index 672bb47e838e0..566bf4432d4e5 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -68,6 +68,10 @@ public void addConsensusPipeConnector(ConsensusPipeConnector consensusPipeConnec consensusPipeConnectorList.add(consensusPipeConnector); } + public void removeConsensusPipeConnector(ConsensusPipeConnector connector) { + consensusPipeConnectorList.remove(connector); + } + /** * SyncLag represents the difference between the current replica users' write progress and the * minimum synchronization progress of all other replicas. The semantics is how much data the @@ -80,6 +84,11 @@ public long calculateSyncLag() { if (minReplicateProgress == Long.MAX_VALUE) { return userWriteProgress; } else { + // since we first update userWriteProgress then update replicateProgress, there may have some + // cases that userWriteProgress is less than replicateProgress. In these cases, we return 0. + if (userWriteProgress < minReplicateProgress) { + return 0; + } return userWriteProgress - minReplicateProgress; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 1a33a4bc3e447..e3673c0330d93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -556,6 +556,8 @@ public synchronized void close() { tabletBatchBuilder.close(); } + PipeConsensusSyncLagManager.getInstance(getConsensusGroupIdStr()) + .removeConsensusPipeConnector(this); MetricService.getInstance().removeMetricSet(this.pipeConsensusConnectorMetrics); } From e6310ddfd0105d28905548fd7ad153a86fc0f3cd Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 00:22:50 +0800 Subject: [PATCH 16/22] fix review --- .../pipe/PipeConsensusServerImpl.java | 4 ++-- .../metric/PipeConsensusServerMetrics.java | 3 +++ .../metric/PipeConsensusSyncLagManager.java | 19 +++++++++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index 7f085b66e55df..cf05c13638a16 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -177,6 +177,7 @@ public synchronized void clear() throws IOException { LOGGER.warn("{} cannot drop all consensus pipes", thisNode); } + MetricService.getInstance().removeMetricSet(this.pipeConsensusServerMetrics); peerManager.clear(); stateMachine.stop(); isStarted.set(false); @@ -302,9 +303,8 @@ public TSStatus write(IConsensusRequest request) { long startWriteTime = System.nanoTime(); TSStatus result = stateMachine.write(request); - PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(System.nanoTime() - startWriteTime); - long writeToStateMachineEndTime = System.nanoTime(); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(writeToStateMachineEndTime - startWriteTime); // statistic the time of writing request into stateMachine pipeConsensusServerMetrics.recordWriteStateMachineTime( writeToStateMachineEndTime - writeToStateMachineStartTime); diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java index 862234fd333b1..2bc1533db981c 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -63,6 +63,9 @@ public void unbindFrom(AbstractMetricService metricService) { unbindAutoGauge(metricService); unbindGauge(metricService); unbindStageTimer(metricService); + + // release corresponding resource + PipeConsensusSyncLagManager.release(impl.getConsensusGroupId()); } public void bindGauge(AbstractMetricService metricService) { diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java index 566bf4432d4e5..6ff754e02ef7a 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusSyncLagManager.java @@ -60,8 +60,14 @@ private void updateUserWriteProgress() { } // since the user write progress of different consensus pipes on the same DataRegion is the // same, we only need to take out one Connector to calculate - ConsensusPipeConnector connector = consensusPipeConnectorList.get(0); - userWriteProgress = connector.getConsensusPipeCommitProgress(); + try { + ConsensusPipeConnector connector = consensusPipeConnectorList.get(0); + userWriteProgress = connector.getConsensusPipeCommitProgress(); + } catch (Exception e) { + // if removing the last connector happens after empty check, we may encounter + // OutOfBoundsException, in this case, we set userWriteProgress to 0. + userWriteProgress = 0; + } } public void addConsensusPipeConnector(ConsensusPipeConnector consensusPipeConnector) { @@ -86,10 +92,7 @@ public long calculateSyncLag() { } else { // since we first update userWriteProgress then update replicateProgress, there may have some // cases that userWriteProgress is less than replicateProgress. In these cases, we return 0. - if (userWriteProgress < minReplicateProgress) { - return 0; - } - return userWriteProgress - minReplicateProgress; + return Math.max(userWriteProgress - minReplicateProgress, 0); } } @@ -116,6 +119,10 @@ public static PipeConsensusSyncLagManager getInstance(String groupId) { groupId, key -> new PipeConsensusSyncLagManager()); } + public static void release(String groupId) { + PipeConsensusSyncLagManagerHolder.CONSENSU_GROUP_ID_2_INSTANCE_MAP.remove(groupId); + } + // Only when consensus protocol is PipeConsensus, this method will be called once when construct // consensus class. public static void build() { From 321f3e0ad3f770ae42f2905248d5dabec1abd89b Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 00:34:07 +0800 Subject: [PATCH 17/22] fix timer --- .../pipe/PipeConsensusServerImpl.java | 4 +- .../metric/PipeConsensusServerMetrics.java | 37 ++++++++++++++++--- .../PipeConsensusReceiverMetrics.java | 4 +- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index cf05c13638a16..685a11ba6d5d6 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -306,7 +306,7 @@ public TSStatus write(IConsensusRequest request) { long writeToStateMachineEndTime = System.nanoTime(); PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(writeToStateMachineEndTime - startWriteTime); // statistic the time of writing request into stateMachine - pipeConsensusServerMetrics.recordWriteStateMachineTime( + pipeConsensusServerMetrics.recordUserWriteStateMachineTime( writeToStateMachineEndTime - writeToStateMachineStartTime); return result; } finally { @@ -330,7 +330,7 @@ public TSStatus writeOnFollowerReplica(IConsensusRequest request) { PERFORMANCE_OVERVIEW_METRICS.recordEngineCost( writeToStateMachineEndTime - writeToStateMachineStartTime); // statistic the time of writing request into stateMachine - pipeConsensusServerMetrics.recordWriteStateMachineTime( + pipeConsensusServerMetrics.recordReplicaWriteStateMachineTime( writeToStateMachineEndTime - writeToStateMachineStartTime); return result; } finally { diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java index 2bc1533db981c..39bcb1e526ff8 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/metric/PipeConsensusServerMetrics.java @@ -34,7 +34,8 @@ public class PipeConsensusServerMetrics implements IMetricSet { private final PipeConsensusSyncLagManager syncLagManager; private Timer getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; - private Timer writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer userWriteStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer replicaWriteStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; public PipeConsensusServerMetrics(PipeConsensusServerImpl impl) { this.impl = impl; @@ -47,8 +48,12 @@ public void recordGetStateMachineLockTime(long costTimeInNanos) { getStateMachineLockTimer.updateNanos(costTimeInNanos); } - public void recordWriteStateMachineTime(long costTimeInNanos) { - writeStateMachineTimer.updateNanos(costTimeInNanos); + public void recordUserWriteStateMachineTime(long costTimeInNanos) { + userWriteStateMachineTimer.updateNanos(costTimeInNanos); + } + + public void recordReplicaWriteStateMachineTime(long costTimeInNanos) { + replicaWriteStateMachineTimer.updateNanos(costTimeInNanos); } @Override @@ -127,21 +132,32 @@ public void bindStageTimer(AbstractMetricService metricService) { "getStateMachineLock", Tag.REGION.toString(), impl.getConsensusGroupId()); - writeStateMachineTimer = + userWriteStateMachineTimer = metricService.getOrCreateTimer( Metric.STAGE.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), Metric.PIPE_CONSENSUS.toString(), Tag.TYPE.toString(), - "writeStateMachine", + "userWriteStateMachine", + Tag.REGION.toString(), + impl.getConsensusGroupId()); + replicaWriteStateMachineTimer = + metricService.getOrCreateTimer( + Metric.PIPE_RECEIVE_EVENT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "replicaWriteStateMachine", Tag.REGION.toString(), impl.getConsensusGroupId()); } public void unbindStageTimer(AbstractMetricService metricService) { getStateMachineLockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; - writeStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + userWriteStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + replicaWriteStateMachineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; metricService.remove( MetricType.TIMER, @@ -161,5 +177,14 @@ public void unbindStageTimer(AbstractMetricService metricService) { "writeStateMachine", Tag.REGION.toString(), impl.getConsensusGroupId()); + metricService.remove( + MetricType.TIMER, + Metric.PIPE_RECEIVE_EVENT.toString(), + Tag.NAME.toString(), + Metric.PIPE_CONSENSUS.toString(), + Tag.TYPE.toString(), + "replicaWriteStateMachine", + Tag.REGION.toString(), + impl.getConsensusGroupId()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java index c94d14885e19e..58d587d8b8d20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusReceiverMetrics.java @@ -231,7 +231,7 @@ public void bindReceiveTimer(AbstractMetricService metricService) { RECEIVER, Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), - Tag.STAGE.toString(), + Tag.TYPE.toString(), "receiveWALEvent"); receiveTsFileTimer = metricService.getOrCreateTimer( @@ -241,7 +241,7 @@ public void bindReceiveTimer(AbstractMetricService metricService) { RECEIVER, Tag.REGION.toString(), pipeConsensusReceiver.getConsensusGroupIdStr(), - Tag.STAGE.toString(), + Tag.TYPE.toString(), "receiveTsFileEvent"); } From 7cd7756c6561391a50493bd79269db0378232b61 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 00:50:29 +0800 Subject: [PATCH 18/22] fix TimeEcho-771 --- .../pipeconsensus/PipeConsensusReceiver.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 810be40233f08..4d5d84d075b60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -122,13 +122,19 @@ public PipeConsensusReceiver( try { this.folderManager = new FolderManager(receiverBaseDirsName, DirectoryStrategyType.SEQUENCE_STRATEGY); - initiateTsFileBufferFolder(); } catch (Exception e) { LOGGER.error( "Fail to create pipeConsensus receiver file folders allocation strategy because all disks of folders are full.", e); throw new RuntimeException(e); } + + try { + initiateTsFileBufferFolder(); + } catch (Exception e) { + LOGGER.error("Fail to initiate file buffer folder, Error msg: {}", e.getMessage()); + throw new RuntimeException(e); + } } /** @@ -871,7 +877,7 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, final String receiverFileBaseDir = getReceiverFileBaseDir(); if (Objects.isNull(receiverFileBaseDir)) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to init pipeConsensus receiver file folder manager because all disks of folders are full.", + "PipeConsensus-ConsensusGroupId-{}: Failed to get pipeConsensus receiver file base directory, because your folderManager is null. May because the disk is full.", consensusGroupId.getId()); throw new DiskSpaceInsufficientException(receiverBaseDirsName); } @@ -879,13 +885,13 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, final File newReceiverDir = new File(receiverFileBaseDir, consensusGroupId.toString()); if (!newReceiverDir.exists() && !newReceiverDir.mkdirs()) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to create receiver file dir {}.", - newReceiverDir.getPath(), - consensusGroupId.getId()); + "PipeConsensus-ConsensusGroupId-{}: Failed to create receiver file dir {}. May because authority or dir already exists etc.", + consensusGroupId.getId(), + newReceiverDir.getPath()); throw new IOException( String.format( - "PipeConsensus-ConsensusGroupId-%s: Failed to create receiver file dir %s.", - newReceiverDir.getPath(), consensusGroupId.getId())); + "PipeConsensus-ConsensusGroupId-%s: Failed to create receiver file dir %s. May because authority or dir already exists etc.", + consensusGroupId.getId(), newReceiverDir.getPath())); } receiverFileDirWithIdSuffix.set(newReceiverDir); From f7f10446cd8e04c9fdb3ecddb8feb511caad4e8e Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 10:11:33 +0800 Subject: [PATCH 19/22] resolve conflict and optimize receiver's log --- .../pipeconsensus/PipeConsensusReceiver.java | 241 +++++++++--------- 1 file changed, 117 insertions(+), 124 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 8b8420c5327d6..12334236eb0bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -94,9 +94,9 @@ public class PipeConsensusReceiver { private final RequestExecutor requestExecutor; private final PipeConsensus pipeConsensus; private final ConsensusGroupId consensusGroupId; - // Used to buffer TsFile when transfer TsFile asynchronously. private final ConsensusPipeName consensusPipeName; private final List receiverBaseDirsName; + // Used to buffer TsFile when transfer TsFile asynchronously. private final PipeConsensusTsFileWriterPool pipeConsensusTsFileWriterPool = new PipeConsensusTsFileWriterPool(); private final AtomicReference receiverFileDirWithIdSuffix = new AtomicReference<>(); @@ -112,7 +112,7 @@ public PipeConsensusReceiver( this.pipeConsensusReceiverMetrics = new PipeConsensusReceiverMetrics(this); this.requestExecutor = new RequestExecutor(pipeConsensusReceiverMetrics); this.consensusPipeName = consensusPipeName; - MetricService.getInstance().addMetricSet(pipeConsensusReceiverMetrics); + MetricService.getInstance().addMetricSet(pipeConsensusReceiverMetrics); // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is // data/datanode/system/pipe/consensus/receiver/__consensus.{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} @@ -206,8 +206,7 @@ private TPipeConsensusTransferResp preCheckForReceiver(final TPipeConsensusTrans if (impl.isReadOnly()) { String message = String.format( - "PipeConsensus-ConsensusGroupId-%s: fail to receive because system is read-only.", - groupId); + "PipeConsensus-PipeName-%s: fail to receive because system is read-only.", groupId); if (LOGGER.isErrorEnabled()) { LOGGER.error(message); } @@ -217,7 +216,7 @@ private TPipeConsensusTransferResp preCheckForReceiver(final TPipeConsensusTrans if (!impl.isActive()) { String message = String.format( - "PipeConsensus-ConsensusGroupId-%s: fail to receive because peer is inactive and not ready.", + "PipeConsensus-PipeName-%s: fail to receive because peer is inactive and not ready.", groupId); if (LOGGER.isWarnEnabled()) { LOGGER.warn(message); @@ -266,13 +265,13 @@ private TPipeConsensusTransferResp loadEvent(final TPipeConsensusTransferReq req TSStatusCode.PIPE_CONSENSUS_TYPE_ERROR, String.format("Unknown PipeConsensusRequestType %s.", rawRequestType)); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Unknown PipeRequestType, response status = {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Unknown PipeRequestType, response status = {}.", + consensusPipeName, status); return new TPipeConsensusTransferResp(status); } catch (Exception e) { final String error = String.format("Serialization error during pipe receiving, %s", e); - LOGGER.warn("PipeConsensus-ConsensusGroupId-{}: {}", consensusGroupId, error, e); + LOGGER.warn("PipeConsensus-PipeName-{}: {}", consensusPipeName, error, e); return new TPipeConsensusTransferResp(RpcUtils.getStatus(TSStatusCode.PIPE_ERROR, error)); } } @@ -280,8 +279,7 @@ private TPipeConsensusTransferResp loadEvent(final TPipeConsensusTransferReq req private TPipeConsensusTransferResp handleTransferTabletInsertNode( final PipeConsensusTabletInsertNodeReq req) throws ConsensusGroupNotExistException { LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: starting to receive tablet insertNode", - consensusGroupId); + "PipeConsensus-PipeName-{}: starting to receive tablet insertNode", consensusPipeName); PipeConsensusServerImpl impl = Optional.ofNullable(pipeConsensus.getImpl(consensusGroupId)) .orElseThrow(() -> new ConsensusGroupNotExistException(consensusGroupId)); @@ -293,8 +291,7 @@ private TPipeConsensusTransferResp handleTransferTabletInsertNode( private TPipeConsensusTransferResp handleTransferTabletBinary( final PipeConsensusTabletBinaryReq req) throws ConsensusGroupNotExistException { - LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: starting to receive tablet binary", consensusGroupId); + LOGGER.info("PipeConsensus-PipeName-{}: starting to receive tablet binary", consensusPipeName); PipeConsensusServerImpl impl = Optional.ofNullable(pipeConsensus.getImpl(consensusGroupId)) .orElseThrow(() -> new ConsensusGroupNotExistException(consensusGroupId)); @@ -306,8 +303,7 @@ private TPipeConsensusTransferResp handleTransferTabletBinary( private TPipeConsensusTransferResp handleTransferFilePiece( final PipeConsensusTransferFilePieceReq req, final boolean isSingleFile) { - LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile pieces", consensusGroupId); + LOGGER.info("PipeConsensus-PipeName-{}: starting to receive tsFile pieces", consensusPipeName); long startBorrowTsFileWriterNanos = System.nanoTime(); PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); @@ -336,8 +332,8 @@ private TPipeConsensusTransferResp handleTransferFilePiece( "Request sender to reset file reader's offset from %s to %s.", req.getStartWritingOffset(), writingFileWriter.length())); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: File offset reset requested by receiver, response status = {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: File offset reset requested by receiver, response status = {}.", + consensusPipeName, status); return PipeConsensusTransferFilePieceResp.toTPipeConsensusTransferResp( status, writingFileWriter.length()); @@ -352,8 +348,8 @@ private TPipeConsensusTransferResp handleTransferFilePiece( RpcUtils.SUCCESS_STATUS, writingFileWriter.length()); } catch (Exception e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to write file piece from req {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to write file piece from req {}.", + consensusPipeName, req, e); final TSStatus status = @@ -370,8 +366,7 @@ private TPipeConsensusTransferResp handleTransferFilePiece( } private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsFileSealReq req) { - LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile seal", consensusGroupId); + LOGGER.info("PipeConsensus-PipeName-{}: starting to receive tsFile seal", consensusPipeName); long startBorrowTsFileWriterNanos = System.nanoTime(); PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); @@ -426,21 +421,21 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Seal file {} successfully.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Seal file {} successfully.", + consensusPipeName, fileAbsolutePath); } else { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {}, because {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {}, because {}.", + consensusPipeName, fileAbsolutePath, status.getMessage()); } return new TPipeConsensusTransferResp(status); } catch (IOException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {} from req {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {} from req {}.", + consensusPipeName, writingFile, req, e); @@ -450,8 +445,8 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF String.format("Failed to seal file %s because %s", writingFile, e.getMessage()))); } catch (LoadFileException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to load file {} from req {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to load file {} from req {}.", + consensusPipeName, writingFile, req, e); @@ -471,8 +466,7 @@ private TPipeConsensusTransferResp handleTransferFileSeal(final PipeConsensusTsF private TPipeConsensusTransferResp handleTransferFileSealWithMods( final PipeConsensusTsFileSealWithModReq req) { LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: starting to receive tsFile seal with mods", - consensusGroupId); + "PipeConsensus-PipeName-{}: starting to receive tsFile seal with mods", consensusPipeName); long startBorrowTsFileWriterNanos = System.nanoTime(); PipeConsensusTsFileWriter tsFileWriter = pipeConsensusTsFileWriterPool.borrowCorrespondingWriter(req.getCommitId()); @@ -544,21 +538,21 @@ private TPipeConsensusTransferResp handleTransferFileSealWithMods( // if transfer success, disk buffer will be released. tsFileWriter.returnSelf(); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Seal file with mods {} successfully.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Seal file with mods {} successfully.", + consensusPipeName, fileAbsolutePaths); } else { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {}, status is {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {}, status is {}.", + consensusPipeName, fileAbsolutePaths, status); } return new TPipeConsensusTransferResp(status); } catch (Exception e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {} from req {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {} from req {}.", + consensusPipeName, files, req, e); @@ -591,8 +585,8 @@ private TPipeConsensusTransferResp checkNonFinalFileSeal( TSStatusCode.PIPE_CONSENSUS_TRANSFER_FILE_ERROR, String.format("Failed to seal file %s, the file does not exist.", fileName)); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {}, because the file does not exist.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {}, because the file does not exist.", + consensusPipeName, fileName); return new TPipeConsensusTransferResp(status); } @@ -606,9 +600,9 @@ private TPipeConsensusTransferResp checkNonFinalFileSeal( + "The original file has length %s, but receiver file has length %s.", fileName, fileLength, writingFileWriter.length())); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {} when check non final seal, because the length of file is not correct. " + "PipeConsensus-PipeName-{}: Failed to seal file {} when check non final seal, because the length of file is not correct. " + "The original file has length {}, but receiver file has length {}.", - consensusGroupId, + consensusPipeName, fileName, fileLength, writingFileWriter.length()); @@ -649,9 +643,9 @@ private boolean isWritingFileNonAvailable(PipeConsensusTsFileWriter diskBuffer) writingFile != null && writingFile.exists() && writingFileWriter != null; if (!isWritingFileAvailable) { LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Writing file {} is not available. " + "PipeConsensus-PipeName-{}: Writing file {} is not available. " + "Writing file is null: {}, writing file exists: {}, writing file writer is null: {}.", - consensusGroupId, + consensusPipeName, writingFile, writingFile == null, writingFile != null && writingFile.exists(), @@ -673,8 +667,8 @@ private TPipeConsensusTransferResp checkFinalFileSeal( String.format( "Failed to seal file %s, because writing file is %s.", fileName, writingFile)); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {}, because writing file is {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to seal file {}, because writing file is {}.", + consensusPipeName, fileName, writingFile); return new TPipeConsensusTransferResp(status); @@ -689,9 +683,9 @@ private TPipeConsensusTransferResp checkFinalFileSeal( + "The original file has length %s, but receiver file has length %s.", fileName, fileLength, writingFileWriter.length())); LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to seal file {} when check final seal file, because the length of file is not correct. " + "PipeConsensus-PipeName-{}: Failed to seal file {} when check final seal file, because the length of file is not correct. " + "The original file has length {}, but receiver file has length {}.", - consensusGroupId, + consensusPipeName, fileName, fileLength, writingFileWriter.length()); @@ -715,8 +709,8 @@ private boolean isWritingFileOffsetNonCorrect( final boolean offsetCorrect = writingFileWriter.length() == offset; if (!offsetCorrect) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Writing file {}'s offset is {}, but request sender's offset is {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Writing file {}'s offset is {}, but request sender's offset is {}.", + consensusPipeName, writingFile.getPath(), writingFileWriter.length(), offset); @@ -729,13 +723,13 @@ private void closeCurrentWritingFileWriter(PipeConsensusTsFileWriter diskBuffer) try { diskBuffer.getWritingFileWriter().close(); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Current writing file writer {} was closed.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Current writing file writer {} was closed.", + consensusPipeName, diskBuffer.getWritingFile() == null ? "null" : diskBuffer.getWritingFile().getPath()); } catch (IOException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to close current writing file writer {}, because {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to close current writing file writer {}, because {}.", + consensusPipeName, diskBuffer.getWritingFile() == null ? "null" : diskBuffer.getWritingFile().getPath(), e.getMessage(), e); @@ -744,8 +738,8 @@ private void closeCurrentWritingFileWriter(PipeConsensusTsFileWriter diskBuffer) } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Current writing file writer is null. No need to close.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Current writing file writer is null. No need to close.", + consensusPipeName.toString()); } } } @@ -755,13 +749,13 @@ private void deleteFile(File file) { try { FileUtils.delete(file); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Original writing file {} was deleted.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Original writing file {} was deleted.", + consensusPipeName, file.getPath()); } catch (IOException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to delete original writing file {}, because {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to delete original writing file {}, because {}.", + consensusPipeName, file.getPath(), e.getMessage(), e); @@ -769,8 +763,8 @@ private void deleteFile(File file) { } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Original file {} is not existed. No need to delete.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Original file {} is not existed. No need to delete.", + consensusPipeName, file.getPath()); } } @@ -782,8 +776,8 @@ private void deleteCurrentWritingFile(PipeConsensusTsFileWriter diskBuffer) { } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Current writing file is null. No need to delete.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Current writing file is null. No need to delete.", + consensusPipeName.toString()); } } } @@ -796,9 +790,9 @@ private void updateWritingFileIfNeeded( } LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Writing file {} is not existed or name is not correct, try to create it. " + "PipeConsensus-PipeName-{}: Writing file {} is not existed or name is not correct, try to create it. " + "Current writing file is {}.", - consensusGroupId, + consensusPipeName, fileName, diskBuffer.getWritingFile() == null ? "null" : diskBuffer.getWritingFile().getPath()); @@ -815,13 +809,13 @@ private void updateWritingFileIfNeeded( if (!receiverFileDirWithIdSuffix.get().exists()) { if (receiverFileDirWithIdSuffix.get().mkdirs()) { LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Receiver file dir {} was created.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Receiver file dir {} was created.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } else { LOGGER.error( - "PipeConsensus-ConsensusGroupId-{}: Failed to create receiver file dir {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to create receiver file dir {}.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } } @@ -829,8 +823,8 @@ private void updateWritingFileIfNeeded( diskBuffer.setWritingFile(new File(receiverFileDirWithIdSuffix.get(), fileName)); diskBuffer.setWritingFileWriter(new RandomAccessFile(diskBuffer.getWritingFile(), "rw")); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Writing file {} was created. Ready to write file pieces.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Writing file {} was created. Ready to write file pieces.", + consensusPipeName, diskBuffer.getWritingFile().getPath()); } @@ -846,13 +840,13 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, try { FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Original receiver file dir {} was deleted successfully.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Original receiver file dir {} was deleted successfully.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } catch (IOException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to delete original receiver file dir {}, because {}.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Failed to delete original receiver file dir {}, because {}.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath(), e.getMessage(), e); @@ -860,16 +854,16 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Original receiver file dir {} is not existed. No need to delete.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Original receiver file dir {} is not existed. No need to delete.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } } receiverFileDirWithIdSuffix.set(null); } else { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Current receiver file dir is null. No need to delete.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Current receiver file dir is null. No need to delete.", + consensusPipeName.toString()); } // initiate receiverFileDirWithIdSuffix @@ -877,8 +871,8 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, final String receiverFileBaseDir = getReceiverFileBaseDir(); if (Objects.isNull(receiverFileBaseDir)) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to get pipeConsensus receiver file base directory, because your folderManager is null. May because the disk is full.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Failed to get pipeConsensus receiver file base directory, because your folderManager is null. May because the disk is full.", + consensusPipeName.toString()); throw new DiskSpaceInsufficientException(receiverBaseDirsName); } // Create a new receiver file dir @@ -886,19 +880,19 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, if (newReceiverDir.exists()) { FileUtils.deleteDirectory(newReceiverDir); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Origin receiver file dir {} was deleted.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Origin receiver file dir {} was deleted.", + consensusPipeName, newReceiverDir.getPath()); } if (!newReceiverDir.mkdirs()) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Failed to create receiver file dir {}. May because authority or dir already exists etc.", - consensusGroupId.getId(), + "PipeConsensus-PipeName-{}: Failed to create receiver file dir {}. May because authority or dir already exists etc.", + consensusPipeName, newReceiverDir.getPath()); throw new IOException( String.format( - "PipeConsensus-ConsensusGroupId-%s: Failed to create receiver file dir %s. May because authority or dir already exists etc.", - consensusGroupId.getId(), newReceiverDir.getPath())); + "PipeConsensus-PipeName-%s: Failed to create receiver file dir %s. May because authority or dir already exists etc.", + consensusPipeName, newReceiverDir.getPath())); } receiverFileDirWithIdSuffix.set(newReceiverDir); @@ -916,7 +910,7 @@ public PipeConsensusRequestVersion getVersion() { public synchronized void handleExit() { // Clear the diskBuffers - pipeConsensusTsFileWriterPool.handleExit(consensusGroupId); + pipeConsensusTsFileWriterPool.handleExit(consensusPipeName); // Clear the original receiver file dir if exists if (receiverFileDirWithIdSuffix.get() != null) { @@ -924,21 +918,21 @@ public synchronized void handleExit() { try { FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Original receiver file dir {} was deleted.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Receiver exit: Original receiver file dir {} was deleted.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } catch (IOException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Delete original receiver file dir {} error.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Receiver exit: Delete original receiver file dir {} error.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath(), e); } } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Original receiver file dir {} does not exist. No need to delete.", - consensusGroupId, + "PipeConsensus-PipeName-{}: Receiver exit: Original receiver file dir {} does not exist. No need to delete.", + consensusPipeName, receiverFileDirWithIdSuffix.get().getPath()); } } @@ -946,8 +940,8 @@ public synchronized void handleExit() { } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Original receiver file dir is null. No need to delete.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Receiver exit: Original receiver file dir is null. No need to delete.", + consensusPipeName.toString()); } } @@ -955,8 +949,7 @@ public synchronized void handleExit() { MetricService.getInstance().removeMetricSet(pipeConsensusReceiverMetrics); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: Receiver exit: Receiver exited.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: Receiver exit: Receiver exited.", consensusPipeName.toString()); } private static class PipeConsensusTsFileWriterPool { @@ -996,7 +989,7 @@ public PipeConsensusTsFileWriter borrowCorrespondingWriter(TCommitId commitId) { return diskBuffer.get(); } - public void handleExit(ConsensusGroupId consensusGroupId) { + public void handleExit(ConsensusPipeName consensusPipeName) { pipeConsensusTsFileWriterPool.forEach( diskBuffer -> { // Wait until diskBuffer is not used by TsFileInsertionEvent or timeout. @@ -1008,13 +1001,13 @@ public void handleExit(ConsensusGroupId consensusGroupId) { Thread.sleep(RETRY_WAIT_TIME); } catch (InterruptedException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: receiver thread get interrupted when exiting.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: receiver thread get interrupted when exiting.", + consensusPipeName.toString()); // avoid infinite loop break; } } - diskBuffer.closeSelf(consensusGroupId); + diskBuffer.closeSelf(consensusPipeName); }); } } @@ -1071,19 +1064,19 @@ public void returnSelf() { this.commitIdOfCorrespondingHolderEvent = null; } - public void closeSelf(ConsensusGroupId consensusGroupId) { + public void closeSelf(ConsensusPipeName consensusPipeName) { // close file writer if (writingFileWriter != null) { try { writingFileWriter.close(); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter-{} exit: Writing file writer was closed.", - consensusGroupId.getId(), + "PipeConsensus-PipeName-{}: TsFileWriter-{} exit: Writing file writer was closed.", + consensusPipeName.toString(), index); } catch (Exception e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter-{} exit: Close Writing file writer error.", - consensusGroupId, + "PipeConsensus-PipeName-{}: TsFileWriter-{} exit: Close Writing file writer error.", + consensusPipeName, index, e); } @@ -1091,8 +1084,8 @@ public void closeSelf(ConsensusGroupId consensusGroupId) { } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter-{} exit: Writing file writer is null. No need to close.", - consensusGroupId.getId(), + "PipeConsensus-PipeName-{}: TsFileWriter-{} exit: Writing file writer is null. No need to close.", + consensusPipeName.toString(), index); } } @@ -1102,13 +1095,13 @@ public void closeSelf(ConsensusGroupId consensusGroupId) { try { FileUtils.delete(writingFile); LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter exit: Writing file {} was deleted.", - consensusGroupId, + "PipeConsensus-PipeName-{}: TsFileWriter exit: Writing file {} was deleted.", + consensusPipeName, writingFile.getPath()); } catch (Exception e) { LOGGER.warn( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter exit: Delete writing file {} error.", - consensusGroupId, + "PipeConsensus-PipeName-{}: TsFileWriter exit: Delete writing file {} error.", + consensusPipeName, writingFile.getPath(), e); } @@ -1116,8 +1109,8 @@ public void closeSelf(ConsensusGroupId consensusGroupId) { } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "PipeConsensus-ConsensusGroupId-{}: TsFileWriter exit: Writing file is null. No need to delete.", - consensusGroupId.getId()); + "PipeConsensus-PipeName-{}: TsFileWriter exit: Writing file is null. No need to delete.", + consensusPipeName.toString()); } } } @@ -1152,8 +1145,8 @@ public RequestExecutor(PipeConsensusReceiverMetrics metric) { private void onSuccess(long nextSyncedCommitIndex, boolean isTransferTsFileSeal) { LOGGER.info( - "PipeConsensus-ConsensusGroupId-{}: process no.{} event successfully!", - consensusGroupId, + "PipeConsensus-PipeName-{}: process no.{} event successfully!", + consensusPipeName, nextSyncedCommitIndex); RequestMeta curMeta = reqExecutionOrderBuffer.pollFirst(); onSyncedCommitIndex = nextSyncedCommitIndex; @@ -1180,8 +1173,8 @@ private TPipeConsensusTransferResp onRequest( TCommitId tCommitId = req.getCommitId(); RequestMeta requestMeta = new RequestMeta(tCommitId); LOGGER.info( - "PipeConsensus-ConsensusGroup-{}: start to receive no.{} event", - consensusGroupId, + "PipeConsensus-PipeName-{}: start to receive no.{} event", + consensusPipeName, tCommitId.getCommitIndex()); // if a req is deprecated, we will discard it // This case may happen in this scenario: leader has transferred {1,2} and is intending to @@ -1199,8 +1192,8 @@ private TPipeConsensusTransferResp onRequest( TSStatusCode.PIPE_CONSENSUS_DEPRECATED_REQUEST, "PipeConsensus receiver received a deprecated request, which may be sent before the connector restart. Consider to discard it")); LOGGER.info( - "PipeConsensus-ConsensusGroup-{}: received a deprecated request, which may be sent before the connector restart. Consider to discard it", - consensusGroupId); + "PipeConsensus-PipeName-{}: received a deprecated request, which may be sent before the connector restart. Consider to discard it", + consensusPipeName); return new TPipeConsensusTransferResp(status); } // Judge whether connector has rebooted or not, if the rebootTimes increases compared to @@ -1304,8 +1297,8 @@ private TPipeConsensusTransferResp onRequest( } } catch (InterruptedException e) { LOGGER.warn( - "PipeConsensus-ConsensusGroup-{}: current waiting is interrupted. onSyncedCommitIndex: {}. Exception: ", - consensusGroupId, + "PipeConsensus-PipeName-{}: current waiting is interrupted. onSyncedCommitIndex: {}. Exception: ", + consensusPipeName, tCommitId.getCommitIndex(), e); Thread.currentThread().interrupt(); @@ -1328,8 +1321,8 @@ private TPipeConsensusTransferResp onRequest( */ private void resetWithNewestRebootTime(int connectorRebootTimes) { LOGGER.info( - "PipeConsensus-ConsensusGroup-{}: receiver detected an newer rebootTimes, which indicates the leader has rebooted. receiver will reset all its data.", - consensusGroupId); + "PipeConsensus-PipeName-{}: receiver detected an newer rebootTimes, which indicates the leader has rebooted. receiver will reset all its data.", + consensusPipeName); this.reqExecutionOrderBuffer.clear(); this.onSyncedCommitIndex = -1; // sync the follower's connectorRebootTimes with connector's actual rebootTimes From 713a290783db2e6876a24270da3c2e75720c298a Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 10:24:59 +0800 Subject: [PATCH 20/22] fix review --- .../apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java | 5 ++--- .../protocol/pipeconsensus/PipeConsensusReceiver.java | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java index 685a11ba6d5d6..95d5a23cdc2fa 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java @@ -300,11 +300,10 @@ public TSStatus write(IConsensusRequest request) { ((ComparableConsensusRequest) request) .setProgressIndex(progressIndexManager.assignProgressIndex(thisNode.getGroupId())); } - - long startWriteTime = System.nanoTime(); TSStatus result = stateMachine.write(request); long writeToStateMachineEndTime = System.nanoTime(); - PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(writeToStateMachineEndTime - startWriteTime); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost( + writeToStateMachineEndTime - writeToStateMachineStartTime); // statistic the time of writing request into stateMachine pipeConsensusServerMetrics.recordUserWriteStateMachineTime( writeToStateMachineEndTime - writeToStateMachineStartTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 12334236eb0bc..d6b106705a12f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -1130,8 +1130,8 @@ private class RequestExecutor { private final PipeConsensusReceiverMetrics metric; private long onSyncedCommitIndex = 0; private int connectorRebootTimes = 0; - private int WALEventCount = 0; - private int tsFileEventCount = 0; + private volatile int WALEventCount = 0; + private volatile int tsFileEventCount = 0; public RequestExecutor(PipeConsensusReceiverMetrics metric) { this.reqExecutionOrderBuffer = From 86a0219a0611f5da6fa124223fb78004215efb38 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 12:49:53 +0800 Subject: [PATCH 21/22] avoid log error msg when it comes to PipeConsensusEnqueueException --- .../PipeConsensusEnqueueException.java | 26 +++++++++++++++++++ .../PipeConsensusAsyncConnector.java | 6 ++--- .../threadpool/WrappedThreadPoolExecutor.java | 4 ++- 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java new file mode 100644 index 0000000000000..73fa6dbbcd384 --- /dev/null +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.api.exception; + +public class PipeConsensusEnqueueException extends PipeException { + public PipeConsensusEnqueueException(String message) { + super(message); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index e3673c0330d93..472cb201245b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -56,7 +56,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; -import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeConsensusEnqueueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -233,7 +233,7 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tabletInsertionEvent); if (!enqueueResult) { - throw new PipeException(ENQUEUE_EXCEPTION_MSG); + throw new PipeConsensusEnqueueException(ENQUEUE_EXCEPTION_MSG); } // batch transfer tablets. if (isTabletBatchModeEnabled) { @@ -328,7 +328,7 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tsFileInsertionEvent); if (!enqueueResult) { - throw new PipeException(ENQUEUE_EXCEPTION_MSG); + throw new PipeConsensusEnqueueException(ENQUEUE_EXCEPTION_MSG); } final PipeTsFileInsertionEvent pipeTsFileInsertionEvent = (PipeTsFileInsertionEvent) tsFileInsertionEvent; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java index 2c24f045ca533..7e39258a0c60d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.concurrent.ThreadPoolMetrics; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.service.JMXService; +import org.apache.iotdb.pipe.api.exception.PipeConsensusEnqueueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,7 +127,8 @@ protected void afterExecute(Runnable r, Throwable t) { Thread.currentThread().interrupt(); } } - if (t != null && !disableErrorLog) { + // we don't expect to log error msg when it comes to PipeConsensusEnqueueException + if (t != null && !disableErrorLog && !(t instanceof PipeConsensusEnqueueException)) { logger.error("Exception in thread pool {}", mbeanName, t); } } From 578634a32f931ff4fc0d378da7b203433f8660f8 Mon Sep 17 00:00:00 2001 From: Peng Junzhi <201250214@smail.nju.edu.cn> Date: Fri, 14 Jun 2024 13:03:50 +0800 Subject: [PATCH 22/22] Revert "avoid log error msg when it comes to PipeConsensusEnqueueException" This reverts commit 86a0219a0611f5da6fa124223fb78004215efb38. --- .../PipeConsensusEnqueueException.java | 26 ------------------- .../PipeConsensusAsyncConnector.java | 6 ++--- .../threadpool/WrappedThreadPoolExecutor.java | 4 +-- 3 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java deleted file mode 100644 index 73fa6dbbcd384..0000000000000 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConsensusEnqueueException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.api.exception; - -public class PipeConsensusEnqueueException extends PipeException { - public PipeConsensusEnqueueException(String message) { - super(message); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index 472cb201245b9..e3673c0330d93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -56,7 +56,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; -import org.apache.iotdb.pipe.api.exception.PipeConsensusEnqueueException; +import org.apache.iotdb.pipe.api.exception.PipeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -233,7 +233,7 @@ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tabletInsertionEvent); if (!enqueueResult) { - throw new PipeConsensusEnqueueException(ENQUEUE_EXCEPTION_MSG); + throw new PipeException(ENQUEUE_EXCEPTION_MSG); } // batch transfer tablets. if (isTabletBatchModeEnabled) { @@ -328,7 +328,7 @@ public void transfer(TsFileInsertionEvent tsFileInsertionEvent) throws Exception boolean enqueueResult = addEvent2Buffer((EnrichedEvent) tsFileInsertionEvent); if (!enqueueResult) { - throw new PipeConsensusEnqueueException(ENQUEUE_EXCEPTION_MSG); + throw new PipeException(ENQUEUE_EXCEPTION_MSG); } final PipeTsFileInsertionEvent pipeTsFileInsertionEvent = (PipeTsFileInsertionEvent) tsFileInsertionEvent; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java index 7e39258a0c60d..2c24f045ca533 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.concurrent.ThreadPoolMetrics; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.service.JMXService; -import org.apache.iotdb.pipe.api.exception.PipeConsensusEnqueueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,8 +126,7 @@ protected void afterExecute(Runnable r, Throwable t) { Thread.currentThread().interrupt(); } } - // we don't expect to log error msg when it comes to PipeConsensusEnqueueException - if (t != null && !disableErrorLog && !(t instanceof PipeConsensusEnqueueException)) { + if (t != null && !disableErrorLog) { logger.error("Exception in thread pool {}", mbeanName, t); } }