Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.iotdb.commons.client.container.PipeConsensusClientMgrContainer;
import org.apache.iotdb.consensus.config.ConsensusConfig;
import org.apache.iotdb.consensus.config.PipeConsensusConfig.ReplicateMode;
import org.apache.iotdb.consensus.pipe.metric.PipeConsensusSyncLagManager;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -66,6 +67,8 @@ public static Optional<IConsensus> getConsensusImpl(
className = REAL_PIPE_CONSENSUS;
// initialize pipeConsensus' thrift component
PipeConsensusClientMgrContainer.build();
// initialize pipeConsensus's metric component
PipeConsensusSyncLagManager.build();
}
Class<?> executor = Class.forName(className);
Constructor<?> executorConstructor =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,19 @@
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.pipe.task.meta.PipeStatus;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.consensus.IStateMachine;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.consensus.common.request.IConsensusRequest;
import org.apache.iotdb.consensus.config.PipeConsensusConfig;
import org.apache.iotdb.consensus.config.PipeConsensusConfig.ReplicateMode;
import org.apache.iotdb.consensus.exception.ConsensusGroupModifyPeerException;
import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeManager;
import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName;
import org.apache.iotdb.consensus.pipe.consensuspipe.ProgressIndexManager;
import org.apache.iotdb.consensus.pipe.metric.PipeConsensusServerMetrics;
import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompletedReq;
import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompletedResp;
import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToCreateConsensusPipeReq;
Expand Down Expand Up @@ -66,7 +70,8 @@
public class PipeConsensusServerImpl {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusServerImpl.class);
private static final long CHECK_TRANSMISSION_COMPLETION_INTERVAL_IN_MILLISECONDS = 2_000L;

private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS =
PerformanceOverviewMetrics.getInstance();
private final Peer thisNode;
private final IStateMachine stateMachine;
private final Lock stateMachineLock = new ReentrantLock();
Expand All @@ -77,6 +82,8 @@ public class PipeConsensusServerImpl {
private final ConsensusPipeManager consensusPipeManager;
private final ProgressIndexManager progressIndexManager;
private final IClientManager<TEndPoint, SyncPipeConsensusServiceClient> syncClientManager;
private final PipeConsensusServerMetrics pipeConsensusServerMetrics;
private final ReplicateMode replicateMode;

private ProgressIndex cachedProgressIndex = MinimumProgressIndex.INSTANCE;

Expand All @@ -98,6 +105,8 @@ public PipeConsensusServerImpl(
this.consensusPipeManager = consensusPipeManager;
this.progressIndexManager = config.getPipe().getProgressIndexManager();
this.syncClientManager = syncClientManager;
this.pipeConsensusServerMetrics = new PipeConsensusServerMetrics(this);
this.replicateMode = config.getReplicateMode();

if (configuration.isEmpty()) {
peerManager.recover();
Expand Down Expand Up @@ -129,6 +138,7 @@ public PipeConsensusServerImpl(

public synchronized void start(boolean startConsensusPipes) throws IOException {
stateMachine.start();
MetricService.getInstance().addMetricSet(this.pipeConsensusServerMetrics);
Comment thread
Pengzna marked this conversation as resolved.
Resolved

if (startConsensusPipes) {
// start all consensus pipes
Expand All @@ -153,7 +163,7 @@ public synchronized void stop() {
// do not roll back, because it will stop anyway
LOGGER.warn("{} cannot stop all consensus pipes", thisNode);
}

MetricService.getInstance().removeMetricSet(this.pipeConsensusServerMetrics);
stateMachine.stop();
isStarted.set(false);
}
Expand All @@ -167,6 +177,7 @@ public synchronized void clear() throws IOException {
LOGGER.warn("{} cannot drop all consensus pipes", thisNode);
}

MetricService.getInstance().removeMetricSet(this.pipeConsensusServerMetrics);
peerManager.clear();
stateMachine.stop();
isStarted.set(false);
Expand All @@ -183,7 +194,11 @@ private List<Peer> createConsensusPipes(List<Peer> peers) {
}
return true;
} catch (Exception e) {
LOGGER.warn("{} cannot create consensus pipe between {} and {}", thisNode, peer, e);
LOGGER.warn(
"{}: cannot create consensus pipe between {} and {}",
e.getMessage(),
thisNode,
peer);
return false;
}
})
Expand All @@ -202,11 +217,11 @@ private List<Peer> updateConsensusPipesStatus(List<Peer> peers, PipeStatus statu
return true;
} catch (Exception e) {
LOGGER.warn(
"{} cannot update consensus pipe between {} and {} to status {}",
"{}: cannot update consensus pipe between {} and {} to status {}",
e.getMessage(),
thisNode,
peer,
status,
e);
status);
return false;
}
})
Expand Down Expand Up @@ -274,21 +289,49 @@ public synchronized void checkConsensusPipe(Map<ConsensusPipeName, PipeStatus> e

public TSStatus write(IConsensusRequest request) {
try {
long consensusWriteStartTime = System.nanoTime();
stateMachineLock.lock();
long getStateMachineLockTime = System.nanoTime();
// statistic the time of acquiring stateMachine lock
pipeConsensusServerMetrics.recordGetStateMachineLockTime(
getStateMachineLockTime - consensusWriteStartTime);
long writeToStateMachineStartTime = System.nanoTime();
if (request instanceof ComparableConsensusRequest) {
((ComparableConsensusRequest) request)
.setProgressIndex(progressIndexManager.assignProgressIndex(thisNode.getGroupId()));
}
return stateMachine.write(request);
TSStatus result = stateMachine.write(request);
long writeToStateMachineEndTime = System.nanoTime();
PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(
writeToStateMachineEndTime - writeToStateMachineStartTime);
// statistic the time of writing request into stateMachine
pipeConsensusServerMetrics.recordUserWriteStateMachineTime(
writeToStateMachineEndTime - writeToStateMachineStartTime);
return result;
} finally {
stateMachineLock.unlock();
}
}

public TSStatus writeOnFollowerReplica(IConsensusRequest request) {
try {
long consensusWriteStartTime = System.nanoTime();
stateMachineLock.lock();
return stateMachine.write(request);
long getStateMachineLockTime = System.nanoTime();
// statistic the time of acquiring stateMachine lock
pipeConsensusServerMetrics.recordGetStateMachineLockTime(
getStateMachineLockTime - consensusWriteStartTime);

long writeToStateMachineStartTime = System.nanoTime();
TSStatus result = stateMachine.write(request);
long writeToStateMachineEndTime = System.nanoTime();

PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(
writeToStateMachineEndTime - writeToStateMachineStartTime);
// statistic the time of writing request into stateMachine
pipeConsensusServerMetrics.recordReplicaWriteStateMachineTime(
writeToStateMachineEndTime - writeToStateMachineStartTime);
return result;
} finally {
stateMachineLock.unlock();
}
Expand Down Expand Up @@ -540,4 +583,12 @@ public boolean containsPeer(Peer peer) {
public List<Peer> getPeers() {
return peerManager.getPeers();
}

public String getConsensusGroupId() {
return consensusGroupId;
}

public long getReplicateMode() {
return (replicateMode == ReplicateMode.BATCH) ? 2 : 1;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iotdb.consensus.pipe.consensuspipe;

public interface ConsensusPipeConnector {
Comment thread
Pengzna marked this conversation as resolved.
Resolved
long getConsensusPipeCommitProgress();

long getConsensusPipeReplicateProgress();
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Map;

import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_GROUP_ID_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_CONSENSUS_PIPE_NAME;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY;
Expand Down Expand Up @@ -78,6 +79,7 @@ public void createConsensusPipe(Peer senderPeer, Peer receiverPeer) throws Excep
.put(
CONNECTOR_CONSENSUS_GROUP_ID_KEY,
String.valueOf(consensusPipeName.getConsensusGroupId().getId()))
.put(CONNECTOR_CONSENSUS_PIPE_NAME, consensusPipeName.toString())
.put(CONNECTOR_IOTDB_IP_KEY, receiverPeer.getEndpoint().ip)
.put(CONNECTOR_IOTDB_PORT_KEY, String.valueOf(receiverPeer.getEndpoint().port))
.put(CONNECTOR_IOTDB_PARALLEL_TASKS_KEY, String.valueOf(1))
Expand Down
Loading