From ab1ff5a3a02a1448806fb86e72c8f87f0a3b8e17 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:35:56 +0800 Subject: [PATCH 1/7] Altered atomic --- ...IoTDBDataNodeCacheLeaderClientManager.java | 21 ++++----- .../PartialPathLastObjectCache.java | 16 +++---- .../wal/utils/WALInsertNodeCache.java | 43 ++++++++++--------- .../event/SubscriptionEventBinaryCache.java | 8 ++-- 4 files changed, 45 insertions(+), 43 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java index be3cb9d4fdaeb..965a2a068bf2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java @@ -50,21 +50,22 @@ class LeaderCacheManager { private final ConcurrentHashMap endPoints = new ConcurrentHashMap<>(); public LeaderCacheManager() { - long initMemorySizeInBytes = PipeResourceManager.memory().getTotalMemorySizeInBytes() / 10; - long maxMemorySizeInBytes = + final long initMemorySizeInBytes = + PipeResourceManager.memory().getTotalMemorySizeInBytes() / 10; + final long maxMemorySizeInBytes = (long) (PipeResourceManager.memory().getTotalMemorySizeInBytes() * CONFIG.getPipeLeaderCacheMemoryUsagePercentage()); // properties required by pipe memory control framework - PipeMemoryBlock allocatedMemoryBlock = + final PipeMemoryBlock allocatedMemoryBlock = PipeResourceManager.memory() .tryAllocate(initMemorySizeInBytes) .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 1)) .setShrinkCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() * ((double) oldMemory / newMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor * ((double) oldMemory / newMemory)); LOGGER.info( "LeaderCacheManager.allocatedMemoryBlock has shrunk from {} to {}.", oldMemory, @@ -74,8 +75,8 @@ public LeaderCacheManager() { oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, maxMemorySizeInBytes)) .setExpandCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() / ((double) newMemory / oldMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor / ((double) newMemory / oldMemory)); LOGGER.info( "LeaderCacheManager.allocatedMemoryBlock has expanded from {} to {}.", oldMemory, @@ -100,16 +101,16 @@ public LeaderCacheManager() { .build(); } - public TEndPoint getLeaderEndPoint(String deviceId) { + public TEndPoint getLeaderEndPoint(final String deviceId) { return deviceId == null ? null : device2endpoint.getIfPresent(deviceId); } - public void updateLeaderEndPoint(String deviceId, TEndPoint endPoint) { + public void updateLeaderEndPoint(final String deviceId, final TEndPoint endPoint) { if (deviceId == null || endPoint == null) { return; } - TEndPoint endPointFromMap = endPoints.putIfAbsent(endPoint, endPoint); + final TEndPoint endPointFromMap = endPoints.putIfAbsent(endPoint, endPoint); if (endPointFromMap != null) { device2endpoint.put(deviceId, endPointFromMap); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/PartialPathLastObjectCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/PartialPathLastObjectCache.java index bbee9babed948..ece3963c6f478 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/PartialPathLastObjectCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/PartialPathLastObjectCache.java @@ -40,15 +40,15 @@ public abstract class PartialPathLastObjectCache implements AutoCloseable { private final Cache partialPath2ObjectCache; - public PartialPathLastObjectCache(long memoryLimitInBytes) { + protected PartialPathLastObjectCache(final long memoryLimitInBytes) { allocatedMemoryBlock = PipeResourceManager.memory() .tryAllocate(memoryLimitInBytes) .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 1)) .setShrinkCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() * ((double) oldMemory / newMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor * ((double) oldMemory / newMemory)); LOGGER.info( "PartialPathLastObjectCache.allocatedMemoryBlock has shrunk from {} to {}.", oldMemory, @@ -57,8 +57,8 @@ public PartialPathLastObjectCache(long memoryLimitInBytes) { .setExpandMethod(oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, memoryLimitInBytes)) .setExpandCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() / ((double) newMemory / oldMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor / ((double) newMemory / oldMemory)); LOGGER.info( "PartialPathLastObjectCache.allocatedMemoryBlock has expanded from {} to {}.", oldMemory, @@ -87,15 +87,15 @@ public PartialPathLastObjectCache(long memoryLimitInBytes) { .build(); } - protected abstract long calculateMemoryUsage(T object); + protected abstract long calculateMemoryUsage(final T object); /////////////////////////// Getter & Setter /////////////////////////// - public T getPartialPathLastObject(String partialPath) { + public T getPartialPathLastObject(final String partialPath) { return partialPath2ObjectCache.getIfPresent(partialPath); } - public void setPartialPathLastObject(String partialPath, T object) { + public void setPartialPathLastObject(final String partialPath, final T object) { partialPath2ObjectCache.put(partialPath, object); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java index 0a5718da6f14d..38e010db7db77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java @@ -71,7 +71,7 @@ public class WALInsertNodeCache { private volatile boolean hasPipeRunning = false; - private WALInsertNodeCache(Integer dataRegionId) { + private WALInsertNodeCache(final Integer dataRegionId) { final long requestedAllocateSize = (long) Math.min( @@ -83,8 +83,8 @@ private WALInsertNodeCache(Integer dataRegionId) { .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 1)) .setShrinkCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() * ((double) oldMemory / newMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor * ((double) oldMemory / newMemory)); isBatchLoadEnabled.set(newMemory >= CONFIG.getWalFileSizeThresholdInByte()); LOGGER.info( "WALInsertNodeCache.allocatedMemoryBlock of dataRegion {} has shrunk from {} to {}.", @@ -96,8 +96,8 @@ private WALInsertNodeCache(Integer dataRegionId) { oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, requestedAllocateSize)) .setExpandCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() / ((double) newMemory / oldMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor / ((double) newMemory / oldMemory)); isBatchLoadEnabled.set(newMemory >= CONFIG.getWalFileSizeThresholdInByte()); LOGGER.info( "WALInsertNodeCache.allocatedMemoryBlock of dataRegion {} has expanded from {} to {}.", @@ -128,7 +128,7 @@ private WALInsertNodeCache(Integer dataRegionId) { /////////////////////////// Getter & Setter /////////////////////////// - public InsertNode getInsertNode(WALEntryPosition position) { + public InsertNode getInsertNode(final WALEntryPosition position) { final Pair pair = getByteBufferOrInsertNode(position); if (pair.getRight() != null) { @@ -145,7 +145,7 @@ public InsertNode getInsertNode(WALEntryPosition position) { final InsertNode insertNode = parse(ByteBuffer.wrap(pair.getLeft().array())); pair.setRight(insertNode); return insertNode; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.error( "Parsing failed when recovering insertNode from wal, walFile:{}, position:{}, size:{}, exception:", position.getWalFile(), @@ -156,7 +156,7 @@ public InsertNode getInsertNode(WALEntryPosition position) { } } - private InsertNode parse(ByteBuffer buffer) { + private InsertNode parse(final ByteBuffer buffer) { final PlanNode node = WALEntry.deserializeForConsensus(buffer); if (node instanceof InsertNode) { return (InsertNode) node; @@ -165,7 +165,7 @@ private InsertNode parse(ByteBuffer buffer) { } } - public ByteBuffer getByteBuffer(WALEntryPosition position) { + public ByteBuffer getByteBuffer(final WALEntryPosition position) { Pair pair = getByteBufferOrInsertNode(position); if (pair.getLeft() != null) { @@ -187,7 +187,7 @@ public ByteBuffer getByteBuffer(WALEntryPosition position) { return ByteBuffer.wrap(pair.getLeft().array()); } - public Pair getByteBufferOrInsertNode(WALEntryPosition position) { + public Pair getByteBufferOrInsertNode(final WALEntryPosition position) { hasPipeRunning = true; final Pair pair = @@ -202,7 +202,8 @@ public Pair getByteBufferOrInsertNode(WALEntryPosition p return pair; } - public void cacheInsertNodeIfNeeded(WALEntryPosition walEntryPosition, InsertNode insertNode) { + public void cacheInsertNodeIfNeeded( + final WALEntryPosition walEntryPosition, final InsertNode insertNode) { // reduce memory usage if (hasPipeRunning) { lruCache.put(walEntryPosition, new Pair<>(null, insertNode)); @@ -225,11 +226,11 @@ public double getCacheRequestCount() { /////////////////////////// MemTable /////////////////////////// - public void addMemTable(long memTableId) { + public void addMemTable(final long memTableId) { memTablesNeedSearch.add(memTableId); } - public void removeMemTable(long memTableId) { + public void removeMemTable(final long memTableId) { memTablesNeedSearch.remove(memTableId); } @@ -239,7 +240,7 @@ class WALInsertNodeCacheLoader implements CacheLoader> { @Override - public @Nullable Pair load(@NonNull WALEntryPosition key) + public @Nullable Pair load(@NonNull final WALEntryPosition key) throws Exception { return new Pair<>(key.read(), null); } @@ -247,10 +248,10 @@ class WALInsertNodeCacheLoader /** Batch load all wal entries in the file when any one key is absent. */ @Override public @NonNull Map<@NonNull WALEntryPosition, @NonNull Pair> loadAll( - @NonNull Iterable walEntryPositions) { + @NonNull final Iterable walEntryPositions) { final Map> loadedEntries = new HashMap<>(); - for (WALEntryPosition walEntryPosition : walEntryPositions) { + for (final WALEntryPosition walEntryPosition : walEntryPositions) { if (loadedEntries.containsKey(walEntryPosition) || !walEntryPosition.canRead()) { continue; } @@ -261,7 +262,7 @@ class WALInsertNodeCacheLoader if (!walEntryPosition.isInSealedFile()) { try { loadedEntries.put(walEntryPosition, load(walEntryPosition)); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.info( "Fail to cache wal entries from the wal file with version id {}", walFileVersionId, @@ -295,7 +296,7 @@ class WALInsertNodeCacheLoader position += size; } - } catch (IOException e) { + } catch (final IOException e) { LOGGER.info( "Fail to cache wal entries from the wal file with version id {}", walFileVersionId, @@ -309,7 +310,7 @@ class WALInsertNodeCacheLoader /////////////////////////// Singleton /////////////////////////// - public static WALInsertNodeCache getInstance(Integer regionId) { + public static WALInsertNodeCache getInstance(final Integer regionId) { return InstanceHolder.getOrCreateInstance(regionId); } @@ -317,7 +318,7 @@ private static class InstanceHolder { private static final Map INSTANCE_MAP = new ConcurrentHashMap<>(); - public static WALInsertNodeCache getOrCreateInstance(Integer key) { + public static WALInsertNodeCache getOrCreateInstance(final Integer key) { return INSTANCE_MAP.computeIfAbsent(key, k -> new WALInsertNodeCache(key)); } @@ -334,7 +335,7 @@ public boolean isBatchLoadEnabled() { } @TestOnly - public void setIsBatchLoadEnabled(boolean isBatchLoadEnabled) { + public void setIsBatchLoadEnabled(final boolean isBatchLoadEnabled) { this.isBatchLoadEnabled.set(isBatchLoadEnabled); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEventBinaryCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEventBinaryCache.java index 4a592f8a86f3b..5f7ea9adc5cbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEventBinaryCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEventBinaryCache.java @@ -106,8 +106,8 @@ private SubscriptionEventBinaryCache() { .setShrinkMethod(oldMemory -> Math.max(oldMemory / 2, 1)) .setShrinkCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() * ((double) oldMemory / newMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor * ((double) oldMemory / newMemory)); LOGGER.info( "SubscriptionEventBinaryCache.allocatedMemoryBlock has shrunk from {} to {}.", oldMemory, @@ -117,8 +117,8 @@ private SubscriptionEventBinaryCache() { oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, maxMemorySizeInBytes)) .setExpandCallback( (oldMemory, newMemory) -> { - memoryUsageCheatFactor.set( - memoryUsageCheatFactor.get() / ((double) newMemory / oldMemory)); + memoryUsageCheatFactor.updateAndGet( + factor -> factor / ((double) newMemory / oldMemory)); LOGGER.info( "SubscriptionEventBinaryCache.allocatedMemoryBlock has expanded from {} to {}.", oldMemory, From 93d5082c4f1f955767f2307a898942315549efdc Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:20:17 +0800 Subject: [PATCH 2/7] bug fix --- .../IoTDBConfigRegionAirGapConnector.java | 7 +++- .../protocol/IoTDBConfigRegionConnector.java | 7 +++- .../db/consensus/DataRegionConsensusImpl.java | 9 +++-- .../schemaregion/SchemaExecutionVisitor.java | 4 +- .../SchemaRegionStateMachine.java | 16 ++++---- ...{PipeAgent.java => PipeDataNodeAgent.java} | 20 +++++----- .../pipe/agent/runtime/PipeAgentLauncher.java | 6 +-- .../runtime/PipeDataNodeRuntimeAgent.java | 10 ++--- .../agent/task/PipeDataNodeTaskAgent.java | 19 ++++------ .../evolvable/builder/PipeEventBatch.java | 18 +++++---- .../IoTDBDataRegionAirGapConnector.java | 16 ++++++-- .../PipeConsensusAsyncConnector.java | 4 +- .../PipeTransferTabletBatchEventHandler.java | 9 +++-- ...peTransferTabletInsertionEventHandler.java | 22 +++++++---- ...peTransferTsFileInsertionEventHandler.java | 8 +++- .../sync/IoTDBDataNodeSyncConnector.java | 1 + .../sync/IoTDBDataRegionSyncConnector.java | 20 ++++++---- .../sync/IoTDBSchemaRegionConnector.java | 6 ++- .../writeback/WriteBackConnector.java | 4 +- .../ConsensusPipeDataNodeDispatcher.java | 4 +- ...ensusPipeDataNodeRuntimeAgentGuardian.java | 4 +- .../ProgressIndexDataNodeManager.java | 6 +-- .../common/terminate/PipeTerminateEvent.java | 4 +- .../PipeRealtimeDataRegionExtractor.java | 8 ++-- ...PipeRealtimeDataRegionHybridExtractor.java | 8 ++-- .../PipeRealtimeDataRegionLogExtractor.java | 11 ++++-- ...PipeRealtimeDataRegionTsFileExtractor.java | 8 ++-- .../PipeInsertionDataNodeListener.java | 4 +- .../IoTDBSchemaRegionExtractor.java | 13 ++++--- .../aggregate/AggregateProcessor.java | 4 +- .../twostage/combiner/PipeCombineHandler.java | 4 +- .../combiner/PipeCombineHandlerManager.java | 4 +- .../protocol/airgap/IoTDBAirGapReceiver.java | 6 +-- .../thrift/IoTDBDataNodeReceiver.java | 4 +- .../resource/memory/PipeMemoryManager.java | 4 +- .../tsfile/PipeTsFileResourceManager.java | 4 +- .../resource/wal/PipeWALResourceManager.java | 4 +- .../task/connection/PipeEventCollector.java | 4 +- .../task/stage/PipeTaskExtractorStage.java | 6 +-- .../task/stage/PipeTaskProcessorStage.java | 6 +-- .../connector/PipeConnectorSubtask.java | 4 +- .../PipeConnectorSubtaskManager.java | 8 ++-- .../processor/PipeProcessorSubtask.java | 4 +- .../thrift/impl/ClientRPCServiceImpl.java | 14 +++---- .../impl/DataNodeInternalRPCServiceImpl.java | 24 ++++++------ .../thrift/impl/DataNodeRegionManager.java | 4 +- .../execution/load/LoadTsFileManager.java | 4 +- .../executor/ClusterConfigTaskExecutor.java | 15 +++++--- .../load/LoadTsFileDispatcherImpl.java | 6 +-- .../scheduler/load/LoadTsFileScheduler.java | 4 +- .../org/apache/iotdb/db/service/DataNode.java | 8 ++-- .../dataregion/memtable/TsFileProcessor.java | 8 ++-- .../file/AbstractTsFileRecoverPerformer.java | 4 +- .../file/UnsealedTsFileRecoverPerformer.java | 4 +- .../broker/SubscriptionPrefetchingQueue.java | 6 +-- .../SubscriptionConnectorSubtaskManager.java | 4 +- .../SchemaRegionListeningQueueTest.java | 22 +++++------ .../PipeTsFileResourceManagerTest.java | 8 ++-- .../iotdb/commons/conf/CommonConfig.java | 10 +++++ .../iotdb/commons/conf/CommonDescriptor.java | 5 +++ .../pipe/agent/task/PipeTaskAgent.java | 8 ++++ .../iotdb/commons/pipe/config/PipeConfig.java | 7 ++++ .../limiter/PipeEndPointRateLimiter.java | 33 ++++++++++++++-- .../protocol/IoTDBAirGapConnector.java | 38 +++++++++++-------- .../connector/protocol/IoTDBConnector.java | 13 +++++-- .../protocol/IoTDBSslSyncConnector.java | 14 ++++--- 66 files changed, 363 insertions(+), 242 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/{PipeAgent.java => PipeDataNodeAgent.java} (81%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java index 9ea34939864c6..1c5febcb4bf51 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java @@ -151,6 +151,7 @@ private void doTransfer( throws PipeException, IOException { if (!send( pipeConfigRegionWritePlanEvent.getPipeName(), + pipeConfigRegionWritePlanEvent.getCreationTime(), socket, PipeTransferConfigPlanReq.toTPipeTransferBytes( pipeConfigRegionWritePlanEvent.getConfigPhysicalPlan()))) { @@ -189,17 +190,19 @@ private void doTransfer( final AirGapSocket socket, final PipeConfigRegionSnapshotEvent pipeConfigRegionSnapshotEvent) throws PipeException, IOException { final String pipeName = pipeConfigRegionSnapshotEvent.getPipeName(); + final long creationTime = pipeConfigRegionSnapshotEvent.getCreationTime(); final File snapshot = pipeConfigRegionSnapshotEvent.getSnapshotFile(); final File templateFile = pipeConfigRegionSnapshotEvent.getTemplateFile(); // 1. Transfer snapshotFile, and template file if exists - transferFilePieces(pipeName, snapshot, socket, true); + transferFilePieces(pipeName, creationTime, snapshot, socket, true); if (Objects.nonNull(templateFile)) { - transferFilePieces(pipeName, templateFile, socket, true); + transferFilePieces(pipeName, creationTime, templateFile, socket, true); } // 2. Transfer file seal signal, which means the snapshots are transferred completely if (!send( pipeName, + creationTime, socket, PipeTransferConfigSnapshotSealReq.toTPipeTransferBytes( // The pattern is surely Non-null diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java index 00279bf22d9f6..25fc9de720a37 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java @@ -130,6 +130,7 @@ private void doTransfer(final PipeConfigRegionWritePlanEvent pipeConfigRegionWri pipeConfigRegionWritePlanEvent.getConfigPhysicalPlan())); rateLimitIfNeeded( pipeConfigRegionWritePlanEvent.getPipeName(), + pipeConfigRegionWritePlanEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); @@ -181,14 +182,15 @@ private void doTransferWrapper(final PipeConfigRegionSnapshotEvent pipeConfigReg private void doTransfer(final PipeConfigRegionSnapshotEvent snapshotEvent) throws PipeException, IOException { final String pipeName = snapshotEvent.getPipeName(); + final long creationTime = snapshotEvent.getCreationTime(); final File snapshotFile = snapshotEvent.getSnapshotFile(); final File templateFile = snapshotEvent.getTemplateFile(); final Pair clientAndStatus = clientManager.getClient(); // 1. Transfer snapshotFile, and template File if exists - transferFilePieces(pipeName, snapshotFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, snapshotFile, clientAndStatus, true); if (Objects.nonNull(templateFile)) { - transferFilePieces(pipeName, templateFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, templateFile, clientAndStatus, true); } // 2. Transfer file seal signal, which means the snapshots are transferred completely final TPipeTransferResp resp; @@ -206,6 +208,7 @@ private void doTransfer(final PipeConfigRegionSnapshotEvent snapshotEvent) snapshotEvent.toSealTypeString())); rateLimitIfNeeded( snapshotEvent.getPipeName(), + snapshotEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 5873a0ff628e1..515a9f54ddca4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataRegionStateMachine; import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.consensus.ConsensusPipeDataNodeDispatcher; import org.apache.iotdb.db.pipe.consensus.ConsensusPipeDataNodeRuntimeAgentGuardian; import org.apache.iotdb.db.pipe.consensus.ProgressIndexDataNodeManager; @@ -80,7 +80,7 @@ private static class DataRegionConsensusImplHolder { // Make sure both statics are initialized. static { reinitializeStatics(); - PipeAgent.receiver().pipeConsensus().initConsensusInRuntime(); + PipeDataNodeAgent.receiver().pipeConsensus().initConsensusInRuntime(); } private static void reinitializeStatics() { @@ -164,8 +164,9 @@ private static ConsensusConfig buildConsensusConfig() { // name .setConsensusPipeDispatcher(new ConsensusPipeDataNodeDispatcher()) .setConsensusPipeGuardian(new ConsensusPipeDataNodeRuntimeAgentGuardian()) - .setConsensusPipeSelector(() -> PipeAgent.task().getAllConsensusPipe()) - .setConsensusPipeReceiver(PipeAgent.receiver().pipeConsensus()) + .setConsensusPipeSelector( + () -> PipeDataNodeAgent.task().getAllConsensusPipe()) + .setConsensusPipeReceiver(PipeDataNodeAgent.receiver().pipeConsensus()) .setProgressIndexManager(new ProgressIndexDataNodeManager()) .setConsensusPipeGuardJobIntervalInSeconds(300) // TODO: move to config .build()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index e82ed9220459d..0dbdeac8ce75e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -28,7 +28,7 @@ import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.exception.metadata.MeasurementAlreadyExistException; import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; @@ -541,7 +541,7 @@ public TSStatus visitPipeEnrichedNonWritePlanNode( public TSStatus visitPipeOperateSchemaQueueNode( final PipeOperateSchemaQueueNode node, final ISchemaRegion schemaRegion) { final SchemaRegionId id = schemaRegion.getSchemaRegionId(); - final SchemaRegionListeningQueue queue = PipeAgent.runtime().schemaListener(id); + final SchemaRegionListeningQueue queue = PipeDataNodeAgent.runtime().schemaListener(id); if (node.isOpen() && !queue.isOpened()) { logger.info("Opened pipe listening queue on schema region {}", id); queue.open(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index ee880200060e0..ca2c5f2821253 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -27,7 +27,7 @@ import org.apache.iotdb.consensus.ratis.utils.Utils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.statemachine.BaseStateMachine; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; @@ -64,7 +64,7 @@ public void start() { @Override public void stop() { // Stop leader related service for schema pipe - PipeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); } @Override @@ -79,7 +79,7 @@ public void notifyLeaderChanged(ConsensusGroupId groupId, int newLeaderId) { newLeaderId); // Shutdown leader related service for schema pipe - PipeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); logger.info( "Current node [nodeId: {}] is no longer the schema region leader [regionId: {}], " @@ -97,7 +97,7 @@ public void notifyLeaderReady() { schemaRegion.getSchemaRegionId()); // Activate leader related service for schema pipe - PipeAgent.runtime().notifySchemaLeaderReady(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().notifySchemaLeaderReady(schemaRegion.getSchemaRegionId()); logger.info( "Current node [nodeId: {}] as schema region leader [regionId: {}] is ready to work", @@ -113,7 +113,7 @@ public boolean isReadOnly() { @Override public boolean takeSnapshot(File snapshotDir) { if (schemaRegion.createSnapshot(snapshotDir) - && PipeAgent.runtime() + && PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) .createSnapshot(snapshotDir)) { listen2Snapshot4PipeListener(true); @@ -125,7 +125,7 @@ public boolean takeSnapshot(File snapshotDir) { @Override public void loadSnapshot(File latestSnapshotRootDir) { schemaRegion.loadSnapshot(latestSnapshotRootDir); - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) .loadSnapshot(latestSnapshotRootDir); // We recompute the snapshot for pipe listener when loading snapshot @@ -141,7 +141,7 @@ public void listen2Snapshot4PipeListener(boolean isTmp) { .toString(), isTmp); SchemaRegionListeningQueue listener = - PipeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId()); + PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId()); if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.getLeft())) { if (listener.isOpened()) { logger.warn( @@ -164,7 +164,7 @@ public TSStatus write(IConsensusRequest request) { try { TSStatus result = ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion); if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) .tryListenToNode((PlanNode) request); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java similarity index 81% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeAgent.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java index 5ed3bed0898bc..e350f0a6f4bbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/PipeDataNodeAgent.java @@ -25,8 +25,8 @@ import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTaskAgent; import org.apache.iotdb.db.service.DataNode; -/** {@link PipeAgent} is the entry point of the pipe module in {@link DataNode}. */ -public class PipeAgent { +/** {@link PipeDataNodeAgent} is the entry point of the pipe module in {@link DataNode}. */ +public class PipeDataNodeAgent { private final PipeDataNodePluginAgent pipeDataNodePluginAgent; private final PipeDataNodeTaskAgent pipeDataNodeTaskAgent; @@ -34,16 +34,16 @@ public class PipeAgent { private final PipeDataNodeReceiverAgent pipeDataNodeReceiverAgent; /** Private constructor to prevent users from creating a new instance. */ - private PipeAgent() { + private PipeDataNodeAgent() { pipeDataNodePluginAgent = new PipeDataNodePluginAgent(); pipeDataNodeTaskAgent = new PipeDataNodeTaskAgent(); pipeDataNodeRuntimeAgent = new PipeDataNodeRuntimeAgent(); pipeDataNodeReceiverAgent = new PipeDataNodeReceiverAgent(); } - /** The singleton holder of {@link PipeAgent}. */ - private static class PipeAgentHolder { - private static final PipeAgent HANDLE = new PipeAgent(); + /** The singleton holder of {@link PipeDataNodeAgent}. */ + private static class PipeDataNodeAgentHolder { + private static final PipeDataNodeAgent HANDLE = new PipeDataNodeAgent(); } /** @@ -52,7 +52,7 @@ private static class PipeAgentHolder { * @return the singleton instance of {@link PipeDataNodeTaskAgent} */ public static PipeDataNodeTaskAgent task() { - return PipeAgentHolder.HANDLE.pipeDataNodeTaskAgent; + return PipeDataNodeAgentHolder.HANDLE.pipeDataNodeTaskAgent; } /** @@ -61,7 +61,7 @@ public static PipeDataNodeTaskAgent task() { * @return the singleton instance of {@link PipeDataNodePluginAgent} */ public static PipeDataNodePluginAgent plugin() { - return PipeAgentHolder.HANDLE.pipeDataNodePluginAgent; + return PipeDataNodeAgentHolder.HANDLE.pipeDataNodePluginAgent; } /** @@ -70,7 +70,7 @@ public static PipeDataNodePluginAgent plugin() { * @return the singleton instance of {@link PipeDataNodeRuntimeAgent} */ public static PipeDataNodeRuntimeAgent runtime() { - return PipeAgentHolder.HANDLE.pipeDataNodeRuntimeAgent; + return PipeDataNodeAgentHolder.HANDLE.pipeDataNodeRuntimeAgent; } /** @@ -79,6 +79,6 @@ public static PipeDataNodeRuntimeAgent runtime() { * @return the singleton instance of {@link PipeDataNodeReceiverAgent} */ public static PipeDataNodeReceiverAgent receiver() { - return PipeAgentHolder.HANDLE.pipeDataNodeReceiverAgent; + return PipeDataNodeAgentHolder.HANDLE.pipeDataNodeReceiverAgent; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java index 45bd440ef7ccc..7138369639a7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java @@ -30,7 +30,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -88,7 +88,7 @@ public static synchronized void launchPipePluginAgent( if (meta.isBuiltin()) { continue; } - PipeAgent.plugin().doRegister(meta); + PipeDataNodeAgent.plugin().doRegister(meta); } } catch (Exception e) { throw new StartupException(e); @@ -159,7 +159,7 @@ public static synchronized void launchPipeTaskAgent() { throw new StartupException("Failed to get pipe task meta from config node."); } - PipeAgent.task() + PipeDataNodeAgent.task() .handlePipeMetaChanges( getAllPipeInfoResp.getAllPipeInfo().stream() .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java index 447fcbc3b6af0..dcf9f8b0435bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java @@ -32,7 +32,7 @@ import org.apache.iotdb.commons.service.ServiceType; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue; import org.apache.iotdb.db.pipe.progress.SimpleProgressIndexAssigner; import org.apache.iotdb.db.pipe.resource.PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner; @@ -70,7 +70,7 @@ public synchronized void preparePipeResources( PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.clean(); // Clean receiver file dir - PipeAgent.receiver().cleanPipeReceiverDirs(); + PipeDataNodeAgent.receiver().cleanPipeReceiverDirs(); PipeAgentLauncher.launchPipePluginAgent(resourcesInformationHolder); simpleProgressIndexAssigner.start(); @@ -83,7 +83,7 @@ public synchronized void start() throws StartupException { registerPeriodicalJob( "PipeTaskAgent#restartAllStuckPipes", - PipeAgent.task()::restartAllStuckPipes, + PipeDataNodeAgent.task()::restartAllStuckPipes, PipeConfig.getInstance().getPipeStuckRestartIntervalSeconds()); pipePeriodicalJobExecutor.start(); @@ -98,7 +98,7 @@ public synchronized void stop() { isShutdown.set(true); pipePeriodicalJobExecutor.stop(); - PipeAgent.task().dropAllPipeTasks(); + PipeDataNodeAgent.task().dropAllPipeTasks(); } public boolean isShutdown() { @@ -207,7 +207,7 @@ public void report(PipeTaskMeta pipeTaskMeta, PipeRuntimeException pipeRuntimeEx // Quick stop all pipes locally if critical exception occurs, // no need to wait for the next heartbeat cycle. if (pipeRuntimeException instanceof PipeRuntimeCriticalException) { - PipeAgent.task().stopAllPipesWithCriticalException(); + PipeDataNodeAgent.task().stopAllPipesWithCriticalException(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 1c3862bb9917d..0255dc05c3555 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter; import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; @@ -96,7 +96,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent { @Override protected boolean isShutdown() { - return PipeAgent.runtime().isShutdown(); + return PipeDataNodeAgent.runtime().isShutdown(); } @Override @@ -204,7 +204,7 @@ private Set clearSchemaRegionListeningQueueIfNecessary( schemaRegionId2ListeningQueueNewFirstIndex.forEach( (schemaRegionId, listeningQueueNewFirstIndex) -> - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .schemaListener(new SchemaRegionId(schemaRegionId)) .removeBefore(listeningQueueNewFirstIndex)); @@ -218,12 +218,12 @@ private void closeSchemaRegionListeningQueueIfNecessary( return; } - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .listeningSchemaRegionIds() .forEach( schemaRegionId -> { if (!validSchemaRegionIds.contains(schemaRegionId.getId()) - && PipeAgent.runtime().isSchemaLeaderReady(schemaRegionId)) { + && PipeDataNodeAgent.runtime().isSchemaLeaderReady(schemaRegionId)) { try { SchemaRegionConsensusImpl.getInstance() .write( @@ -297,7 +297,7 @@ public void collectPipeMetaList(final TDataNodeHeartbeatResp resp) throws TExcep private void collectPipeMetaListInternal(final TDataNodeHeartbeatResp resp) throws TException { // Do nothing if data node is removing or removed, or request does not need pipe meta list - if (PipeAgent.runtime().isShutdown()) { + if (PipeDataNodeAgent.runtime().isShutdown()) { return; } @@ -375,7 +375,7 @@ private void collectPipeMetaListInternal(final TDataNodeHeartbeatResp resp) thro protected void collectPipeMetaListInternal( final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException { // Do nothing if data node is removing or removed, or request does not need pipe meta list - if (PipeAgent.runtime().isShutdown()) { + if (PipeDataNodeAgent.runtime().isShutdown()) { return; } LOGGER.info("Received pipe heartbeat request {} from config node.", req.heartbeatId); @@ -631,11 +631,6 @@ public Set getPipeTaskRegionIdSet(final String pipeName, final long cre ///////////////////////// Pipe Consensus ///////////////////////// - public long getPipeCreationTime(final String pipeName) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); - } - public ProgressIndex getPipeTaskProgressIndex(final String pipeName, final int consensusGroupId) { if (!tryReadLockWithTimeOut(10)) { throw new PipeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/builder/PipeEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/builder/PipeEventBatch.java index 44d72b1e1c794..9707031330897 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/builder/PipeEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/builder/PipeEventBatch.java @@ -30,6 +30,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; @@ -64,7 +65,7 @@ public class PipeEventBatch implements AutoCloseable { private long totalBufferSize = 0; // Used to rate limit when transferring data - private final Map pipeName2BytesAccumulated = new HashMap<>(); + private final Map, Long> pipe2BytesAccumulated = new HashMap<>(); public PipeEventBatch(int maxDelayInMs, long requestMaxBatchSizeInBytes) { this.maxDelayInMs = maxDelayInMs; @@ -117,8 +118,9 @@ public synchronized boolean onEvent(final TabletInsertionEvent event) final int bufferSize = buildTabletInsertionBuffer(event); totalBufferSize += bufferSize; - pipeName2BytesAccumulated.compute( - ((EnrichedEvent) event).getPipeName(), + pipe2BytesAccumulated.compute( + new Pair<>( + ((EnrichedEvent) event).getPipeName(), ((EnrichedEvent) event).getCreationTime()), (pipeName, bytesAccumulated) -> bytesAccumulated == null ? bufferSize : bytesAccumulated + bufferSize); @@ -146,7 +148,7 @@ public synchronized void onSuccess() { firstEventProcessingTime = Long.MIN_VALUE; totalBufferSize = 0; - pipeName2BytesAccumulated.clear(); + pipe2BytesAccumulated.clear(); } public PipeTransferTabletBatchReq toTPipeTransferReq() throws IOException { @@ -170,12 +172,12 @@ public List deepCopyRequestCommitIds() { return new ArrayList<>(requestCommitIds); } - public Map deepCopyPipeName2BytesAccumulated() { - return new HashMap<>(pipeName2BytesAccumulated); + public Map, Long> deepCopyPipeName2BytesAccumulated() { + return new HashMap<>(pipe2BytesAccumulated); } - public Map getPipeName2BytesAccumulated() { - return pipeName2BytesAccumulated; + public Map, Long> getPipe2BytesAccumulated() { + return pipe2BytesAccumulated; } private int buildTabletInsertionBuffer(final TabletInsertionEvent event) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java index e83cd623fed64..22377cc9a6dd7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java @@ -174,7 +174,11 @@ private void doTransfer( pipeInsertNodeTabletInsertionEvent.getByteBuffer()) : PipeTransferTabletInsertNodeReq.toTPipeTransferBytes(insertNode); - if (!send(pipeInsertNodeTabletInsertionEvent.getPipeName(), socket, bytes)) { + if (!send( + pipeInsertNodeTabletInsertionEvent.getPipeName(), + pipeInsertNodeTabletInsertionEvent.getCreationTime(), + socket, + bytes)) { final String errorMessage = String.format( "Transfer PipeInsertNodeTabletInsertionEvent %s error. Socket: %s", @@ -208,6 +212,7 @@ private void doTransfer( throws PipeException, IOException { if (!send( pipeRawTabletInsertionEvent.getPipeName(), + pipeRawTabletInsertionEvent.getCreationTime(), socket, PipeTransferTabletRawReq.toTPipeTransferBytes( pipeRawTabletInsertionEvent.convertToTablet(), @@ -244,17 +249,19 @@ private void doTransfer( final AirGapSocket socket, final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) throws PipeException, IOException { final String pipeName = pipeTsFileInsertionEvent.getPipeName(); + final long creationTime = pipeTsFileInsertionEvent.getCreationTime(); final File tsFile = pipeTsFileInsertionEvent.getTsFile(); final String errorMessage = String.format("Seal file %s error. Socket %s.", tsFile, socket); // 1. Transfer file piece by piece, and mod if needed if (pipeTsFileInsertionEvent.isWithMod() && supportModsIfIsDataNodeReceiver) { final File modFile = pipeTsFileInsertionEvent.getModFile(); - transferFilePieces(pipeName, modFile, socket, true); - transferFilePieces(pipeName, tsFile, socket, true); + transferFilePieces(pipeName, creationTime, modFile, socket, true); + transferFilePieces(pipeName, creationTime, tsFile, socket, true); // 2. Transfer file seal signal with mod, which means the file is transferred completely if (!send( pipeName, + creationTime, socket, PipeTransferTsFileSealWithModReq.toTPipeTransferBytes( modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length()))) { @@ -267,10 +274,11 @@ private void doTransfer( LOGGER.info("Successfully transferred file {}.", tsFile); } } else { - transferFilePieces(pipeName, tsFile, socket, false); + transferFilePieces(pipeName, creationTime, tsFile, socket, false); // 2. Transfer file seal signal without mod, which means the file is transferred completely if (!send( pipeName, + creationTime, socket, PipeTransferTsFileSealReq.toTPipeTransferBytes(tsFile.getName(), tsFile.length()))) { receiverStatusHandler.handle( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java index e3673c0330d93..526773bf81a98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java @@ -37,7 +37,7 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTabletBatchEventHandler; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTabletInsertNodeEventHandler; import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.handler.PipeConsensusTsFileInsertionEventHandler; @@ -573,7 +573,7 @@ public int getRetryBufferSize() { @Override public long getConsensusPipeCommitProgress() { - long creationTime = PipeAgent.task().getPipeCreationTime(consensusPipeName); + long creationTime = PipeDataNodeAgent.task().getPipeCreationTime(consensusPipeName); String committerKey = String.format("%s_%s_%s", consensusPipeName, consensusGroupId, creationTime); return PipeEventCommitManager.getInstance().getGivenConsensusPipeCommitId(committerKey); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java index 038da9781538f..adc508e47e883 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java @@ -51,7 +51,7 @@ public class PipeTransferTabletBatchEventHandler implements AsyncMethodCallback< private final List requestCommitIds; private final List events; - private final Map pipeName2BytesAccumulated; + private final Map, Long> pipeName2BytesAccumulated; private final TPipeTransferReq req; private final double reqCompressionRatio; @@ -78,9 +78,12 @@ public PipeTransferTabletBatchEventHandler( } public void transfer(final AsyncPipeDataTransferServiceClient client) throws TException { - for (final Map.Entry entry : pipeName2BytesAccumulated.entrySet()) { + for (final Map.Entry, Long> entry : pipeName2BytesAccumulated.entrySet()) { connector.rateLimitIfNeeded( - entry.getKey(), client.getEndPoint(), (long) (entry.getValue() * reqCompressionRatio)); + entry.getKey().getLeft(), + entry.getKey().getRight(), + client.getEndPoint(), + (long) (entry.getValue() * reqCompressionRatio)); } client.pipeTransfer(req, this); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java index 2e54a8c7c0bee..40561121724bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java @@ -46,26 +46,32 @@ public abstract class PipeTransferTabletInsertionEventHandler endPointAndBatch) { + private void doTransfer(final Pair endPointAndBatch) { final Pair clientAndStatus = clientManager.getClient(endPointAndBatch.getLeft()); final PipeEventBatch batchToTransfer = endPointAndBatch.getRight(); @@ -186,10 +186,11 @@ private void doTransfer(Pair endPointAndBatch) { final double compressionRatio = (double) compressedSize / uncompressedSize; - for (final Map.Entry entry : - batchToTransfer.getPipeName2BytesAccumulated().entrySet()) { + for (final Map.Entry, Long> entry : + batchToTransfer.getPipe2BytesAccumulated().entrySet()) { rateLimitIfNeeded( - entry.getKey(), + entry.getKey().getLeft(), + entry.getKey().getRight(), clientAndStatus.getLeft().getEndPoint(), (long) (entry.getValue() * compressionRatio)); } @@ -262,6 +263,7 @@ private void doTransfer( pipeInsertNodeTabletInsertionEvent.getByteBuffer())); rateLimitIfNeeded( pipeInsertNodeTabletInsertionEvent.getPipeName(), + pipeInsertNodeTabletInsertionEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); @@ -323,6 +325,7 @@ private void doTransfer(final PipeRawTabletInsertionEvent pipeRawTabletInsertion pipeRawTabletInsertionEvent.isAligned())); rateLimitIfNeeded( pipeRawTabletInsertionEvent.getPipeName(), + pipeRawTabletInsertionEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); @@ -370,6 +373,7 @@ private void doTransferWrapper(final PipeTsFileInsertionEvent pipeTsFileInsertio private void doTransfer(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) throws PipeException, IOException { final String pipeName = pipeTsFileInsertionEvent.getPipeName(); + final long creationTime = pipeTsFileInsertionEvent.getCreationTime(); final File tsFile = pipeTsFileInsertionEvent.getTsFile(); final File modFile = pipeTsFileInsertionEvent.getModFile(); final Pair clientAndStatus = clientManager.getClient(); @@ -377,8 +381,8 @@ private void doTransfer(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) // 1. Transfer tsFile, and mod file if exists and receiver's version >= 2 if (pipeTsFileInsertionEvent.isWithMod() && clientManager.supportModsIfIsDataNodeReceiver()) { - transferFilePieces(pipeName, modFile, clientAndStatus, true); - transferFilePieces(pipeName, tsFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, modFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, tsFile, clientAndStatus, true); // 2. Transfer file seal signal with mod, which means the file is transferred completely try { final TPipeTransferReq req = @@ -387,6 +391,7 @@ private void doTransfer(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length())); rateLimitIfNeeded( pipeTsFileInsertionEvent.getPipeName(), + pipeTsFileInsertionEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); @@ -398,7 +403,7 @@ private void doTransfer(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) e); } } else { - transferFilePieces(pipeName, tsFile, clientAndStatus, false); + transferFilePieces(pipeName, creationTime, tsFile, clientAndStatus, false); // 2. Transfer file seal signal without mod, which means the file is transferred completely try { final TPipeTransferReq req = @@ -406,6 +411,7 @@ private void doTransfer(final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) PipeTransferTsFileSealReq.toTPipeTransferReq(tsFile.getName(), tsFile.length())); rateLimitIfNeeded( pipeTsFileInsertionEvent.getPipeName(), + pipeTsFileInsertionEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java index 210bbec3dc0d1..93ec5e4ea439b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java @@ -89,15 +89,16 @@ private void doTransferWrapper(final PipeSchemaRegionSnapshotEvent pipeSchemaReg private void doTransfer(final PipeSchemaRegionSnapshotEvent snapshotEvent) throws PipeException, IOException { final String pipeName = snapshotEvent.getPipeName(); + final long creationTime = snapshotEvent.getCreationTime(); final File mTreeSnapshotFile = snapshotEvent.getMTreeSnapshotFile(); final File tagLogSnapshotFile = snapshotEvent.getTagLogSnapshotFile(); final Pair clientAndStatus = clientManager.getClient(); final TPipeTransferResp resp; // 1. Transfer mTreeSnapshotFile, and tLog file if exists - transferFilePieces(pipeName, mTreeSnapshotFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, mTreeSnapshotFile, clientAndStatus, true); if (Objects.nonNull(tagLogSnapshotFile)) { - transferFilePieces(pipeName, tagLogSnapshotFile, clientAndStatus, true); + transferFilePieces(pipeName, creationTime, tagLogSnapshotFile, clientAndStatus, true); } // 2. Transfer file seal signal, which means the snapshots are transferred completely try { @@ -114,6 +115,7 @@ private void doTransfer(final PipeSchemaRegionSnapshotEvent snapshotEvent) snapshotEvent.toSealTypeString())); rateLimitIfNeeded( snapshotEvent.getPipeName(), + snapshotEvent.getCreationTime(), clientAndStatus.getLeft().getEndPoint(), req.getBody().length); resp = clientAndStatus.getLeft().pipeTransfer(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java index ce49050943ffa..8a575922fa776 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBinaryReq; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletInsertNodeReq; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq; @@ -132,7 +132,7 @@ private void doTransfer( pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible(); if (Objects.isNull(insertNode)) { status = - PipeAgent.receiver() + PipeDataNodeAgent.receiver() .thrift() .receive( PipeTransferTabletBinaryReq.toTPipeTransferReq( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeDispatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeDispatcher.java index 82a6231cdd16a..9a95d8337886b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeDispatcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeDispatcher.java @@ -25,7 +25,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeDispatcher; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -116,6 +116,6 @@ public void dropPipe(ConsensusPipeName pipeName) throws Exception { throw new PipeException("Failed to drop consensus pipe", e); } // Release corresponding receiver's resource - PipeAgent.receiver().pipeConsensus().handleDropPipeConsensusTask(pipeName); + PipeDataNodeAgent.receiver().pipeConsensus().handleDropPipeConsensusTask(pipeName); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeRuntimeAgentGuardian.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeRuntimeAgentGuardian.java index 096cfdc6985e1..2f3cf532fc33a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeRuntimeAgentGuardian.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ConsensusPipeDataNodeRuntimeAgentGuardian.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.consensus; import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeGuardian; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ public synchronized void start(String id, Runnable guardJob, long intervalInSeco "Registering periodical job {} with interval in seconds {}.", id, intervalInSeconds); this.registered = true; - PipeAgent.runtime().registerPeriodicalJob(id, guardJob, intervalInSeconds); + PipeDataNodeAgent.runtime().registerPeriodicalJob(id, guardJob, intervalInSeconds); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ProgressIndexDataNodeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ProgressIndexDataNodeManager.java index 3f7cf9239db69..371349560b47f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ProgressIndexDataNodeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/ProgressIndexDataNodeManager.java @@ -29,7 +29,7 @@ import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName; import org.apache.iotdb.consensus.pipe.consensuspipe.ProgressIndexManager; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -108,7 +108,7 @@ private ProgressIndex extractLocalSimpleProgressIndex(ProgressIndex progressInde @Override public ProgressIndex getProgressIndex(ConsensusPipeName consensusPipeName) { - return PipeAgent.task() + return PipeDataNodeAgent.task() .getPipeTaskProgressIndex( consensusPipeName.toString(), consensusPipeName.getConsensusGroupId().getId()); } @@ -118,7 +118,7 @@ public ProgressIndex assignProgressIndex(ConsensusGroupId consensusGroupId) { return groupId2MaxProgressIndex .computeIfAbsent(consensusGroupId, o -> MinimumProgressIndex.INSTANCE) .updateToMinimumEqualOrIsAfterProgressIndex( - PipeAgent.runtime().assignProgressIndexForPipeConsensus()); + PipeDataNodeAgent.runtime().assignProgressIndexForPipeConsensus()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java index cc423f1bf3d65..a9c1d2fa6d4e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.pattern.PipePattern; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.task.PipeDataNodeTask; @@ -86,7 +86,7 @@ public boolean mayEventTimeOverlappedWithTimeRange() { @Override public void reportProgress() { - PipeAgent.task().markCompleted(pipeName, dataRegionId); + PipeDataNodeAgent.task().markCompleted(pipeName, dataRegionId); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java index c26db376da096..721703dc7a308 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java @@ -28,7 +28,7 @@ import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter; @@ -341,7 +341,8 @@ protected void extractDeletion(final PipeRealtimeEvent event) { + "has reached capacity, discard deletion event %s", this.getClass().getSimpleName(), this, event); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // Ignore the event. event.decreaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName(), false); @@ -379,7 +380,8 @@ protected Event supplyDeletion(final PipeRealtimeEvent event) { + "the data represented by this event is lost", event.getEvent()); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); return null; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java index a0f075298a59e..9854fa963a26b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -113,7 +113,7 @@ private void extractTabletInsertion(final PipeRealtimeEvent event) { + "has reached capacity, discard tablet event %s, current state %s", this, event, event.getTsFileEpoch().getState(this)); LOGGER.error(errorMessage); - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // Ignore the tablet event. @@ -184,7 +184,7 @@ private void extractTsFileInsertion(final PipeRealtimeEvent event) { + "has reached capacity, discard TsFile event %s, current state %s", this, event, event.getTsFileEpoch().getState(this)); LOGGER.error(errorMessage); - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // Ignore the tsfile event. @@ -360,7 +360,7 @@ private Event supplyTsFileInsertion(final PipeRealtimeEvent event) { + "the data represented by this event is lost", event.getEvent()); LOGGER.error(errorMessage); - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java index ed1d0f93a8286..40d8616a18135 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -70,7 +70,8 @@ private void extractTabletInsertion(PipeRealtimeEvent event) { + "has reached capacity, discard tablet event %s, current state %s", this, event, event.getTsFileEpoch().getState(this)); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // ignore this event. event.decreaseReferenceCount(PipeRealtimeDataRegionLogExtractor.class.getName(), false); @@ -101,7 +102,8 @@ private void extractTsFileInsertion(PipeRealtimeEvent event) { + "has reached capacity, discard loaded tsFile event %s, current state %s", this, event, event.getTsFileEpoch().getState(this)); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // ignore this event. event.decreaseReferenceCount(PipeRealtimeDataRegionLogExtractor.class.getName(), false); @@ -144,7 +146,8 @@ public Event supply() { + "the data represented by this event is lost", realtimeEvent.getEvent()); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); } realtimeEvent.decreaseReferenceCount( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java index ad92796c79177..57e00fa51c1a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; @@ -64,7 +64,8 @@ protected void doExtract(PipeRealtimeEvent event) { + "has reached capacity, discard TsFile event %s, current state %s", this, event, event.getTsFileEpoch().getState(this)); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); // Ignore the event. event.decreaseReferenceCount(PipeRealtimeDataRegionTsFileExtractor.class.getName(), false); @@ -106,7 +107,8 @@ public Event supply() { + "the data represented by this event is lost", realtimeEvent.getEvent()); LOGGER.error(errorMessage); - PipeAgent.runtime().report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); + PipeDataNodeAgent.runtime() + .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage)); } realtimeEvent.decreaseReferenceCount( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java index 1b6282cac9844..3eb118701a78e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeDataRegionAssigner; @@ -149,7 +149,7 @@ public void listenToDeleteData(DeleteDataNode node) { /////////////////////////////// singleton /////////////////////////////// private PipeInsertionDataNodeListener() { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "PipeInsertionDataNodeListener#listenToHeartbeat(false)", () -> listenToHeartbeat(false), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java index 3682095999b39..209593979c410 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java @@ -29,7 +29,7 @@ import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.metric.PipeDataNodeRemainingEventAndTimeMetrics; @@ -81,14 +81,15 @@ public void customize( @Override public void start() throws Exception { // Delay the start process to schema region leader ready - if (!PipeAgent.runtime().isSchemaLeaderReady(schemaRegionId) + if (!PipeDataNodeAgent.runtime().isSchemaLeaderReady(schemaRegionId) || hasBeenStarted.get() || hasBeenClosed.get()) { return; } // Try open the queue if it is the first task - if (PipeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId) == 1) { + if (PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId) + == 1) { SchemaRegionConsensusImpl.getInstance() .write(schemaRegionId, new PipeOperateSchemaQueueNode(new PlanNodeId(""), true)); } @@ -113,7 +114,7 @@ protected void triggerSnapshot() { // This method will return events only after schema region leader gets ready @Override public synchronized EnrichedEvent supply() throws Exception { - return PipeAgent.runtime().isSchemaLeaderReady(schemaRegionId) ? super.supply() : null; + return PipeDataNodeAgent.runtime().isSchemaLeaderReady(schemaRegionId) ? super.supply() : null; } @Override @@ -133,7 +134,7 @@ protected Optional trimRealtimeEventByPipePattern( @Override protected AbstractPipeListeningQueue getListeningQueue() { - return PipeAgent.runtime().schemaListener(schemaRegionId); + return PipeDataNodeAgent.runtime().schemaListener(schemaRegionId); } @Override @@ -166,7 +167,7 @@ public synchronized void close() throws Exception { if (!listenedTypeSet.isEmpty()) { // The queue is not closed here, and is closed iff the PipeMetaKeeper // has no schema pipe after one successful sync - PipeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId); + PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId); } if (Objects.nonNull(taskID)) { PipeSchemaRegionExtractorMetrics.getInstance().deregister(taskID); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index 0aeae73da75b2..3e6ed81f7136a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -29,7 +29,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.agent.plugin.dataregion.PipeDataRegionPluginAgent; import org.apache.iotdb.db.pipe.event.common.row.PipeResetTabletRow; import org.apache.iotdb.db.pipe.event.common.row.PipeRow; @@ -250,7 +250,7 @@ public void customize( // Load the useful aggregators' and their corresponding intermediate results' computational // logic. final Set declaredIntermediateResultSet = new HashSet<>(); - final PipeDataRegionPluginAgent agent = PipeAgent.plugin().dataRegion(); + final PipeDataRegionPluginAgent agent = PipeDataNodeAgent.plugin().dataRegion(); for (final String pipePluginName : agent.getSubProcessorNamesWithSpecifiedParent(AbstractOperatorProcessor.class)) { // Children are allowed to validate and configure the computational logic diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java index 55ef5afbfe9de..afa4e2cd64a1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java @@ -27,7 +27,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.processor.twostage.exchange.payload.FetchCombineResultResponse; import org.apache.iotdb.db.pipe.processor.twostage.operator.Operator; import org.apache.iotdb.db.pipe.processor.twostage.state.State; @@ -147,7 +147,7 @@ private Map fetchExpectedRegionId2DataNodeIdMap() { } final Set pipeRelatedRegionIdSet = - new HashSet<>(PipeAgent.task().getPipeTaskRegionIdSet(pipeName, creationTime)); + new HashSet<>(PipeDataNodeAgent.task().getPipeTaskRegionIdSet(pipeName, creationTime)); pipeRelatedRegionIdSet.removeIf( regionId -> !ALL_REGION_ID_2_DATANODE_ID_MAP.containsKey(regionId)); if (LOGGER.isInfoEnabled()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java index c08cd23d0c8c4..1b8342af77e97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.processor.twostage.combiner; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.processor.twostage.exchange.payload.CombineRequest; import org.apache.iotdb.db.pipe.processor.twostage.exchange.payload.FetchCombineResultRequest; import org.apache.iotdb.db.pipe.processor.twostage.exchange.payload.FetchCombineResultResponse; @@ -146,7 +146,7 @@ private static String generatePipeId(String pipeName, long creationTime) { /////////////////////////////// Singleton /////////////////////////////// private PipeCombineHandlerManager() { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "CombineHandlerManager#fetchExpectedRegionIdSetAndCleanOutdatedCombiner", this::fetchExpectedRegionIdSetAndCleanOutdatedCombiner, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java index 4cd97d7cb37ac..9fd3bcd40927a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapELanguageConstant; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapOneByteResponse; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiverAgent; import org.apache.iotdb.pipe.api.exception.PipeConnectionException; import org.apache.iotdb.rpc.TSStatusCode; @@ -63,7 +63,7 @@ public IoTDBAirGapReceiver(final Socket socket, final long receiverId) { this.socket = socket; this.receiverId = receiverId; - agent = PipeAgent.receiver().thrift(); + agent = PipeDataNodeAgent.receiver().thrift(); } @Override @@ -90,7 +90,7 @@ public void runMayThrow() throws Throwable { e); throw e; } finally { - PipeAgent.receiver().thrift().handleClientExit(); + PipeDataNodeAgent.receiver().thrift().handleClientExit(); socket.close(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 55fe372ea8f35..da7065392786a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferPlanNodeReq; @@ -356,7 +356,7 @@ private String getConfigReceiverId() { configReceiverId.set( IoTDBDescriptor.getInstance().getConfig().getDataNodeId() + "_" - + PipeAgent.runtime().getRebootTimes() + + PipeDataNodeAgent.runtime().getRebootTimes() + "_" + CONFIG_RECEIVER_ID_GENERATOR.incrementAndGet()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index f611fbc12620d..68e377ecfd389 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; @@ -66,7 +66,7 @@ public class PipeMemoryManager { private final Set allocatedBlocks = new HashSet<>(); public PipeMemoryManager() { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "PipeMemoryManager#tryExpandAll()", this::tryExpandAll, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index 5e75e068fa271..1cf6732786e7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.utils.FileUtils; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.agent.runtime.PipePeriodicalJobExecutor; import org.apache.iotdb.db.pipe.resource.PipeResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; @@ -52,7 +52,7 @@ public class PipeTsFileResourceManager { private final ReentrantLock lock = new ReentrantLock(); public PipeTsFileResourceManager() { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "PipeTsFileResourceManager#ttlCheck()", this::tryTtlCheck, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java index 7e0adbacd3208..06b5d68ef96af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.resource.wal; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.PipeResourceManager; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler; @@ -54,7 +54,7 @@ protected PipeWALResourceManager() { memtableIdSegmentLocks[i] = new ReentrantLock(); } - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "PipeWALResourceManager#ttlCheck()", this::ttlCheck, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java index fd673fe9b95e0..ddc7ee5a2d339 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.pipe.pattern.IoTDBPipePattern; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; @@ -160,7 +160,7 @@ private void collectEvent(final Event event) { .enrichWithCommitterKeyAndCommitId((EnrichedEvent) event, creationTime, regionId); // Assign a rebootTime for pipeConsensus - ((EnrichedEvent) event).setRebootTimes(PipeAgent.runtime().getRebootTimes()); + ((EnrichedEvent) event).setRebootTimes(PipeDataNodeAgent.runtime().getRebootTimes()); } if (event instanceof PipeHeartbeatEvent) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskExtractorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskExtractorStage.java index 06206b03bffde..32b8ca8f77543 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskExtractorStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskExtractorStage.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.pipe.task.EventSupplier; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.task.stage.PipeTaskStage; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.pipe.api.PipeExtractor; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; @@ -49,8 +49,8 @@ public PipeTaskExtractorStage( PipeTaskMeta pipeTaskMeta) { pipeExtractor = StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId)) - ? PipeAgent.plugin().dataRegion().reflectExtractor(extractorParameters) - : PipeAgent.plugin().schemaRegion().reflectExtractor(extractorParameters); + ? PipeDataNodeAgent.plugin().dataRegion().reflectExtractor(extractorParameters) + : PipeDataNodeAgent.plugin().schemaRegion().reflectExtractor(extractorParameters); // Validate and customize should be called before createSubtask. this allows extractor exposing // exceptions in advance. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskProcessorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskProcessorStage.java index c7ceaf1f97845..37d3e14ddc542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskProcessorStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/stage/PipeTaskProcessorStage.java @@ -28,7 +28,7 @@ import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.task.stage.PipeTaskStage; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.execution.PipeProcessorSubtaskExecutor; import org.apache.iotdb.db.pipe.task.connection.PipeEventCollector; import org.apache.iotdb.db.pipe.task.subtask.processor.PipeProcessorSubtask; @@ -74,7 +74,7 @@ public PipeTaskProcessorStage( pipeName, creationTime, regionId, pipeTaskMeta)); final PipeProcessor pipeProcessor = StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId)) - ? PipeAgent.plugin() + ? PipeDataNodeAgent.plugin() .dataRegion() .getConfiguredProcessor( pipeProcessorParameters.getStringOrDefault( @@ -82,7 +82,7 @@ public PipeTaskProcessorStage( BuiltinPipePlugin.DO_NOTHING_PROCESSOR.getPipePluginName()), pipeProcessorParameters, runtimeConfiguration) - : PipeAgent.plugin() + : PipeDataNodeAgent.plugin() .schemaRegion() .getConfiguredProcessor( pipeProcessorParameters.getStringOrDefault( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java index c2f7adaf8dd4f..56197b4f0f9a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.task.subtask.PipeAbstractConnectorSubtask; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector; import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; @@ -286,6 +286,6 @@ protected String getRootCause(final Throwable throwable) { @Override protected void report(final EnrichedEvent event, final PipeRuntimeException exception) { - PipeAgent.runtime().report(event, exception); + PipeDataNodeAgent.runtime().report(event, exception); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtaskManager.java index d85f04b68eafc..f8809b6b689a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtaskManager.java @@ -27,7 +27,7 @@ import org.apache.iotdb.commons.pipe.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.execution.PipeConnectorSubtaskExecutor; import org.apache.iotdb.db.pipe.metric.PipeDataNodeRemainingEventAndTimeMetrics; import org.apache.iotdb.db.pipe.metric.PipeDataRegionEventCounter; @@ -119,8 +119,10 @@ public synchronized String register( for (int connectorIndex = 0; connectorIndex < connectorNum; connectorIndex++) { final PipeConnector pipeConnector = isDataRegionConnector - ? PipeAgent.plugin().dataRegion().reflectConnector(pipeConnectorParameters) - : PipeAgent.plugin().schemaRegion().reflectConnector(pipeConnectorParameters); + ? PipeDataNodeAgent.plugin().dataRegion().reflectConnector(pipeConnectorParameters) + : PipeDataNodeAgent.plugin() + .schemaRegion() + .reflectConnector(pipeConnectorParameters); // 1. Construct, validate and customize PipeConnector, and then handshake (create // connection) with the target try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java index 32f37e208182e..ea53bab48eb06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/processor/PipeProcessorSubtask.java @@ -27,7 +27,7 @@ import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.task.EventSupplier; import org.apache.iotdb.commons.pipe.task.subtask.PipeReportableSubtask; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.metric.PipeProcessorMetrics; @@ -258,6 +258,6 @@ protected String getRootCause(final Throwable throwable) { @Override protected void report(final EnrichedEvent event, final PipeRuntimeException exception) { - PipeAgent.runtime().report(event, exception); + PipeDataNodeAgent.runtime().report(event, exception); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 78f7d03fd2ff6..c6025c658cff6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -48,7 +48,7 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; @@ -2630,7 +2630,7 @@ public TSStatus createTimeseriesUsingSchemaTemplate(TCreateTimeseriesUsingSchema @Override public TSStatus handshake(final TSyncIdentityInfo info) throws TException { - return PipeAgent.receiver() + return PipeDataNodeAgent.receiver() .legacy() .handshake( info, @@ -2641,18 +2641,18 @@ public TSStatus handshake(final TSyncIdentityInfo info) throws TException { @Override public TSStatus sendPipeData(final ByteBuffer buff) throws TException { - return PipeAgent.receiver().legacy().transportPipeData(buff); + return PipeDataNodeAgent.receiver().legacy().transportPipeData(buff); } @Override public TSStatus sendFile(final TSyncTransportMetaInfo metaInfo, final ByteBuffer buff) throws TException { - return PipeAgent.receiver().legacy().transportFile(metaInfo, buff); + return PipeDataNodeAgent.receiver().legacy().transportFile(metaInfo, buff); } @Override public TPipeTransferResp pipeTransfer(final TPipeTransferReq req) { - return PipeAgent.receiver().thrift().receive(req); + return PipeDataNodeAgent.receiver().thrift().receive(req); } @Override @@ -2802,8 +2802,8 @@ public void handleClientExit() { TSCloseSessionReq req = new TSCloseSessionReq(); closeSession(req); } - PipeAgent.receiver().thrift().handleClientExit(); - PipeAgent.receiver().legacy().handleClientExit(); + PipeDataNodeAgent.receiver().thrift().handleClientExit(); + PipeDataNodeAgent.receiver().legacy().handleClientExit(); SubscriptionAgent.receiver().handleClientExit(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 0db6eff2e7123..2c427a35ca381 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -74,7 +74,7 @@ import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.cn.DnToCnInternalServiceAsyncRequestManager; import org.apache.iotdb.db.protocol.client.cn.DnToCnRequestType; @@ -460,7 +460,7 @@ public TLoadResp sendLoadCommand(TLoadCommandReq req) { progressIndex = ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex())); } else { // fallback to use local generated progress index for compatibility - progressIndex = PipeAgent.runtime().getNextProgressIndexForTsFileLoad(); + progressIndex = PipeDataNodeAgent.runtime().getNextProgressIndexForTsFileLoad(); LOGGER.info( "Use local generated load progress index {} for uuid {}.", progressIndex, req.uuid); } @@ -1050,7 +1050,7 @@ public TPushPipeMetaResp pushPipeMeta(TPushPipeMetaReq req) { } try { List exceptionMessages = - PipeAgent.task().handlePipeMetaChanges(pipeMetas); + PipeDataNodeAgent.task().handlePipeMetaChanges(pipeMetas); return exceptionMessages.isEmpty() ? new TPushPipeMetaResp() @@ -1070,10 +1070,10 @@ public TPushPipeMetaResp pushSinglePipeMeta(TPushSinglePipeMetaReq req) { try { TPushPipeMetaRespExceptionMessage exceptionMessage; if (req.isSetPipeNameToDrop()) { - exceptionMessage = PipeAgent.task().handleDropPipe(req.getPipeNameToDrop()); + exceptionMessage = PipeDataNodeAgent.task().handleDropPipe(req.getPipeNameToDrop()); } else if (req.isSetPipeMeta()) { final PipeMeta pipeMeta = PipeMeta.deserialize(ByteBuffer.wrap(req.getPipeMeta())); - exceptionMessage = PipeAgent.task().handleSinglePipeMetaChanges(pipeMeta); + exceptionMessage = PipeDataNodeAgent.task().handleSinglePipeMetaChanges(pipeMeta); } else { throw new Exception("Invalid TPushSinglePipeMetaReq"); } @@ -1099,7 +1099,7 @@ public TPushPipeMetaResp pushMultiPipeMeta(TPushMultiPipeMetaReq req) { if (req.isSetPipeNamesToDrop()) { for (String pipeNameToDrop : req.getPipeNamesToDrop()) { TPushPipeMetaRespExceptionMessage message = - PipeAgent.task().handleDropPipe(pipeNameToDrop); + PipeDataNodeAgent.task().handleDropPipe(pipeNameToDrop); exceptionMessages.add(message); if (message != null) { // If there is any exception, skip the remaining pipes @@ -1111,7 +1111,7 @@ public TPushPipeMetaResp pushMultiPipeMeta(TPushMultiPipeMetaReq req) { for (ByteBuffer byteBuffer : req.getPipeMetas()) { final PipeMeta pipeMeta = PipeMeta.deserialize(byteBuffer); TPushPipeMetaRespExceptionMessage message = - PipeAgent.task().handleSinglePipeMetaChanges(pipeMeta); + PipeDataNodeAgent.task().handleSinglePipeMetaChanges(pipeMeta); exceptionMessages.add(message); if (message != null) { // If there is any exception, skip the remaining pipes @@ -1291,7 +1291,7 @@ public TPushConsumerGroupMetaResp pushSingleConsumerGroupMeta( @Override public TPipeHeartbeatResp pipeHeartbeat(TPipeHeartbeatReq req) throws TException { final TPipeHeartbeatResp resp = new TPipeHeartbeatResp(); - PipeAgent.task().collectPipeMetaList(req, resp); + PipeDataNodeAgent.task().collectPipeMetaList(req, resp); return resp; } @@ -1591,7 +1591,7 @@ public TDataNodeHeartbeatResp getDataNodeHeartBeat(TDataNodeHeartbeatReq req) th // Update pipe meta if necessary if (req.isNeedPipeMetaList()) { - PipeAgent.task().collectPipeMetaList(resp); + PipeDataNodeAgent.task().collectPipeMetaList(resp); } if (req.isSetConfigNodeEndPoints()) { @@ -1853,7 +1853,7 @@ public TSStatus setSystemStatus(String status) throws TException { try { commonConfig.setNodeStatus(NodeStatus.parse(status)); if (commonConfig.getNodeStatus().equals(NodeStatus.Removing)) { - PipeAgent.runtime().stop(); + PipeDataNodeAgent.runtime().stop(); } } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); @@ -2250,7 +2250,7 @@ private TEndPoint getConsensusEndPoint( public TSStatus createPipePlugin(TCreatePipePluginInstanceReq req) { try { PipePluginMeta pipePluginMeta = PipePluginMeta.deserialize(req.pipePluginMeta); - PipeAgent.plugin().register(pipePluginMeta, req.jarFile); + PipeDataNodeAgent.plugin().register(pipePluginMeta, req.jarFile); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } catch (Exception e) { return new TSStatus(TSStatusCode.CREATE_PIPE_PLUGIN_ON_DATANODE_ERROR.getStatusCode()) @@ -2261,7 +2261,7 @@ public TSStatus createPipePlugin(TCreatePipePluginInstanceReq req) { @Override public TSStatus dropPipePlugin(TDropPipePluginInstanceReq req) { try { - PipeAgent.plugin().deregister(req.getPipePluginName(), req.isNeedToDeleteJar()); + PipeDataNodeAgent.plugin().deregister(req.getPipePluginName(), req.isNeedToDeleteJar()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } catch (Exception e) { return new TSStatus(TSStatusCode.DROP_PIPE_PLUGIN_ON_DATANODE_ERROR.getStatusCode()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java index 22f624eb7df4b..18ddd6db478a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; import org.apache.iotdb.db.exception.DataRegionException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.rpc.RpcUtils; @@ -203,7 +203,7 @@ public TSStatus deleteDataRegion(DataRegionId dataRegionId) { public TSStatus deleteSchemaRegion(SchemaRegionId schemaRegionId) { try { schemaEngine.deleteSchemaRegion(schemaRegionId); - PipeAgent.runtime().schemaListener(schemaRegionId).close(); + PipeDataNodeAgent.runtime().schemaListener(schemaRegionId).close(); schemaRegionLockMap.remove(schemaRegionId); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } catch (MetadataException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java index 054c664b48efe..25d524154cdc7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java @@ -31,7 +31,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.exception.LoadFileException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -96,7 +96,7 @@ public LoadTsFileManager() { } private void registerCleanupTaskExecutor() { - PipeAgent.runtime() + PipeDataNodeAgent.runtime() .registerPeriodicalJob( "LoadTsFileManager#cleanupTasks", this::cleanupTasks, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 538337ffa1644..32754e99d16e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -115,7 +115,7 @@ import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.sql.SemanticException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -1728,7 +1728,7 @@ public SettableFuture createPipe(CreatePipeStatement createPip // Validate pipe plugin before creation try { - PipeAgent.plugin() + PipeDataNodeAgent.plugin() .validate( createPipeStatement.getPipeName(), createPipeStatement.getExtractorAttributes(), @@ -1786,11 +1786,12 @@ public SettableFuture alterPipe(AlterPipeStatement alterPipeSt try { if (!alterPipeStatement.getProcessorAttributes().isEmpty() && alterPipeStatement.isReplaceAllProcessorAttributes()) { - PipeAgent.plugin().validateProcessor(alterPipeStatement.getProcessorAttributes()); + PipeDataNodeAgent.plugin().validateProcessor(alterPipeStatement.getProcessorAttributes()); } if (!alterPipeStatement.getConnectorAttributes().isEmpty() && alterPipeStatement.isReplaceAllConnectorAttributes()) { - PipeAgent.plugin().validateConnector(pipeName, alterPipeStatement.getConnectorAttributes()); + PipeDataNodeAgent.plugin() + .validateConnector(pipeName, alterPipeStatement.getConnectorAttributes()); } } catch (Exception e) { LOGGER.info("Failed to validate alter pipe statement, because {}", e.getMessage(), e); @@ -1988,8 +1989,10 @@ public SettableFuture createTopic(CreateTopicStatement createT CommonDescriptor.getInstance().getConfig().getTimestampPrecision()), topicAttributes); try { - PipeAgent.plugin().validateExtractor(temporaryTopicMeta.generateExtractorAttributes()); - PipeAgent.plugin().validateProcessor(temporaryTopicMeta.generateProcessorAttributes()); + PipeDataNodeAgent.plugin() + .validateExtractor(temporaryTopicMeta.generateExtractorAttributes()); + PipeDataNodeAgent.plugin() + .validateProcessor(temporaryTopicMeta.generateProcessorAttributes()); } catch (Exception e) { LOGGER.info("Failed to validate create topic statement, because {}", e.getMessage(), e); future.setException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java index a630c94797b22..a0f6767813788 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -33,7 +33,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -166,7 +166,7 @@ public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDi } else if (planNode instanceof LoadSingleTsFileNode) { // do not need to split final TsFileResource tsFileResource = ((LoadSingleTsFileNode) planNode).getTsFileResource(); try { - PipeAgent.runtime().assignProgressIndexForTsFileLoad(tsFileResource); + PipeDataNodeAgent.runtime().assignProgressIndexForTsFileLoad(tsFileResource); tsFileResource.serialize(); StorageEngine.getInstance() @@ -257,7 +257,7 @@ private void dispatchLocally(TLoadCommandReq loadCommandReq) ProgressIndexType.deserializeFrom(ByteBuffer.wrap(loadCommandReq.getProgressIndex())); } else { // fallback to use local generated progress index for compatibility - progressIndex = PipeAgent.runtime().getNextProgressIndexForTsFileLoad(); + progressIndex = PipeDataNodeAgent.runtime().getNextProgressIndexForTsFileLoad(); LOGGER.info( "Use local generated load progress index {} for uuid {}.", progressIndex, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index d28681c161a8e..a36e75aed2fa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.exception.LoadFileException; import org.apache.iotdb.db.exception.LoadReadOnlyException; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.PlanFragmentId; import org.apache.iotdb.db.queryengine.execution.QueryStateMachine; @@ -368,7 +368,7 @@ private boolean secondPhase( } private ByteBuffer assignProgressIndex(TsFileResource tsFileResource) throws IOException { - PipeAgent.runtime().assignProgressIndexForTsFileLoad(tsFileResource); + PipeDataNodeAgent.runtime().assignProgressIndexForTsFileLoad(tsFileResource); try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 255349cab8971..874c016c59130 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -66,7 +66,7 @@ import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -653,7 +653,7 @@ private void setUp() throws StartupException { // Register subscription agent before pipe agent registerManager.register(SubscriptionAgent.runtime()); - registerManager.register(PipeAgent.runtime()); + registerManager.register(PipeDataNodeAgent.runtime()); } /** Set up RPC and protocols after DataNode is available */ @@ -953,7 +953,7 @@ private void getTriggerInformationList(List allTriggerInformation) { private void preparePipeResources() throws StartupException { long startTime = System.currentTimeMillis(); - PipeAgent.runtime().preparePipeResources(resourcesInformationHolder); + PipeDataNodeAgent.runtime().preparePipeResources(resourcesInformationHolder); long endTime = System.currentTimeMillis(); logger.info("Prepare pipe resources successfully, which takes {} ms.", (endTime - startTime)); } @@ -1039,7 +1039,7 @@ private void initProtocols() throws StartupException { registerManager.register(RestService.getInstance()); } if (PipeConfig.getInstance().getPipeAirGapReceiverEnabled()) { - registerManager.register(PipeAgent.receiver().airGap()); + registerManager.register(PipeDataNodeAgent.receiver().airGap()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 74aed59ab887e..5f92c30fe8015 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -35,7 +35,7 @@ import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet; @@ -300,7 +300,7 @@ public void insert(InsertRowNode insertRowNode, long[] costsForMetrics) startTime = System.nanoTime(); - PipeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowNode); + PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowNode); if (!insertRowNode.isGeneratedByPipe()) { workMemTable.markAsNotGeneratedByPipe(); } @@ -374,7 +374,7 @@ public void insert(InsertRowsNode insertRowsNode, long[] costsForMetrics) startTime = System.nanoTime(); - PipeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowsNode); + PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowsNode); if (!insertRowsNode.isGeneratedByPipe()) { workMemTable.markAsNotGeneratedByPipe(); } @@ -485,7 +485,7 @@ public void insertTablet( startTime = System.nanoTime(); - PipeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertTabletNode); + PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertTabletNode); if (!insertTabletNode.isGeneratedByPipe()) { workMemTable.markAsNotGeneratedByPipe(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java index a23d0fb23b8ae..0880697b02c22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.recover.file; import org.apache.iotdb.db.exception.DataRegionException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; @@ -120,7 +120,7 @@ protected void reconstructResourceFile() throws IOException { } // set progress index for pipe to avoid data loss - PipeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource); + PipeDataNodeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource); tsFileResource.serialize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 5c05f68d6d078..6bb9ef59898a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.DataRegionException; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -257,7 +257,7 @@ public void endRecovery() throws WALRecoverException { } // set recover progress index for pipe - PipeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource); + PipeDataNodeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource); // if we put following codes in the 'if' clause above, this file can be continued writing // into it diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index d93f64e2f1848..09e076653eacc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.SubscriptionEventBinaryCache; import org.apache.iotdb.pipe.api.event.Event; @@ -123,7 +123,7 @@ protected SubscriptionCommitContext generateSubscriptionCommitContext() { // leader transfers or restarts. return new SubscriptionCommitContext( IoTDBDescriptor.getInstance().getConfig().getDataNodeId(), - PipeAgent.runtime().getRebootTimes(), + PipeDataNodeAgent.runtime().getRebootTimes(), topicName, brokerId, subscriptionCommitIdGenerator.getAndIncrement()); @@ -132,7 +132,7 @@ protected SubscriptionCommitContext generateSubscriptionCommitContext() { private SubscriptionCommitContext generateInvalidSubscriptionCommitContext() { return new SubscriptionCommitContext( IoTDBDescriptor.getInstance().getConfig().getDataNodeId(), - PipeAgent.runtime().getRebootTimes(), + PipeDataNodeAgent.runtime().getRebootTimes(), topicName, brokerId, -1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java index 35a4a3ab4ac81..45d62174e4be8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java @@ -26,7 +26,7 @@ import org.apache.iotdb.commons.pipe.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.task.connection.UnboundedBlockingPendingQueue; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.execution.PipeConnectorSubtaskExecutor; import org.apache.iotdb.db.pipe.metric.PipeDataRegionEventCounter; import org.apache.iotdb.db.pipe.task.subtask.connector.PipeConnectorSubtask; @@ -102,7 +102,7 @@ public synchronized String register( : new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter()); final PipeConnector pipeConnector = - PipeAgent.plugin().dataRegion().reflectConnector(pipeConnectorParameters); + PipeDataNodeAgent.plugin().dataRegion().reflectConnector(pipeConnectorParameters); // 1. Construct, validate and customize PipeConnector, and then handshake (create connection) // with the target try { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/SchemaRegionListeningQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/SchemaRegionListeningQueueTest.java index 008204ab160ac..daafda17c5b81 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/SchemaRegionListeningQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/SchemaRegionListeningQueueTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.queue.ConcurrentIterableLinkedQueue; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -59,7 +59,7 @@ public static void setup() { @AfterClass public static void cleanup() throws IOException { - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).close(); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).close(); if (snapshotDir.exists()) { FileUtils.deleteDirectory(snapshotDir); } @@ -67,8 +67,8 @@ public static void cleanup() throws IOException { @Test public void testSnapshot() throws TException, IOException, AuthException, IllegalPathException { - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).open(); - PipeAgent.runtime().notifySchemaLeaderReady(new SchemaRegionId(0)); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).open(); + PipeDataNodeAgent.runtime().notifySchemaLeaderReady(new SchemaRegionId(0)); CreateTimeSeriesNode node1 = new CreateTimeSeriesNode( @@ -87,19 +87,19 @@ public void testSnapshot() throws TException, IOException, AuthException, Illega new ActivateTemplateNode( new PlanNodeId("ActivateTemplateNode"), new PartialPath("root.sg.d1.s1"), 2, 1)); - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).tryListenToNode(node1); - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).tryListenToNode(node2); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).tryListenToNode(node1); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).tryListenToNode(node2); // tryListenToSnapshots() cannot be tested here since we cannot operate the reference count of // the original or deserialized events - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).createSnapshot(snapshotDir); - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).close(); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).createSnapshot(snapshotDir); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).close(); - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).loadSnapshot(snapshotDir); - Assert.assertTrue(PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).isOpened()); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).loadSnapshot(snapshotDir); + Assert.assertTrue(PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).isOpened()); ConcurrentIterableLinkedQueue.DynamicIterator itr = - PipeAgent.runtime().schemaListener(new SchemaRegionId(0)).newIterator(0); + PipeDataNodeAgent.runtime().schemaListener(new SchemaRegionId(0)).newIterator(0); Event event1 = itr.next(0); Assert.assertEquals(node1, ((PipeSchemaRegionWritePlanEvent) event1).getPlanNode()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java index be5e2c3a63391..0281bb0351946 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.FileUtils; -import org.apache.iotdb.db.pipe.agent.PipeAgent; +import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResource; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; @@ -65,7 +65,7 @@ public class PipeTsFileResourceManagerTest { @Before public void setUp() throws Exception { pipeTsFileResourceManager = new PipeTsFileResourceManager(); - PipeAgent.runtime().startPeriodicalJobExecutor(); + PipeDataNodeAgent.runtime().startPeriodicalJobExecutor(); createTsfile(TS_FILE_NAME); creatModsFile(MODS_FILE_NAME); @@ -151,8 +151,8 @@ public void tearDown() throws Exception { FileUtils.deleteFileOrDirectory(pipeFolder); } - PipeAgent.runtime().stopPeriodicalJobExecutor(); - PipeAgent.runtime().clearPeriodicalJobExecutor(); + PipeDataNodeAgent.runtime().stopPeriodicalJobExecutor(); + PipeDataNodeAgent.runtime().clearPeriodicalJobExecutor(); } @Test diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index c21f853b8a722..6c2157a70c2db 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -202,6 +202,7 @@ public class CommonConfig { private int pipeAsyncConnectorMaxClientNumber = 16; private double pipeAllSinksRateLimitBytesPerSecond = -1; + private int pipeEndPointRateLimiterDropCheckIntervalMs = 1000; private boolean isSeperatedPipeHeartbeatEnabled = true; private int pipeHeartbeatIntervalSecondsForCollectingPipeMeta = 100; @@ -1050,6 +1051,15 @@ public void setPipeAllSinksRateLimitBytesPerSecond(double pipeAllSinksRateLimitB this.pipeAllSinksRateLimitBytesPerSecond = pipeAllSinksRateLimitBytesPerSecond; } + public int getPipeEndPointRateLimiterDropCheckIntervalMs() { + return pipeEndPointRateLimiterDropCheckIntervalMs; + } + + public void setPipeEndPointRateLimiterDropCheckIntervalMs( + int pipeEndPointRateLimiterDropCheckIntervalMs) { + this.pipeEndPointRateLimiterDropCheckIntervalMs = pipeEndPointRateLimiterDropCheckIntervalMs; + } + public long getTwoStageAggregateMaxCombinerLiveTimeInMs() { return twoStageAggregateMaxCombinerLiveTimeInMs; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 67718e1224c9d..b989e8e505552 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -392,6 +392,11 @@ private void loadPipeProps(Properties properties) { properties.getProperty( "pipe_all_sinks_rate_limit_bytes_per_second", String.valueOf(config.getPipeAllSinksRateLimitBytesPerSecond())))); + config.setPipeEndPointRateLimiterDropCheckIntervalMs( + Integer.parseInt( + properties.getProperty( + "pipe_end_point_rate_limiter_drop_check_interval_ms", + String.valueOf(config.getPipeEndPointRateLimiterDropCheckIntervalMs())))); config.setSeperatedPipeHeartbeatEnabled( Boolean.parseBoolean( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 63dd14c46c9a4..b79e8a41ce359 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeConnectorCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException; +import org.apache.iotdb.commons.pipe.connector.limiter.PipeEndPointRateLimiter; import org.apache.iotdb.commons.pipe.task.PipeTask; import org.apache.iotdb.commons.pipe.task.PipeTaskManager; import org.apache.iotdb.commons.pipe.task.meta.PipeMeta; @@ -77,6 +78,7 @@ public abstract class PipeTaskAgent { protected PipeTaskAgent() { pipeMetaKeeper = new PipeMetaKeeper(); pipeTaskManager = new PipeTaskManager(); + PipeEndPointRateLimiter.setTaskAgent(this); } ////////////////////////// PipeMeta Lock Control ////////////////////////// @@ -1017,4 +1019,10 @@ public void collectPipeMetaList(final TPipeHeartbeatReq req, final TPipeHeartbea protected abstract void collectPipeMetaListInternal( final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException; + + ///////////////////////// Pipe info getter ///////////////////////// + public long getPipeCreationTime(final String pipeName) { + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java index 5fa4aa8a812a3..818ef38dd84e3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java @@ -128,6 +128,10 @@ public double getPipeAllConnectorsRateLimitBytesPerSecond() { return COMMON_CONFIG.getPipeAllSinksRateLimitBytesPerSecond(); } + public int getPipeEndPointRateLimiterDropCheckIntervalMs() { + return COMMON_CONFIG.getPipeEndPointRateLimiterDropCheckIntervalMs(); + } + public float getPipeLeaderCacheMemoryUsagePercentage() { return COMMON_CONFIG.getPipeLeaderCacheMemoryUsagePercentage(); } @@ -344,6 +348,9 @@ public void printAllConfigs() { LOGGER.info( "PipeAllConnectorsRateLimitBytesPerSecond: {}", getPipeAllConnectorsRateLimitBytesPerSecond()); + LOGGER.info( + "PipeEndPointRateLimiterDropCheckIntervalMs: {}", + getPipeEndPointRateLimiterDropCheckIntervalMs()); LOGGER.info("SeperatedPipeHeartbeatEnabled: {}", isSeperatedPipeHeartbeatEnabled()); LOGGER.info( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java index 8b15164457667..86cbbf7dc0507 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java @@ -20,19 +20,28 @@ package org.apache.iotdb.commons.pipe.connector.limiter; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; +import org.apache.iotdb.commons.pipe.config.PipeConfig; import com.google.common.util.concurrent.RateLimiter; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; public class PipeEndPointRateLimiter { private final double bytesPerSecondLimit; + private final String pipeName; + private final long creationTime; + private static volatile PipeTaskAgent taskAgent; private final ConcurrentMap endPointRateLimiterMap; - public PipeEndPointRateLimiter(double bytesPerSecondLimit) { + public PipeEndPointRateLimiter( + final String pipeName, final long creationTime, final double bytesPerSecondLimit) { + this.pipeName = pipeName; + this.creationTime = creationTime; this.bytesPerSecondLimit = bytesPerSecondLimit; endPointRateLimiterMap = new ConcurrentHashMap<>(); } @@ -48,12 +57,30 @@ public void acquire(final TEndPoint endPoint, long bytes) { while (bytes > 0) { if (bytes > Integer.MAX_VALUE) { - rateLimiter.acquire(Integer.MAX_VALUE); + if (!tryAcquireWithPipeCheck(rateLimiter, Integer.MAX_VALUE)) { + return; + } bytes -= Integer.MAX_VALUE; } else { - rateLimiter.acquire((int) bytes); + tryAcquireWithPipeCheck(rateLimiter, (int) bytes); return; } } } + + private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes) { + while (!rateLimiter.tryAcquire( + bytes, + PipeConfig.getInstance().getPipeEndPointRateLimiterDropCheckIntervalMs(), + TimeUnit.MILLISECONDS)) { + if (taskAgent.getPipeCreationTime(pipeName) != creationTime) { + return false; + } + } + return true; + } + + public static void setTaskAgent(final PipeTaskAgent agent) { + taskAgent = agent; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java index 7aac1ffc90df1..5d8e87fbd8784 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java @@ -94,7 +94,8 @@ public String toString() { protected boolean supportModsIfIsDataNodeReceiver = true; @Override - public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) throws Exception { super.customize(parameters, configuration); @@ -157,7 +158,7 @@ public void handshake() throws Exception { if (sockets.get(i) != null) { try { sockets.set(i, null).close(); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Failed to close socket with target server ip: {}, port: {}, because: {}. Ignore it.", ip, @@ -173,7 +174,7 @@ public void handshake() throws Exception { socket.setKeepAlive(true); sockets.set(i, socket); LOGGER.info("Successfully connected to target server ip: {}, port: {}.", ip, port); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Failed to connect to target server ip: {}, port: {}, because: {}. Ignore it.", ip, @@ -195,7 +196,7 @@ public void handshake() throws Exception { String.format("All target servers %s are not available.", nodeUrls)); } - protected void sendHandshakeReq(AirGapSocket socket) throws IOException { + protected void sendHandshakeReq(final AirGapSocket socket) throws IOException { socket.setSoTimeout(handshakeTimeoutMs); // Try to handshake by PipeTransferHandshakeV2Req. If failed, retry to handshake by // PipeTransferHandshakeV1Req. If failed again, throw PipeConnectionException. @@ -219,7 +220,7 @@ protected void sendHandshakeReq(AirGapSocket socket) throws IOException { public void heartbeat() { try { handshake(); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Failed to reconnect to target server, because: {}. Try to reconnect later.", e.getMessage(), @@ -228,7 +229,11 @@ public void heartbeat() { } protected void transferFilePieces( - String pipeName, File file, AirGapSocket socket, boolean isMultiFile) + final String pipeName, + final long creationTime, + final File file, + final AirGapSocket socket, + final boolean isMultiFile) throws PipeException, IOException { final int readFileBufferSize = PipeConfig.getInstance().getPipeConnectorReadFileBufferSize(); final byte[] readBuffer = new byte[readFileBufferSize]; @@ -246,6 +251,7 @@ protected void transferFilePieces( : Arrays.copyOfRange(readBuffer, 0, readLength); if (!send( pipeName, + creationTime, socket, isMultiFile ? getTransferMultiFilePieceBytes(file.getName(), position, payload) @@ -272,23 +278,25 @@ protected void transferFilePieces( protected abstract boolean mayNeedHandshakeWhenFail(); protected abstract byte[] getTransferSingleFilePieceBytes( - String fileName, long position, byte[] payLoad) throws IOException; + final String fileName, final long position, final byte[] payLoad) throws IOException; protected abstract byte[] getTransferMultiFilePieceBytes( - String fileName, long position, byte[] payLoad) throws IOException; + final String fileName, final long position, final byte[] payLoad) throws IOException; protected int nextSocketIndex() { return loadBalancer.nextSocketIndex(); } - protected boolean send(String pipeName, AirGapSocket socket, byte[] bytes) throws IOException { + protected boolean send( + final String pipeName, final long creationTime, final AirGapSocket socket, byte[] bytes) + throws IOException { if (!socket.isConnected()) { return false; } bytes = compressIfNeeded(bytes); - rateLimitIfNeeded(pipeName, socket.getEndPoint(), bytes.length); + rateLimitIfNeeded(pipeName, creationTime, socket.getEndPoint(), bytes.length); final BufferedOutputStream outputStream = new BufferedOutputStream(socket.getOutputStream()); bytes = enrichWithLengthAndChecksum(bytes); @@ -300,11 +308,11 @@ protected boolean send(String pipeName, AirGapSocket socket, byte[] bytes) throw return size > 0 && Arrays.equals(AirGapOneByteResponse.OK, response); } - protected boolean send(AirGapSocket socket, byte[] bytes) throws IOException { - return send(null, socket, bytes); + protected boolean send(final AirGapSocket socket, final byte[] bytes) throws IOException { + return send(null, 0, socket, bytes); } - private byte[] enrichWithLengthAndChecksum(byte[] bytes) { + private byte[] enrichWithLengthAndChecksum(final byte[] bytes) { // Length of checksum and bytes payload final byte[] length = BytesUtils.intToBytes(bytes.length + LONG_LEN); @@ -316,7 +324,7 @@ private byte[] enrichWithLengthAndChecksum(byte[] bytes) { Arrays.asList(length, length, BytesUtils.longToBytes(crc32.getValue()), bytes)); } - private byte[] enrichWithELanguage(byte[] bytes) { + private byte[] enrichWithELanguage(final byte[] bytes) { return BytesUtils.concatByteArrayList( Arrays.asList( AirGapELanguageConstant.E_LANGUAGE_PREFIX, @@ -331,7 +339,7 @@ public void close() { if (sockets.get(i) != null) { sockets.set(i, null).close(); } - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn("Failed to close client {}.", i, e); } finally { isSocketAlive.set(i, false); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java index d81292e34d542..0f0719972b726 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java @@ -35,6 +35,7 @@ import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,8 +108,8 @@ public abstract class IoTDBConnector implements PipeConnector { private boolean isRpcCompressionEnabled; private final List compressors = new ArrayList<>(); - private static final Map PIPE_END_POINT_RATE_LIMITER_MAP = - new ConcurrentHashMap<>(); + private static final Map, PipeEndPointRateLimiter> + PIPE_END_POINT_RATE_LIMITER_MAP = new ConcurrentHashMap<>(); private double endPointRateLimitBytesPerSecond = -1; private static final GlobalRateLimiter GLOBAL_RATE_LIMITER = new GlobalRateLimiter(); @@ -383,11 +384,15 @@ public List getCompressors() { } public void rateLimitIfNeeded( - final String pipeName, final TEndPoint endPoint, final long bytesLength) { + final String pipeName, + final long creationTime, + final TEndPoint endPoint, + final long bytesLength) { if (pipeName != null && endPointRateLimitBytesPerSecond > 0) { PIPE_END_POINT_RATE_LIMITER_MAP .computeIfAbsent( - pipeName, endpoint -> new PipeEndPointRateLimiter(endPointRateLimitBytesPerSecond)) + new Pair<>(pipeName, creationTime), + endpoint -> getRateLimiter(pipeName, creationTime, endPointRateLimitBytesPerSecond)) .acquire(endPoint, bytesLength); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java index 713f25df29ef0..a51edf46cd4c3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java @@ -144,10 +144,11 @@ public void heartbeat() { } protected void transferFilePieces( - String pipeName, - File file, - Pair clientAndStatus, - boolean isMultiFile) + final String pipeName, + final long creationTime, + final File file, + final Pair clientAndStatus, + final boolean isMultiFile) throws PipeException, IOException { final int readFileBufferSize = PipeConfig.getInstance().getPipeConnectorReadFileBufferSize(); final byte[] readBuffer = new byte[readFileBufferSize]; @@ -171,7 +172,10 @@ protected void transferFilePieces( ? getTransferMultiFilePieceReq(file.getName(), position, payLoad) : getTransferSingleFilePieceReq(file.getName(), position, payLoad)); rateLimitIfNeeded( - pipeName, clientAndStatus.getLeft().getEndPoint(), req.getBody().length); + pipeName, + creationTime, + clientAndStatus.getLeft().getEndPoint(), + req.getBody().length); resp = PipeTransferFilePieceResp.fromTPipeTransferResp( clientAndStatus.getLeft().pipeTransfer(req)); From fe2f5bff4baa8d3e5bcaab81db153c6b4e427a04 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:26:22 +0800 Subject: [PATCH 3/7] Update IoTDBConnector.java --- .../iotdb/commons/pipe/connector/protocol/IoTDBConnector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java index 0f0719972b726..aa398d6e9ff9c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java @@ -392,7 +392,9 @@ public void rateLimitIfNeeded( PIPE_END_POINT_RATE_LIMITER_MAP .computeIfAbsent( new Pair<>(pipeName, creationTime), - endpoint -> getRateLimiter(pipeName, creationTime, endPointRateLimitBytesPerSecond)) + endpoint -> + new PipeEndPointRateLimiter( + pipeName, creationTime, endPointRateLimitBytesPerSecond)) .acquire(endPoint, bytesLength); } From 679114fdd4db87f2d8c8b52d836af437854e8a05 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:42:24 +0800 Subject: [PATCH 4/7] bug fix --- .../protocol/airgap/IoTDBDataNodeAirGapConnector.java | 1 + .../protocol/airgap/IoTDBSchemaRegionAirGapConnector.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataNodeAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataNodeAirGapConnector.java index f6e80a2891760..b66c0012700c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataNodeAirGapConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataNodeAirGapConnector.java @@ -127,6 +127,7 @@ private void doTransfer( throws PipeException, IOException { if (!send( pipeSchemaRegionWritePlanEvent.getPipeName(), + pipeSchemaRegionWritePlanEvent.getCreationTime(), socket, PipeTransferPlanNodeReq.toTPipeTransferBytes( pipeSchemaRegionWritePlanEvent.getPlanNode()))) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java index a97408266b7a9..e53423779d0db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java @@ -103,17 +103,19 @@ private void doTransfer( final AirGapSocket socket, final PipeSchemaRegionSnapshotEvent pipeSchemaRegionSnapshotEvent) throws PipeException, IOException { final String pipeName = pipeSchemaRegionSnapshotEvent.getPipeName(); + final long creationTime = pipeSchemaRegionSnapshotEvent.getCreationTime(); final File mtreeSnapshotFile = pipeSchemaRegionSnapshotEvent.getMTreeSnapshotFile(); final File tagLogSnapshotFile = pipeSchemaRegionSnapshotEvent.getTagLogSnapshotFile(); // 1. Transfer mTreeSnapshotFile, and tLog file if exists - transferFilePieces(pipeName, mtreeSnapshotFile, socket, true); + transferFilePieces(pipeName, creationTime, mtreeSnapshotFile, socket, true); if (Objects.nonNull(tagLogSnapshotFile)) { - transferFilePieces(pipeName, tagLogSnapshotFile, socket, true); + transferFilePieces(pipeName, creationTime, tagLogSnapshotFile, socket, true); } // 2. Transfer file seal signal, which means the snapshots is transferred completely if (!send( pipeName, + creationTime, socket, PipeTransferSchemaSnapshotSealReq.toTPipeTransferBytes( // The pattern is surely Non-null From d484881da848dc387ce10c2163386d544366a59c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:25:38 +0800 Subject: [PATCH 5/7] Changed global / tsfile load rate limiter --- .../execution/load/LoadTsFileRateLimiter.java | 44 +++++++++++++------ .../connector/limiter/GlobalRateLimiter.java | 43 ++++++++++++------ 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java index ce7046bf5ee54..d98a1d102fc62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java @@ -19,12 +19,15 @@ package org.apache.iotdb.db.queryengine.execution.load; +import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import com.google.common.util.concurrent.AtomicDouble; import com.google.common.util.concurrent.RateLimiter; +import java.util.concurrent.TimeUnit; + public class LoadTsFileRateLimiter { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); @@ -34,31 +37,46 @@ public class LoadTsFileRateLimiter { private final RateLimiter loadWriteRateLimiter; public void acquire(long bytes) { - final double throughputBytesPerSecondLimit = CONFIG.getLoadWriteThroughputBytesPerSecond(); - - if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) { - throughputBytesPerSecond.set(throughputBytesPerSecondLimit); - loadWriteRateLimiter.setRate( - // if throughput <= 0, disable rate limiting - throughputBytesPerSecondLimit <= 0 ? Double.MAX_VALUE : throughputBytesPerSecondLimit); - } - - // For performance, we don't need to acquire rate limiter if throughput <= 0 - if (throughputBytesPerSecondLimit <= 0) { + if (reloadParams()) { return; } while (bytes > 0) { if (bytes > Integer.MAX_VALUE) { - loadWriteRateLimiter.acquire(Integer.MAX_VALUE); + tryAcquireWithRateCheck(Integer.MAX_VALUE); bytes -= Integer.MAX_VALUE; } else { - loadWriteRateLimiter.acquire((int) bytes); + tryAcquireWithRateCheck((int) bytes); + return; + } + } + } + + private void tryAcquireWithRateCheck(final int bytes) { + while (!loadWriteRateLimiter.tryAcquire( + bytes, + PipeConfig.getInstance().getPipeEndPointRateLimiterDropCheckIntervalMs(), + TimeUnit.MILLISECONDS)) { + if (reloadParams()) { return; } } } + private boolean reloadParams() { + final double throughputBytesPerSecondLimit = CONFIG.getLoadWriteThroughputBytesPerSecond(); + + if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) { + throughputBytesPerSecond.set(throughputBytesPerSecondLimit); + loadWriteRateLimiter.setRate( + // if throughput <= 0, disable rate limiting + throughputBytesPerSecondLimit <= 0 ? Double.MAX_VALUE : throughputBytesPerSecondLimit); + } + + // For performance, we don't need to acquire rate limiter if throughput <= 0 + return throughputBytesPerSecondLimit <= 0; + } + //////////////////////////// Singleton //////////////////////////// private LoadTsFileRateLimiter() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java index 575731d04a5ec..fbabb60fa8d36 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java @@ -24,6 +24,8 @@ import com.google.common.util.concurrent.AtomicDouble; import com.google.common.util.concurrent.RateLimiter; +import java.util.concurrent.TimeUnit; + /** This is a global rate limiter for all connectors. */ public class GlobalRateLimiter { @@ -42,6 +44,33 @@ public GlobalRateLimiter() { } public void acquire(long bytes) { + if (reloadParams()) { + return; + } + + while (bytes > 0) { + if (bytes > Integer.MAX_VALUE) { + tryAcquireWithRateCheck(Integer.MAX_VALUE); + bytes -= Integer.MAX_VALUE; + } else { + tryAcquireWithRateCheck((int) bytes); + return; + } + } + } + + private void tryAcquireWithRateCheck(final int bytes) { + while (!rateLimiter.tryAcquire( + bytes, + PipeConfig.getInstance().getPipeEndPointRateLimiterDropCheckIntervalMs(), + TimeUnit.MILLISECONDS)) { + if (reloadParams()) { + return; + } + } + } + + private boolean reloadParams() { final double throughputBytesPerSecondLimit = CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond(); @@ -53,18 +82,6 @@ public void acquire(long bytes) { } // For performance, we don't need to acquire rate limiter if throughput <= 0 - if (throughputBytesPerSecondLimit <= 0) { - return; - } - - while (bytes > 0) { - if (bytes > Integer.MAX_VALUE) { - rateLimiter.acquire(Integer.MAX_VALUE); - bytes -= Integer.MAX_VALUE; - } else { - rateLimiter.acquire((int) bytes); - return; - } - } + return throughputBytesPerSecondLimit <= 0; } } From 7986eeb4ebcdc4c2957cf570786e9a9e73ebd4a1 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 20 Jun 2024 12:55:20 +0800 Subject: [PATCH 6/7] refactor --- .../commons/pipe/agent/task/PipeTaskAgent.java | 3 ++- .../limiter/PipeEndPointRateLimiter.java | 18 +++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index b79e8a41ce359..67a224694f2c8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -78,6 +78,8 @@ public abstract class PipeTaskAgent { protected PipeTaskAgent() { pipeMetaKeeper = new PipeMetaKeeper(); pipeTaskManager = new PipeTaskManager(); + + // Help PipeEndPointRateLimiter to check if the pipe is still alive PipeEndPointRateLimiter.setTaskAgent(this); } @@ -1020,7 +1022,6 @@ public void collectPipeMetaList(final TPipeHeartbeatReq req, final TPipeHeartbea protected abstract void collectPipeMetaListInternal( final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException; - ///////////////////////// Pipe info getter ///////////////////////// public long getPipeCreationTime(final String pipeName) { final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java index 86cbbf7dc0507..b7529a8b7d6b3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java @@ -25,16 +25,20 @@ import com.google.common.util.concurrent.RateLimiter; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; public class PipeEndPointRateLimiter { - private final double bytesPerSecondLimit; + // The task agent is used to check if the pipe is still alive + private static volatile PipeTaskAgent taskAgent; + private final String pipeName; private final long creationTime; - private static volatile PipeTaskAgent taskAgent; + + private final double bytesPerSecondLimit; private final ConcurrentMap endPointRateLimiterMap; @@ -46,6 +50,10 @@ public PipeEndPointRateLimiter( endPointRateLimiterMap = new ConcurrentHashMap<>(); } + public static void setTaskAgent(final PipeTaskAgent taskAgent) { + PipeEndPointRateLimiter.taskAgent = taskAgent; + } + public void acquire(final TEndPoint endPoint, long bytes) { if (endPoint == null) { return; @@ -73,14 +81,10 @@ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes, PipeConfig.getInstance().getPipeEndPointRateLimiterDropCheckIntervalMs(), TimeUnit.MILLISECONDS)) { - if (taskAgent.getPipeCreationTime(pipeName) != creationTime) { + if (Objects.nonNull(taskAgent) && taskAgent.getPipeCreationTime(pipeName) != creationTime) { return false; } } return true; } - - public static void setTaskAgent(final PipeTaskAgent agent) { - taskAgent = agent; - } } From 3c98081d71f4168b245ef6bc9ef887d0d92e347a Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 20 Jun 2024 12:57:19 +0800 Subject: [PATCH 7/7] Update PipeEndPointRateLimiter.java --- .../pipe/connector/limiter/PipeEndPointRateLimiter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java index b7529a8b7d6b3..46f16a8eb11c1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java @@ -81,7 +81,9 @@ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int bytes, PipeConfig.getInstance().getPipeEndPointRateLimiterDropCheckIntervalMs(), TimeUnit.MILLISECONDS)) { - if (Objects.nonNull(taskAgent) && taskAgent.getPipeCreationTime(pipeName) != creationTime) { + final PipeTaskAgent finalTaskAgent = taskAgent; + if (Objects.nonNull(finalTaskAgent) + && finalTaskAgent.getPipeCreationTime(pipeName) != creationTime) { return false; } }