From 24ba808ff9db89b80c678d19d7d666f3385724f2 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 13 Jun 2024 22:26:27 +0800 Subject: [PATCH 1/5] fix wal deletion & create receiver dir --- .../protocol/pipeconsensus/PipeConsensusReceiver.java | 9 ++++++++- .../apache/iotdb/db/storageengine/StorageEngine.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 60a0b88beb902..bd63eac63e089 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -838,7 +838,14 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, } // Create a new receiver file dir final File newReceiverDir = new File(receiverFileBaseDir, consensusGroupId.toString()); - if (!newReceiverDir.exists() && !newReceiverDir.mkdirs()) { + if (newReceiverDir.exists()) { + FileUtils.deleteDirectory(newReceiverDir); + LOGGER.info( + "PipeConsensus-ConsensusGroupId-{}: Origin receiver file dir {} was deleted.", + consensusGroupId, + newReceiverDir.getPath()); + } + if (!newReceiverDir.mkdirs()) { LOGGER.warn( "PipeConsensus-ConsensusGroupId-{}: Failed to create receiver file dir {}.", newReceiverDir.getPath(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index e0455b3329ff8..54c8f9c43b096 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -690,6 +690,16 @@ public void deleteDataRegion(DataRegionId regionId) { } } } + } else if (CONFIG + .getDataRegionConsensusProtocolClass() + .equals(ConsensusFactory.FAST_IOT_CONSENSUS) + || CONFIG + .getDataRegionConsensusProtocolClass() + .equals(ConsensusFactory.IOTV2_CONSENSUS)) { + // delete wal + WALManager.getInstance() + .deleteWALNode( + region.getDatabaseName() + FILE_NAME_SEPARATOR + region.getDataRegionId()); } } catch (Exception e) { LOGGER.error( From 3982eeb7ed18f6fdb064cc9f6840a0ddc4a39618 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 13 Jun 2024 22:38:20 +0800 Subject: [PATCH 2/5] update consensus protocol class --- .../java/org/apache/iotdb/db/storageengine/StorageEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 2628793985d21..8747b0d375d46 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -737,7 +737,7 @@ public void deleteDataRegion(DataRegionId regionId) { .equals(ConsensusFactory.FAST_IOT_CONSENSUS) || CONFIG .getDataRegionConsensusProtocolClass() - .equals(ConsensusFactory.IOTV2_CONSENSUS)) { + .equals(ConsensusFactory.IOT_CONSENSUS_V2)) { // delete wal WALManager.getInstance() .deleteWALNode( From fd291f5055496685c2b61981683f5b437e92b0a5 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 13 Jun 2024 22:51:41 +0800 Subject: [PATCH 3/5] merge judge --- .../iotdb/db/storageengine/StorageEngine.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 8747b0d375d46..e30069f70f001 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -713,7 +713,13 @@ public void deleteDataRegion(DataRegionId regionId) { region.abortCompaction(); region.syncDeleteDataFiles(); region.deleteFolder(systemDir); - if (CONFIG.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)) { + if (CONFIG.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS) + || CONFIG + .getDataRegionConsensusProtocolClass() + .equals(ConsensusFactory.FAST_IOT_CONSENSUS) + || CONFIG + .getDataRegionConsensusProtocolClass() + .equals(ConsensusFactory.IOT_CONSENSUS_V2)) { // delete wal WALManager.getInstance() .deleteWALNode( @@ -732,16 +738,6 @@ public void deleteDataRegion(DataRegionId regionId) { } } } - } else if (CONFIG - .getDataRegionConsensusProtocolClass() - .equals(ConsensusFactory.FAST_IOT_CONSENSUS) - || CONFIG - .getDataRegionConsensusProtocolClass() - .equals(ConsensusFactory.IOT_CONSENSUS_V2)) { - // delete wal - WALManager.getInstance() - .deleteWALNode( - region.getDatabaseName() + FILE_NAME_SEPARATOR + region.getDataRegionId()); } } catch (Exception e) { LOGGER.error( From abf1eeb1da0e1679b800cda62c27e7d6bba40868 Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 13 Jun 2024 23:37:40 +0800 Subject: [PATCH 4/5] modify directory path --- .../protocol/pipeconsensus/PipeConsensusReceiver.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index bd63eac63e089..85c418757fd87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -93,6 +93,7 @@ public class PipeConsensusReceiver { private final PipeConsensus pipeConsensus; private final ConsensusGroupId consensusGroupId; // Used to buffer TsFile when transfer TsFile asynchronously. + private final ConsensusPipeName consensusPipeName; private final List receiverBaseDirsName; private final PipeConsensusTsFileWriterPool pipeConsensusTsFileWriterPool = new PipeConsensusTsFileWriterPool(); @@ -105,12 +106,12 @@ public PipeConsensusReceiver( ConsensusPipeName consensusPipeName) { this.pipeConsensus = pipeConsensus; this.consensusGroupId = consensusGroupId; + this.consensusPipeName = consensusPipeName; // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is - // data/datanode/system/pipe/consensus/receiver/__consensus{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} + // data/datanode/system/pipe/consensus/receiver/__consensus.{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} receiverBaseDirsName = Arrays.stream(IoTDBDescriptor.getInstance().getConfig().getPipeConsensusReceiverFileDirs()) - .map(s -> s + File.separator + consensusPipeName) .collect(Collectors.toList()); try { @@ -837,7 +838,7 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, throw new DiskSpaceInsufficientException(receiverBaseDirsName); } // Create a new receiver file dir - final File newReceiverDir = new File(receiverFileBaseDir, consensusGroupId.toString()); + final File newReceiverDir = new File(receiverFileBaseDir, consensusPipeName.toString()); if (newReceiverDir.exists()) { FileUtils.deleteDirectory(newReceiverDir); LOGGER.info( From 6f24268f7944a90c5692254f88015fc867801d7f Mon Sep 17 00:00:00 2001 From: yschengzi Date: Thu, 13 Jun 2024 23:44:43 +0800 Subject: [PATCH 5/5] modify directory path --- .../receiver/protocol/pipeconsensus/PipeConsensusReceiver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 85c418757fd87..5d9eb0947c1e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -111,8 +111,7 @@ public PipeConsensusReceiver( // Each pipeConsensusReceiver has its own base directories. for example, a default dir path is // data/datanode/system/pipe/consensus/receiver/__consensus.{consensusGroupId}_{leaderDataNodeId}_{followerDataNodeId} receiverBaseDirsName = - Arrays.stream(IoTDBDescriptor.getInstance().getConfig().getPipeConsensusReceiverFileDirs()) - .collect(Collectors.toList()); + Arrays.asList(IoTDBDescriptor.getInstance().getConfig().getPipeConsensusReceiverFileDirs()); try { this.folderManager =