From b621eba502b8c84dd8a64a7eb038b72256c7446d Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 17 Jun 2024 17:44:06 +0800 Subject: [PATCH 1/2] file fileOverlap is not correctly marked --- .../CompactionValidationFailedException.java | 14 ++++++++------ .../execute/task/AbstractCompactionTask.java | 17 +++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java index 86b9eb4219c4a..b60dd6845fe37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java @@ -21,22 +21,24 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import java.util.List; + public class CompactionValidationFailedException extends RuntimeException { - private TsFileResource overlappedTsFileResource = null; + private List overlappedTsFileResources = null; public CompactionValidationFailedException(String msg) { super(msg); } - public CompactionValidationFailedException(TsFileResource overlappedTsFileResource) { + public CompactionValidationFailedException(List overlappedTsFileResources) { super( "Failed to pass compaction validation, sequence files has overlap, file is " - + overlappedTsFileResource); - this.overlappedTsFileResource = overlappedTsFileResource; + + overlappedTsFileResources); + this.overlappedTsFileResources = overlappedTsFileResources; } - public TsFileResource getOverlappedTsFileResource() { - return overlappedTsFileResource; + public List getOverlappedTsFileResources() { + return overlappedTsFileResources; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java index 0a1152044aa8a..75556001fd625 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionTaskStage; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.TsFileIdentifier; +import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairDataFileScanUtil; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; @@ -149,10 +150,10 @@ protected void handleException(Logger logger, Exception e) { } else { CompactionValidationFailedException validationException = (CompactionValidationFailedException) e; - TsFileResource overlappedTsFileResource = validationException.getOverlappedTsFileResource(); - if (overlappedTsFileResource != null) { - unsortedTsFileResources.add(overlappedTsFileResource); - } + List overlappedTsFileResource = + validationException.getOverlappedTsFileResources(); + unsortedTsFileResources = + overlappedTsFileResource == null ? unsortedTsFileResources : overlappedTsFileResource; } // these exceptions generally caused by unsorted data, mark all source files as NEED_TO_REPAIR for (TsFileResource resource : unsortedTsFileResources) { @@ -472,15 +473,15 @@ protected void validateCompactionResult( Long.parseLong(f2.getTsFile().getName().split("-")[1])) : timeDiff; }); - if (!validator.validateTsFilesIsHasNoOverlap(timePartitionSeqFiles)) { + List overlapFilesInTimePartition = + RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles); + if (!overlapFilesInTimePartition.isEmpty()) { LOGGER.error( "Failed to pass compaction validation, source seq files: {}, source unseq files: {}, target files: {}", sourceSeqFiles, sourceUnseqFiles, targetFiles); - throw new CompactionValidationFailedException( - "Failed to pass compaction validation, sequence files has overlap, time partition id is " - + timePartition); + throw new CompactionValidationFailedException(overlapFilesInTimePartition); } } if (needToValidateTsFileCorrectness && !validator.validateTsFiles(validTargetFiles)) { From 9ea4bba20eaccd07738ac460be8216e120bf2a6f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 19 Jun 2024 10:05:48 +0800 Subject: [PATCH 2/2] fix repair compaction may generate too large chunk --- .../RepairUnsortedFileCompactionWriter.java | 18 ++++++------- .../RepairUnsortedFileCompactionTest.java | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java index 5dce41978c607..3c9c25de6265d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java @@ -63,24 +63,22 @@ public void endMeasurement(int subTaskId) throws IOException { mergeTimeValuePair(timeValuePair, previousTimeValuePair); continue; } - writeDataPoint( - previousTimeValuePair.getTimestamp(), - previousTimeValuePair.getValue(), - chunkWriters[subTaskId]); - chunkPointNumArray[subTaskId]++; + writeToChunkWriter(previousTimeValuePair, subTaskId); previousTimeValuePair = timeValuePair; } // write last time value pair - writeDataPoint( - previousTimeValuePair.getTimestamp(), - previousTimeValuePair.getValue(), - chunkWriters[subTaskId]); - chunkPointNumArray[subTaskId]++; + writeToChunkWriter(previousTimeValuePair, subTaskId); dataOfCurrentSeriesArr[subTaskId] = null; super.endMeasurement(subTaskId); } + private void writeToChunkWriter(TimeValuePair timeValuePair, int subTaskId) throws IOException { + writeDataPoint(timeValuePair.getTimestamp(), timeValuePair.getValue(), chunkWriters[subTaskId]); + chunkPointNumArray[subTaskId]++; + checkChunkSizeAndMayOpenANewChunk(fileWriter, chunkWriters[subTaskId], subTaskId); + } + private void mergeTimeValuePair(TimeValuePair from, TimeValuePair to) { if (!isAlign) { // ignore not aligned TimeValuePair with same timestamp diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index 0b2ce0652aced..08add998ccfdc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -995,4 +995,29 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { } } } + + @Test + public void testSplitChunk() throws IOException { + TsFileResource resource = createEmptyFileAndResource(true); + try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { + writer.startChunkGroup("d1"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s1", "s2", "s3"), + new TimeRange[][] {new TimeRange[] {new TimeRange(100000, 300000)}}, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(true, false, false)); + writer.endChunkGroup(); + writer.endFile(); + } + RepairUnsortedFileCompactionTask task = + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0); + Assert.assertTrue(task.start()); + TsFileResource target = tsFileManager.getTsFileList(false).get(0); + try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { + List chunkMetadataList = + reader.getAlignedChunkMetadata(new PlainDeviceID("root.testsg.d1")); + Assert.assertEquals(3, chunkMetadataList.size()); + } + } }