From e5ae9ad64bf8052ce939441eadd24cdf1194dc62 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:31:06 +0800 Subject: [PATCH 1/8] Update PipeHistoricalDataRegionTsFileExtractor.java --- ...peHistoricalDataRegionTsFileExtractor.java | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 1c150dd54093c..86e64fdd311c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -45,6 +45,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.tsfile.file.metadata.PlainDeviceID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +59,7 @@ import java.util.Map; import java.util.Objects; import java.util.Queue; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_END_TIME_KEY; @@ -99,6 +101,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa private long historicalDataExtractionEndTime = Long.MAX_VALUE; // Event time private long historicalDataExtractionTimeLowerBound; // Arrival time + private boolean sloppyPattern; private boolean sloppyTimeRange; // true to disable time range filter after extraction private boolean shouldExtractInsertion; @@ -202,6 +205,25 @@ public void validate(final PipeParameterValidator validator) { PipeExtractorConstant.SOURCE_MODE_KEY), PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); + + final Set sloppyOptionSet = + Arrays.stream( + parameters + .getStringOrDefault( + Arrays.asList( + EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), + "") + .split(",")) + .map(String::trim) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + sloppyTimeRange = sloppyOptionSet.remove("time"); + sloppyPattern = sloppyOptionSet.remove("path"); + if (!sloppyOptionSet.isEmpty()) { + throw new PipeParameterNotValidException( + String.format("%s is not allowed in 'history.loose-range'", sloppyOptionSet)); + } + } catch (final Exception e) { // Compatible with the current validation framework throw new PipeParameterNotValidException(e.getMessage()); @@ -282,28 +304,17 @@ public void customize( } } - sloppyTimeRange = - Arrays.stream( - parameters - .getStringOrDefault( - Arrays.asList( - EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), - "") - .split(",")) - .map(String::trim) - .map(String::toLowerCase) - .collect(Collectors.toSet()) - .contains("time"); - - LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}", - pipeName, - dataRegionId, - DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), - historicalDataExtractionStartTime, - DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), - historicalDataExtractionEndTime, - sloppyTimeRange); + if (LOGGER.isInfoEnabled()) { + LOGGER.info( + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}", + pipeName, + dataRegionId, + DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), + historicalDataExtractionStartTime, + DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), + historicalDataExtractionEndTime, + sloppyTimeRange); + } } private void flushDataRegionAllTsFiles() { @@ -382,6 +393,7 @@ public synchronized void start() { // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. !resource.isClosed() || mayTsFileContainUnprocessedData(resource) + && mayTsFileResourceOverlappedWithPattern(resource) && isTsFileResourceOverlappedWithTimeRange(resource) && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) .collect(Collectors.toList()); @@ -395,6 +407,7 @@ && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. !resource.isClosed() || mayTsFileContainUnprocessedData(resource) + && mayTsFileResourceOverlappedWithPattern(resource) && isTsFileResourceOverlappedWithTimeRange(resource) && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) .collect(Collectors.toList()); @@ -458,6 +471,13 @@ private boolean mayTsFileContainUnprocessedData(final TsFileResource resource) { return !startIndex.isAfter(resource.getMaxProgressIndexAfterClose()); } + private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { + // TODO: use IDeviceID + return resource.getDevices().stream() + .anyMatch( + deviceID -> pipePattern.mayOverlapWithDevice(((PlainDeviceID) deviceID).toStringID())); + } + private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) { return !(resource.getFileEndTime() < historicalDataExtractionStartTime || historicalDataExtractionEndTime < resource.getFileStartTime()); @@ -514,7 +534,7 @@ public synchronized Event supply() { pipePattern, historicalDataExtractionStartTime, historicalDataExtractionEndTime); - if (isDbNameCoveredByPattern) { + if (sloppyPattern || isDbNameCoveredByPattern) { event.skipParsingPattern(); } From 1741749be421c0e4ad7b51c4e35883772bfb5df3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:47:58 +0800 Subject: [PATCH 2/8] Add IT --- .../it/autocreate/IoTDBPipeExtractorIT.java | 98 +++++++++++++++++++ ...peHistoricalDataRegionTsFileExtractor.java | 3 +- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java index 1befb2b765d7b..bee2f0c0d7bcf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java @@ -24,7 +24,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; @@ -32,6 +34,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -51,6 +54,39 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2AutoCreateSchema.class}) public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT { + + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + + // TODO: delete ratis configurations + senderEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + // Disable sender compaction for tsfile determination in loose range test + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true) + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS); + + // 10 min, assert that the operations will not time out + senderEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + receiverEnv.getConfig().getCommonConfig().setCnConnectionTimeoutMs(600000); + + senderEnv.initClusterEnvironment(); + receiverEnv.initClusterEnvironment(); + } + @Test public void testExtractorValidParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -867,6 +903,68 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex } } + @Test + public void testLooseRange() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 1, extracted without parse + "insert into root.db.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", + // TsFile 2, not extracted because pattern not overlapped + "insert into root.db1.d1 (time, at1, at2)" + " values (1000, 1, 2), (2000, 3, 4)", + "flush"))) { + return; + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + // TsFile 3, not extracted because time range not overlapped + "insert into root.db.d1 (time, at1, at2)" + " values (3000, 1, 2), (4000, 3, 4)", + "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("source.path", "root.db.d1.at1"); + extractorAttributes.put("source.inclusion", "data.insert"); + extractorAttributes.put("source.history.start-time", "1500"); + extractorAttributes.put("source.history.end-time", "2500"); + extractorAttributes.put("source.history.loose-range", "time, path"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + TSStatus status = + client.createPipe( + new TCreatePipeReq("p1", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.** group by level=0", + "count(root.*.*.*),", + Collections.singleton("4,"), + 10); + } + } + private void assertTimeseriesCountOnReceiver(BaseEnv receiverEnv, int count) { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "count timeseries", "count(timeseries),", Collections.singleton(count + ",")); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 86e64fdd311c2..36b41090b2c3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -306,13 +306,14 @@ public void customize( if (LOGGER.isInfoEnabled()) { LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}", + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy pattern {}, sloppy time range {}", pipeName, dataRegionId, DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), historicalDataExtractionStartTime, DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), historicalDataExtractionEndTime, + sloppyPattern, sloppyTimeRange); } } From 7943fa4e99e48156bf091aae49187a868e23ce74 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:48:58 +0800 Subject: [PATCH 3/8] Update IoTDBPipeExtractorIT.java --- .../apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java index bee2f0c0d7bcf..f83699f30b7a4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java @@ -960,8 +960,7 @@ public void testLooseRange() throws Exception { receiverEnv, "select count(*) from root.** group by level=0", "count(root.*.*.*),", - Collections.singleton("4,"), - 10); + Collections.singleton("4,")); } } From 1af708e8036500adf120bcd1b8d1ac98737a31c8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:01:29 +0800 Subject: [PATCH 4/8] bug fix --- .../historical/PipeHistoricalDataRegionTsFileExtractor.java | 5 ++++- .../commons/pipe/config/constant/PipeExtractorConstant.java | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 36b41090b2c3e..96082f2ff3784 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -66,6 +66,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE; @@ -212,11 +213,13 @@ public void validate(final PipeParameterValidator validator) { .getStringOrDefault( Arrays.asList( EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), - "") + EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE) .split(",")) .map(String::trim) .map(String::toLowerCase) .collect(Collectors.toSet()); + // Avoid empty string + sloppyOptionSet.remove(""); sloppyTimeRange = sloppyOptionSet.remove("time"); sloppyPattern = sloppyOptionSet.remove("path"); if (!sloppyOptionSet.isEmpty()) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index a67b594d361da..7a1f14411f93a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -64,6 +64,7 @@ public class PipeExtractorConstant { public static final String SOURCE_HISTORY_END_TIME_KEY = "source.history.end-time"; public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_KEY = "extractor.history.loose-range"; public static final String SOURCE_HISTORY_LOOSE_RANGE_KEY = "source.history.loose-range"; + public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE = ""; public static final String EXTRACTOR_MODS_ENABLE_KEY = "extractor.mods.enable"; public static final String SOURCE_MODS_ENABLE_KEY = "source.mods.enable"; public static final boolean EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE = false; From 8c2395b4f309a1c8aba0953c3dc942ff88d5d500 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 4 Jun 2024 18:49:51 +0800 Subject: [PATCH 5/8] Update PipeHistoricalDataRegionTsFileExtractor.java --- ...peHistoricalDataRegionTsFileExtractor.java | 65 +++++++++---------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index d0b55f7b21f46..3519ffd2dd044 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -127,25 +127,25 @@ public void validate(final PipeParameterValidator validator) { throw new PipeParameterNotValidException(e.getMessage()); } - final Set sloppyOptionSet = - Arrays.stream( - parameters - .getStringOrDefault( - Arrays.asList( - EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), - EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE) - .split(",")) - .map(String::trim) - .map(String::toLowerCase) - .collect(Collectors.toSet()); - // Avoid empty string - sloppyOptionSet.remove(""); - sloppyTimeRange = sloppyOptionSet.remove("time"); - sloppyPattern = sloppyOptionSet.remove("path"); - if (!sloppyOptionSet.isEmpty()) { - throw new PipeParameterNotValidException( - String.format("%s is not allowed in 'history.loose-range'", sloppyOptionSet)); - } + final Set sloppyOptionSet = + Arrays.stream( + parameters + .getStringOrDefault( + Arrays.asList( + EXTRACTOR_HISTORY_LOOSE_RANGE_KEY, SOURCE_HISTORY_LOOSE_RANGE_KEY), + EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE) + .split(",")) + .map(String::trim) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + // Avoid empty string + sloppyOptionSet.remove(""); + sloppyTimeRange = sloppyOptionSet.remove("time"); + sloppyPattern = sloppyOptionSet.remove("path"); + if (!sloppyOptionSet.isEmpty()) { + throw new PipeParameterNotValidException( + String.format("%s is not allowed in 'history.loose-range'", sloppyOptionSet)); + } if (parameters.hasAnyAttributes( SOURCE_START_TIME_KEY, @@ -300,7 +300,6 @@ public void customize( } } - shouldTransferModFile = parameters.getBooleanOrDefault( Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY), @@ -317,19 +316,19 @@ public void customize( PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE) .equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE); - if (LOGGER.isInfoEnabled()) { - LOGGER.info( - "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy pattern {}, sloppy time range {}, should transfer mod file {}, should terminate pipe on all historical events consumed {}", - pipeName, - dataRegionId, - DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), - historicalDataExtractionStartTime, - DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), - historicalDataExtractionEndTime, - sloppyPattern, - sloppyTimeRange, - shouldTransferModFile, - shouldTerminatePipeOnAllHistoricalEventsConsumed); + if (LOGGER.isInfoEnabled()) { + LOGGER.info( + "Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy pattern {}, sloppy time range {}, should transfer mod file {}, should terminate pipe on all historical events consumed {}", + pipeName, + dataRegionId, + DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime), + historicalDataExtractionStartTime, + DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime), + historicalDataExtractionEndTime, + sloppyPattern, + sloppyTimeRange, + shouldTransferModFile, + shouldTerminatePipeOnAllHistoricalEventsConsumed); } } From 7ff1b02c388d6cbc28b5a0bba369c5f93d514399 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 20:07:07 +0800 Subject: [PATCH 6/8] Read from cache --- ...peHistoricalDataRegionTsFileExtractor.java | 23 ++++++++++- .../tsfile/PipeTsFileResourceManager.java | 38 ++++++++++--------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 3519ffd2dd044..285c81aec43d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter; import org.apache.iotdb.db.pipe.resource.PipeResourceManager; +import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; @@ -43,8 +44,10 @@ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -488,11 +491,29 @@ private boolean mayTsFileContainUnprocessedData(final TsFileResource resource) { private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { // TODO: use IDeviceID - return resource.getDevices().stream() + return getTsFileDevice(resource).stream() .anyMatch( deviceID -> pipePattern.mayOverlapWithDevice(((PlainDeviceID) deviceID).toStringID())); } + private Set getTsFileDevice(final TsFileResource resource) { + try { + final Map deviceIsAlignedMap = + PipeResourceManager.tsfile() + .getDeviceIsAlignedMapFromCache( + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile())); + return Objects.nonNull(deviceIsAlignedMap) + ? deviceIsAlignedMap.keySet() + : resource.getDevices(); + } catch (final IOException e) { + throw new PipeException( + String.format( + "Failed to get devices of %s when trying to filter it by the devices and pattern", + resource.getTsFile()), + e); + } + } + private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) { return !(resource.getFileEndTime() < historicalDataExtractionStartTime || historicalDataExtractionEndTime < resource.getFileStartTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index 0d064cd8b067c..5e75e068fa271 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -71,7 +71,7 @@ private void tryTtlCheck() { } else { LOGGER.warn("failed to try lock when checking TTL because of timeout ({}s)", timeout); } - } catch (InterruptedException e) { + } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.warn("failed to try lock when checking TTL because of interruption", e); } @@ -102,7 +102,7 @@ private void ttlCheck() { entry.getKey(), entry.getValue().getReferenceCount())); } - } catch (IOException e) { + } catch (final IOException e) { LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e); } } @@ -129,7 +129,8 @@ private void ttlCheck() { * @return the hardlink or copied file * @throws IOException when create hardlink or copy file failed */ - public File increaseFileReference(File file, boolean isTsFile, TsFileResource tsFileResource) + public File increaseFileReference( + final File file, final boolean isTsFile, final TsFileResource tsFileResource) throws IOException { lock.lock(); try { @@ -165,7 +166,7 @@ public File increaseFileReference(File file, boolean isTsFile, TsFileResource ts } } - private boolean increaseReferenceIfExists(String path) { + private boolean increaseReferenceIfExists(final String path) { final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(path); if (resource != null) { resource.increaseAndGetReference(); @@ -174,10 +175,10 @@ private boolean increaseReferenceIfExists(String path) { return false; } - private static File getHardlinkOrCopiedFileInPipeDir(File file) throws IOException { + public static File getHardlinkOrCopiedFileInPipeDir(final File file) throws IOException { try { return new File(getPipeTsFileDirPath(file), getRelativeFilePath(file)); - } catch (Exception e) { + } catch (final Exception e) { throw new IOException( String.format( "failed to get hardlink or copied file in pipe dir " @@ -218,7 +219,7 @@ private static String getRelativeFilePath(File file) { * * @param hardlinkOrCopiedFile the copied or hardlinked file */ - public void decreaseFileReference(File hardlinkOrCopiedFile) { + public void decreaseFileReference(final File hardlinkOrCopiedFile) { lock.lock(); try { final String filePath = hardlinkOrCopiedFile.getPath(); @@ -237,7 +238,7 @@ public void decreaseFileReference(File hardlinkOrCopiedFile) { * @param hardlinkOrCopiedFile the copied or hardlinked file * @return the reference count of the file */ - public int getFileReferenceCount(File hardlinkOrCopiedFile) { + public int getFileReferenceCount(final File hardlinkOrCopiedFile) { lock.lock(); try { final String filePath = hardlinkOrCopiedFile.getPath(); @@ -254,7 +255,7 @@ public int getFileReferenceCount(File hardlinkOrCopiedFile) { * @return {@code true} if the maps are successfully put into cache or already cached. {@code * false} if they can not be cached. */ - public boolean cacheObjectsIfAbsent(File hardlinkOrCopiedTsFile) throws IOException { + public boolean cacheObjectsIfAbsent(final File hardlinkOrCopiedTsFile) throws IOException { lock.lock(); try { final PipeTsFileResource resource = @@ -265,8 +266,8 @@ public boolean cacheObjectsIfAbsent(File hardlinkOrCopiedTsFile) throws IOExcept } } - public Map> getDeviceMeasurementsMapFromCache(File hardlinkOrCopiedTsFile) - throws IOException { + public Map> getDeviceMeasurementsMapFromCache( + final File hardlinkOrCopiedTsFile) throws IOException { lock.lock(); try { final PipeTsFileResource resource = @@ -277,7 +278,7 @@ public Map> getDeviceMeasurementsMapFromCache(File hardl } } - public Map getDeviceIsAlignedMapFromCache(File hardlinkOrCopiedTsFile) + public Map getDeviceIsAlignedMapFromCache(final File hardlinkOrCopiedTsFile) throws IOException { lock.lock(); try { @@ -289,8 +290,8 @@ public Map getDeviceIsAlignedMapFromCache(File hardlinkOrCop } } - public Map getMeasurementDataTypeMapFromCache(File hardlinkOrCopiedTsFile) - throws IOException { + public Map getMeasurementDataTypeMapFromCache( + final File hardlinkOrCopiedTsFile) throws IOException { lock.lock(); try { final PipeTsFileResource resource = @@ -301,7 +302,8 @@ public Map getMeasurementDataTypeMapFromCache(File hardlinkO } } - public void pinTsFileResource(TsFileResource resource, boolean withMods) throws IOException { + public void pinTsFileResource(final TsFileResource resource, final boolean withMods) + throws IOException { lock.lock(); try { increaseFileReference(resource.getTsFile(), true, resource); @@ -313,13 +315,13 @@ public void pinTsFileResource(TsFileResource resource, boolean withMods) throws } } - public void unpinTsFileResource(TsFileResource resource) throws IOException { + public void unpinTsFileResource(final TsFileResource resource) throws IOException { lock.lock(); try { - File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()); + final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()); decreaseFileReference(pinnedFile); - File modFile = new File(pinnedFile + ModificationFile.FILE_SUFFIX); + final File modFile = new File(pinnedFile + ModificationFile.FILE_SUFFIX); if (modFile.exists()) { decreaseFileReference(modFile); } From f126531b362dbd577d6e6f5302977a9df4fe572c Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 20:10:13 +0800 Subject: [PATCH 7/8] Update PipeHistoricalDataRegionTsFileExtractor.java --- ...peHistoricalDataRegionTsFileExtractor.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 285c81aec43d8..00238a5bf3d22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -44,7 +44,6 @@ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.event.Event; -import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.file.metadata.IDeviceID; @@ -411,9 +410,9 @@ public synchronized void start() { // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. !resource.isClosed() || mayTsFileContainUnprocessedData(resource) - && mayTsFileResourceOverlappedWithPattern(resource) && isTsFileResourceOverlappedWithTimeRange(resource) - && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) + && isTsFileGeneratedAfterExtractionTimeLowerBound(resource) + && mayTsFileResourceOverlappedWithPattern(resource)) .collect(Collectors.toList()); resourceList.addAll(sequenceTsFileResources); @@ -425,9 +424,9 @@ && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them. !resource.isClosed() || mayTsFileContainUnprocessedData(resource) - && mayTsFileResourceOverlappedWithPattern(resource) && isTsFileResourceOverlappedWithTimeRange(resource) - && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)) + && isTsFileGeneratedAfterExtractionTimeLowerBound(resource) + && mayTsFileResourceOverlappedWithPattern(resource)) .collect(Collectors.toList()); resourceList.addAll(unsequenceTsFileResources); @@ -490,28 +489,32 @@ private boolean mayTsFileContainUnprocessedData(final TsFileResource resource) { } private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { - // TODO: use IDeviceID - return getTsFileDevice(resource).stream() - .anyMatch( - deviceID -> pipePattern.mayOverlapWithDevice(((PlainDeviceID) deviceID).toStringID())); - } + if (!sloppyPattern) { + return true; + } - private Set getTsFileDevice(final TsFileResource resource) { + final Set deviceSet; try { final Map deviceIsAlignedMap = PipeResourceManager.tsfile() .getDeviceIsAlignedMapFromCache( PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile())); - return Objects.nonNull(deviceIsAlignedMap) - ? deviceIsAlignedMap.keySet() - : resource.getDevices(); + deviceSet = + Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices(); } catch (final IOException e) { - throw new PipeException( - String.format( - "Failed to get devices of %s when trying to filter it by the devices and pattern", - resource.getTsFile()), + LOGGER.warn( + "Pipe {}@{}: failed to get devices from TsFile {}, extract it anyway", + pipeName, + dataRegionId, + resource.getTsFilePath(), e); + return true; } + + return deviceSet.stream() + .anyMatch( + // TODO: use IDeviceID + deviceID -> pipePattern.mayOverlapWithDevice(((PlainDeviceID) deviceID).toStringID())); } private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) { From d515ee633c7446a71b2b2fd28d94bc2e72a0fad9 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 20:13:29 +0800 Subject: [PATCH 8/8] refactor --- .../PipeHistoricalDataRegionTsFileExtractor.java | 9 ++++++--- .../pipe/config/constant/PipeExtractorConstant.java | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java index 00238a5bf3d22..b750d87f89e0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java @@ -71,6 +71,8 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODS_ENABLE_KEY; @@ -142,11 +144,12 @@ public void validate(final PipeParameterValidator validator) { .collect(Collectors.toSet()); // Avoid empty string sloppyOptionSet.remove(""); - sloppyTimeRange = sloppyOptionSet.remove("time"); - sloppyPattern = sloppyOptionSet.remove("path"); + sloppyTimeRange = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE); + sloppyPattern = sloppyOptionSet.remove(EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE); if (!sloppyOptionSet.isEmpty()) { throw new PipeParameterNotValidException( - String.format("%s is not allowed in 'history.loose-range'", sloppyOptionSet)); + String.format( + "Parameters in set %s are not allowed in 'history.loose-range'", sloppyOptionSet)); } if (parameters.hasAnyAttributes( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index 7a1f14411f93a..99efcd48d613d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -64,6 +64,8 @@ public class PipeExtractorConstant { public static final String SOURCE_HISTORY_END_TIME_KEY = "source.history.end-time"; public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_KEY = "extractor.history.loose-range"; public static final String SOURCE_HISTORY_LOOSE_RANGE_KEY = "source.history.loose-range"; + public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE = "time"; + public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE = "path"; public static final String EXTRACTOR_HISTORY_LOOSE_RANGE_DEFAULT_VALUE = ""; public static final String EXTRACTOR_MODS_ENABLE_KEY = "extractor.mods.enable"; public static final String SOURCE_MODS_ENABLE_KEY = "source.mods.enable";