From 759e930eb80587255258551c8f975a864109ee87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=AD=90=E5=9D=A4?= <55695098+DanielWang2035@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:47:21 +0800 Subject: [PATCH 1/2] Pipe: Filter devices by pattern before reading device metadata from TsFile --- .../tsfile/TsFileInsertionDataContainer.java | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java index 03e61cf280986..04a4c80d874cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.pattern.PipePattern; import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.resource.PipeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; @@ -77,6 +78,7 @@ public class TsFileInsertionDataContainer implements AutoCloseable { private boolean shouldParsePattern = false; + @TestOnly public TsFileInsertionDataContainer( final File tsFile, final PipePattern pattern, final long startTime, final long endTime) throws IOException { @@ -122,15 +124,21 @@ public TsFileInsertionDataContainer( deviceIsAlignedMap = readDeviceIsAlignedMap(); memoryRequiredInBytes += PipeMemoryWeighUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap); - measurementDataTypeMap = tsFileSequenceReader.getFullPathDataTypeMap(); + // Filter devices that may overlap with pattern first + // to avoid reading all time-series of all devices. + final List devices = + filterDevicesByPattern(tsFileSequenceReader.getAllDevices()); + + measurementDataTypeMap = readFilteredFullPathDataTypeMap(devices); memoryRequiredInBytes += PipeMemoryWeighUtil.memoryOfStr2TSDataType(measurementDataTypeMap); - deviceMeasurementsMap = tsFileSequenceReader.getDeviceMeasurementsMap(); + deviceMeasurementsMap = readFilteredDeviceMeasurementsMap(devices); memoryRequiredInBytes += PipeMemoryWeighUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap); } allocatedMemoryBlock = PipeResourceManager.memory().forceAllocate(memoryRequiredInBytes); + // Filter again to get the final deviceMeasurementsMap that exactly matches the pattern. deviceMeasurementsMapIterator = filterDeviceMeasurementsMapByPattern(deviceMeasurementsMap).entrySet().iterator(); @@ -198,6 +206,67 @@ private Map readDeviceIsAlignedMap() throws IOException { return deviceIsAlignedResultMap; } + private List filterDevicesByPattern(final List devices) { + if (Objects.isNull(pattern) || pattern.isRoot()) { + return devices; + } + + final List filteredDevices = new ArrayList<>(); + for (final IDeviceID device : devices) { + final String deviceId = ((PlainDeviceID) device).toStringID(); + if (pattern.coversDevice(deviceId) || pattern.mayOverlapWithDevice(deviceId)) { + filteredDevices.add(device); + } + } + return filteredDevices; + } + + /** + * This method is similar to {@link TsFileSequenceReader#getFullPathDataTypeMap()}, but only reads + * the given devices. + */ + private Map readFilteredFullPathDataTypeMap(final List devices) + throws IOException { + final Map result = new HashMap<>(); + + for (IDeviceID device : devices) { + tsFileSequenceReader + .readDeviceMetadata(device) + .values() + .forEach( + timeseriesMetadata -> + result.put( + ((PlainDeviceID) device).toStringID() + + "." + + timeseriesMetadata.getMeasurementId(), + timeseriesMetadata.getTsDataType())); + } + + return result; + } + + /** + * This method is similar to {@link TsFileSequenceReader#getDeviceMeasurementsMap()}, but only + * reads the given devices. + */ + private Map> readFilteredDeviceMeasurementsMap( + final List devices) throws IOException { + final Map> result = new HashMap<>(); + + for (IDeviceID device : devices) { + tsFileSequenceReader + .readDeviceMetadata(device) + .values() + .forEach( + timeseriesMetadata -> + result + .computeIfAbsent(device, d -> new ArrayList<>()) + .add(timeseriesMetadata.getMeasurementId())); + } + + return result; + } + /** * @return {@link TabletInsertionEvent} in a streaming way */ From 4a3a464ac90cf2044c4f3a4528ecaa838a64ecb0 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 19 Jun 2024 20:22:04 +0800 Subject: [PATCH 2/2] Update TsFileInsertionDataContainer.java --- .../common/tsfile/TsFileInsertionDataContainer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java index 04a4c80d874cf..8030cce93244d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java @@ -51,11 +51,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; public class TsFileInsertionDataContainer implements AutoCloseable { @@ -126,8 +128,7 @@ public TsFileInsertionDataContainer( // Filter devices that may overlap with pattern first // to avoid reading all time-series of all devices. - final List devices = - filterDevicesByPattern(tsFileSequenceReader.getAllDevices()); + final Set devices = filterDevicesByPattern(deviceIsAlignedMap.keySet()); measurementDataTypeMap = readFilteredFullPathDataTypeMap(devices); memoryRequiredInBytes += PipeMemoryWeighUtil.memoryOfStr2TSDataType(measurementDataTypeMap); @@ -206,12 +207,12 @@ private Map readDeviceIsAlignedMap() throws IOException { return deviceIsAlignedResultMap; } - private List filterDevicesByPattern(final List devices) { + private Set filterDevicesByPattern(final Set devices) { if (Objects.isNull(pattern) || pattern.isRoot()) { return devices; } - final List filteredDevices = new ArrayList<>(); + final Set filteredDevices = new HashSet<>(); for (final IDeviceID device : devices) { final String deviceId = ((PlainDeviceID) device).toStringID(); if (pattern.coversDevice(deviceId) || pattern.mayOverlapWithDevice(deviceId)) { @@ -225,7 +226,7 @@ private List filterDevicesByPattern(final List devices) { * This method is similar to {@link TsFileSequenceReader#getFullPathDataTypeMap()}, but only reads * the given devices. */ - private Map readFilteredFullPathDataTypeMap(final List devices) + private Map readFilteredFullPathDataTypeMap(final Set devices) throws IOException { final Map result = new HashMap<>(); @@ -250,7 +251,7 @@ private Map readFilteredFullPathDataTypeMap(final List> readFilteredDeviceMeasurementsMap( - final List devices) throws IOException { + final Set devices) throws IOException { final Map> result = new HashMap<>(); for (IDeviceID device : devices) {