From 137998b6a6172975d158d3af3d9615ba4c58dc6a Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 11 Jun 2024 12:09:39 +0800 Subject: [PATCH 01/12] done --- .../common/schematree/ClusterSchemaTree.java | 10 ++- .../plan/analyze/AnalyzeVisitor.java | 80 ++++++++++--------- .../analyze/schema/ClusterSchemaFetcher.java | 9 ++- .../plan/analyze/schema/ISchemaFetcher.java | 4 +- 4 files changed, 64 insertions(+), 39 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index c461d0da83ffc..dfd02bc5b2975 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -55,6 +56,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; import static org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode.SCHEMA_ENTITY_NODE; import static org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode.SCHEMA_MEASUREMENT_NODE; @@ -73,6 +75,8 @@ public class ClusterSchemaTree implements ISchemaTree { /** used to judge schema tree type */ private boolean hasNormalTimeSeries = false; + private PathPatternTree authorityScope = ALL_MATCH_SCOPE; + private Map templateMap = new HashMap<>(); public ClusterSchemaTree() { @@ -87,6 +91,10 @@ public void setTemplateMap(Map templateMap) { this.templateMap = templateMap; } + public void setAuthorityScope(PathPatternTree authorityScope) { + this.authorityScope = authorityScope; + } + /** * Return all measurement paths for given path pattern and filter the result by slimit and offset. * @@ -99,7 +107,7 @@ public Pair, Integer> searchMeasurementPaths( PartialPath pathPattern, int slimit, int soffset, boolean isPrefixMatch) { try (SchemaTreeVisitorWithLimitOffsetWrapper visitor = SchemaTreeVisitorFactory.createSchemaTreeMeasurementVisitor( - root, pathPattern, isPrefixMatch, slimit, soffset)) { + root, pathPattern, isPrefixMatch, slimit, soffset, authorityScope)) { visitor.setTemplateMap(templateMap); return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 038514ee02f03..5a281cd412822 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -34,6 +34,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -57,6 +58,7 @@ import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; +import org.apache.iotdb.db.queryengine.common.schematree.MeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.execution.operator.window.WindowType; import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; @@ -2856,58 +2858,64 @@ private boolean analyzeTimeseriesRegionScan( WhereCondition timeCondition, PathPatternTree patternTree, Analysis analysis, - MPPQueryContext context) + MPPQueryContext context, + PathPatternTree authorityScope) throws IllegalPathException { analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); - ISchemaTree schemaTree = schemaFetcher.fetchSchemaWithTags(patternTree, false, context); + ISchemaTree schemaTree = schemaFetcher.fetchSchemaWithTags(PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), false, context); + if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return false; } removeLogicViewMeasurement(schemaTree); + Map> deviceToMeasurementMap = new HashMap<>(); + Map deviceToAlignedMap = new HashMap<>(); + Map> deviceToMeasurementSchemaMap = new HashMap<>(); + Map> deviceToTimeseriesSchemaMap = new HashMap<>(); + for(PartialPath pattern: patternTree.getAllPathPatterns()) { + List measurementPathList = schemaTree.searchMeasurementPaths(pattern).left; + for (MeasurementPath measurementPath : measurementPathList) { + boolean isAligned = measurementPath.isUnderAlignedEntity(); + PartialPath devicePath = measurementPath.getDevicePath(); + deviceToAlignedMap.put(devicePath, isAligned); + deviceToMeasurementMap + .computeIfAbsent(devicePath, k -> new ArrayList<>()).add(measurementPath.getMeasurement()); + deviceToMeasurementSchemaMap + .computeIfAbsent(devicePath, k -> new ArrayList<>()) + .add(measurementPath.getMeasurementSchema()); + deviceToTimeseriesSchemaMap.computeIfAbsent(devicePath, k -> new ArrayList<>()) + .add(new TimeseriesSchemaInfo(new MeasurementSchemaInfo(measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), measurementPath.getMeasurementAlias(), measurementPath.getTagMap()))); + } + } + Map>> deviceToTimeseriesSchemaInfo = - new HashMap<>(); - List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); - Set deviceSet = new HashSet<>(); - for (DeviceSchemaInfo deviceSchemaInfo : deviceSchemaInfoList) { - boolean isAligned = deviceSchemaInfo.isAligned(); - PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - deviceSet.add(devicePath.getFullPath()); - if (isAligned) { - List measurementList = new ArrayList<>(); - List schemaList = new ArrayList<>(); - List timeseriesSchemaInfoList = new ArrayList<>(); - for (IMeasurementSchemaInfo measurementSchemaInfo : - deviceSchemaInfo.getMeasurementSchemaInfoList()) { - schemaList.add(measurementSchemaInfo.getSchema()); - measurementList.add(measurementSchemaInfo.getName()); - timeseriesSchemaInfoList.add(new TimeseriesSchemaInfo(measurementSchemaInfo)); - } - AlignedPath alignedPath = - new AlignedPath(devicePath.getNodes(), measurementList, schemaList); + new HashMap<>(); + for(PartialPath devicePath: deviceToAlignedMap.keySet()){ + if(deviceToAlignedMap.get(devicePath)) { + List measurementList = deviceToMeasurementMap.get(devicePath); + List schemaList = deviceToMeasurementSchemaMap.get(devicePath); + List timeseriesSchemaInfoList = deviceToTimeseriesSchemaMap.get(devicePath); + AlignedPath alignedPath = new AlignedPath(devicePath.getNodes(), measurementList, schemaList); deviceToTimeseriesSchemaInfo - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(alignedPath, timeseriesSchemaInfoList); - } else { - for (IMeasurementSchemaInfo measurementSchemaInfo : - deviceSchemaInfo.getMeasurementSchemaInfoList()) { - MeasurementPath measurementPath = - new MeasurementPath( - devicePath.concatNode(measurementSchemaInfo.getName()).getNodes()); + .computeIfAbsent(devicePath, k -> new HashMap<>()) + .put(alignedPath, timeseriesSchemaInfoList); + }else{ + for(String measurement: deviceToMeasurementMap.get(devicePath)){ + MeasurementPath measurementPath = new MeasurementPath(devicePath.concatNode(measurement).getNodes()); deviceToTimeseriesSchemaInfo - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put( - measurementPath, - Collections.singletonList(new TimeseriesSchemaInfo(measurementSchemaInfo))); + .computeIfAbsent(devicePath, k -> new HashMap<>()) + .put(measurementPath, Collections.singletonList(new TimeseriesSchemaInfo(new MeasurementSchemaInfo(measurement, deviceToMeasurementSchemaMap.get(devicePath).get(0), null, null)))); } } } + analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesSchemaInfo); // fetch Data partition - DataPartition dataPartition = fetchDataPartitionByDevices(deviceSet, schemaTree, context); + DataPartition dataPartition = fetchDataPartitionByDevices(deviceToAlignedMap.keySet().stream().map(PartialPath::getFullPath).collect(Collectors.toSet()), schemaTree, context); analysis.setDataPartitionInfo(dataPartition); return true; } @@ -2926,7 +2934,7 @@ public Analysis visitShowTimeSeries( // If there is time condition in SHOW TIMESERIES, we need to scan the raw data boolean hasSchema = analyzeTimeseriesRegionScan( - showTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context); + showTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context, showTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader()); return analysis; @@ -3137,7 +3145,7 @@ public Analysis visitCountTimeSeries( try { boolean hasSchema = analyzeTimeseriesRegionScan( - countTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context); + countTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context, countTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getCountTimeSeriesHeader()); return analysis; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index bae7472ab8d09..d9fa5212e2e50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -155,11 +155,18 @@ public ClusterSchemaTree fetchSchema( } } + @Override + public ISchemaTree fetchSchemaInDeviceLevel(PathPatternTree patternTree, MPPQueryContext context) { + patternTree.constructTree(); + + return ISchemaFetcher.super.fetchSchemaInDeviceLevel(patternTree, context); + } + @Override public ClusterSchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { patternTree.constructTree(); - return clusterSchemaFetchExecutor.fetchSchemaOfFuzzyMatch( + return clusterSchemaFetchExecutor.fetchSchemaOfFuzzyMatch( patternTree, true, withTemplate, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java index a51f8af0ab139..b723b2847f325 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java @@ -39,9 +39,11 @@ public interface ISchemaFetcher { /** - * Fetch all the schema of existing timeseries matched by the given patternTree + * Fetch all the schema of existing timeseries matched by the given patternTree. The result schema tree may contain extra measurement nodes if + * withTemplate is true. It recommends to use pattern to filter the result schema tree. * * @param patternTree used for matching the timeseries + * @param withTemplate whether to include the template in the schema * @return the matched timeseries schema organized as tree structure logically */ ISchemaTree fetchSchema( From adfe1030851ddb0663c68eb2ad92882590fe8299 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 11:46:32 +0800 Subject: [PATCH 02/12] done --- .../common/TimeseriesSchemaInfo.java | 2 +- .../common/schematree/ClusterSchemaTree.java | 33 +++ .../schema/SchemaFetchScanOperator.java | 51 +++- .../plan/analyze/AnalyzeVisitor.java | 152 ++++++++---- .../schema/ClusterSchemaFetchExecutor.java | 19 +- .../analyze/schema/ClusterSchemaFetcher.java | 10 +- .../plan/analyze/schema/ISchemaFetcher.java | 19 +- .../plan/planner/LogicalPlanBuilder.java | 35 ++- .../plan/planner/LogicalPlanVisitor.java | 32 ++- .../plan/planner/OperatorTreeGenerator.java | 27 ++- .../distribution/ExchangeNodeAdder.java | 12 +- .../planner/distribution/SourceRewriter.java | 4 +- .../plan/planner/plan/node/PlanNodeType.java | 13 +- .../plan/planner/plan/node/PlanVisitor.java | 9 +- .../read/DeviceSchemaFetchScanNode.java | 108 +++++++++ .../metedata/read/SchemaFetchScanNode.java | 219 +++++------------- .../read/SeriesSchemaFetchScanNode.java | 148 ++++++++++++ .../plan/statement/StatementVisitor.java | 13 +- .../internal/DeviceSchemaFetchStatement.java | 58 +++++ ...t.java => SeriesSchemaFetchStatement.java} | 6 +- .../schemaregion/ISchemaRegion.java | 13 +- .../impl/SchemaRegionMemoryImpl.java | 8 +- .../impl/SchemaRegionPBTreeImpl.java | 8 +- .../impl/mem/MTreeBelowSGMemoryImpl.java | 20 ++ .../impl/pbtree/MTreeBelowSGCachedImpl.java | 20 ++ .../schemaRegion/SchemaRegionBasicTest.java | 10 +- .../SchemaRegionSimpleRecoverTest.java | 4 +- .../SchemaRegionTemplateTest.java | 8 +- .../operator/OperatorMemoryTest.java | 2 +- .../schema/SchemaFetchScanOperatorTest.java | 4 +- .../read/DeviceSchemaFetchScanNodeTest.java | 55 +++++ .../read/SchemaFetchMergeNodeTest.java | 8 +- ...ava => SeriesSchemaFetchScanNodeTest.java} | 13 +- .../commons/path/PathPatternTreeUtils.java | 5 + 34 files changed, 861 insertions(+), 287 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/{SchemaFetchStatement.java => SeriesSchemaFetchStatement.java} (93%) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/{SchemaFetchScanNodeTest.java => SeriesSchemaFetchScanNodeTest.java} (85%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java index 946a3cfa90482..56dd0fa6a8078 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java @@ -39,7 +39,7 @@ public class TimeseriesSchemaInfo { private final String tags; private final String alias; - // TODO: Currently we can't get attributes from fetchSchema in query + // TODO: Currently we can't get attributes from fetchSeriesSchema in query // private final String attributes; private final String deadband; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index dfd02bc5b2975..336afb0274569 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -314,6 +314,39 @@ public void appendTemplateDevice( templateMap.putIfAbsent(templateId, template); } + /** + * Append a template device to the schema tree. + * + * @param devicePath device path + * @param isAligned whether the device is aligned + */ + public void appendDevice(PartialPath devicePath, Boolean isAligned) { + String[] nodes = devicePath.getNodes(); + SchemaNode cur = root; + SchemaNode child; + for (int i = 1; i < nodes.length - 1; i++) { + child = cur.getChild(nodes[i]); + if (child == null) { + child = new SchemaInternalNode(nodes[i]); + cur.addChild(nodes[i], child); + } + cur = child; + } + String deviceName = nodes[nodes.length - 1]; + child = cur.getChild(deviceName); + if (child == null) { + SchemaEntityNode entityNode = new SchemaEntityNode(deviceName); + entityNode.setAligned(isAligned); + cur.addChild(deviceName, entityNode); + } else if (child.isEntity()) { + child.getAsEntityNode().setAligned(isAligned); + } else { + SchemaEntityNode entityNode = new SchemaEntityNode(deviceName); + entityNode.setAligned(isAligned); + cur.replaceChild(deviceName, entityNode); + } + } + public void appendMeasurementPaths(List measurementPathList) { for (MeasurementPath measurementPath : measurementPathList) { appendSingleMeasurementPath(measurementPath); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java index c6236d018a981..47a80841a4bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -40,6 +41,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -53,7 +55,9 @@ public class SchemaFetchScanOperator implements SourceOperator { private final ISchemaRegion schemaRegion; private final boolean withTags; private final boolean withTemplate; + private final boolean fetchDevice; private boolean isFinished = false; + private final PathPatternTree authorityScope; private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); @@ -61,7 +65,29 @@ public class SchemaFetchScanOperator implements SourceOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(SchemaFetchScanOperator.class); - public SchemaFetchScanOperator( + public static SchemaFetchScanOperator ofSeries( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + Map templateMap, + ISchemaRegion schemaRegion, + boolean withTags, + boolean withTemplate) { + return new SchemaFetchScanOperator( + planNodeId, context, patternTree, templateMap, schemaRegion, withTags, withTemplate); + } + + public static SchemaFetchScanOperator ofDevice( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + PathPatternTree authorityScope, + ISchemaRegion schemaRegion) { + return new SchemaFetchScanOperator( + planNodeId, context, patternTree, authorityScope, schemaRegion); + } + + private SchemaFetchScanOperator( PlanNodeId planNodeId, OperatorContext context, PathPatternTree patternTree, @@ -76,6 +102,25 @@ public SchemaFetchScanOperator( this.templateMap = templateMap; this.withTags = withTags; this.withTemplate = withTemplate; + this.fetchDevice = false; + this.authorityScope = SchemaConstant.ALL_MATCH_SCOPE; + } + + private SchemaFetchScanOperator( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + PathPatternTree authorityScope, + ISchemaRegion schemaRegion) { + this.sourceId = planNodeId; + this.operatorContext = context; + this.patternTree = patternTree; + this.schemaRegion = schemaRegion; + this.templateMap = Collections.emptyMap(); + this.withTags = false; + this.withTemplate = false; + this.fetchDevice = true; + this.authorityScope = authorityScope; } @Override @@ -118,7 +163,9 @@ public PlanNodeId getSourceId() { private TsBlock fetchSchema() throws MetadataException { ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema(patternTree, templateMap, withTags, withTemplate); + fetchDevice + ? schemaRegion.fetchDeviceSchema(patternTree, authorityScope) + : schemaRegion.fetchSeriesSchema(patternTree, templateMap, withTags, withTemplate); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 5a281cd412822..894881dc6cd8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -56,7 +56,6 @@ import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.MeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.execution.operator.window.WindowType; @@ -108,10 +107,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; @@ -2863,8 +2863,12 @@ private boolean analyzeTimeseriesRegionScan( throws IllegalPathException { analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); - - ISchemaTree schemaTree = schemaFetcher.fetchSchemaWithTags(PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), false, context); + patternTree.constructTree(); + ISchemaTree schemaTree = + schemaFetcher.fetchSchemaWithTags( + PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), + false, + context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); @@ -2876,46 +2880,71 @@ private boolean analyzeTimeseriesRegionScan( Map deviceToAlignedMap = new HashMap<>(); Map> deviceToMeasurementSchemaMap = new HashMap<>(); Map> deviceToTimeseriesSchemaMap = new HashMap<>(); - for(PartialPath pattern: patternTree.getAllPathPatterns()) { - List measurementPathList = schemaTree.searchMeasurementPaths(pattern).left; + for (PartialPath pattern : patternTree.getAllPathPatterns()) { + List measurementPathList = schemaTree.searchMeasurementPaths(pattern).left; for (MeasurementPath measurementPath : measurementPathList) { boolean isAligned = measurementPath.isUnderAlignedEntity(); PartialPath devicePath = measurementPath.getDevicePath(); deviceToAlignedMap.put(devicePath, isAligned); deviceToMeasurementMap - .computeIfAbsent(devicePath, k -> new ArrayList<>()).add(measurementPath.getMeasurement()); + .computeIfAbsent(devicePath, k -> new ArrayList<>()) + .add(measurementPath.getMeasurement()); deviceToMeasurementSchemaMap - .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(measurementPath.getMeasurementSchema()); - deviceToTimeseriesSchemaMap.computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(new TimeseriesSchemaInfo(new MeasurementSchemaInfo(measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), measurementPath.getMeasurementAlias(), measurementPath.getTagMap()))); + .computeIfAbsent(devicePath, k -> new ArrayList<>()) + .add(measurementPath.getMeasurementSchema()); + deviceToTimeseriesSchemaMap + .computeIfAbsent(devicePath, k -> new ArrayList<>()) + .add( + new TimeseriesSchemaInfo( + new MeasurementSchemaInfo( + measurementPath.getMeasurement(), + measurementPath.getMeasurementSchema(), + measurementPath.getMeasurementAlias(), + measurementPath.getTagMap()))); } } Map>> deviceToTimeseriesSchemaInfo = - new HashMap<>(); - for(PartialPath devicePath: deviceToAlignedMap.keySet()){ - if(deviceToAlignedMap.get(devicePath)) { + new HashMap<>(); + for (PartialPath devicePath : deviceToAlignedMap.keySet()) { + if (deviceToAlignedMap.get(devicePath)) { List measurementList = deviceToMeasurementMap.get(devicePath); List schemaList = deviceToMeasurementSchemaMap.get(devicePath); - List timeseriesSchemaInfoList = deviceToTimeseriesSchemaMap.get(devicePath); - AlignedPath alignedPath = new AlignedPath(devicePath.getNodes(), measurementList, schemaList); + List timeseriesSchemaInfoList = + deviceToTimeseriesSchemaMap.get(devicePath); + AlignedPath alignedPath = + new AlignedPath(devicePath.getNodes(), measurementList, schemaList); deviceToTimeseriesSchemaInfo - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(alignedPath, timeseriesSchemaInfoList); - }else{ - for(String measurement: deviceToMeasurementMap.get(devicePath)){ - MeasurementPath measurementPath = new MeasurementPath(devicePath.concatNode(measurement).getNodes()); + .computeIfAbsent(devicePath, k -> new HashMap<>()) + .put(alignedPath, timeseriesSchemaInfoList); + } else { + for (String measurement : deviceToMeasurementMap.get(devicePath)) { + MeasurementPath measurementPath = + new MeasurementPath(devicePath.concatNode(measurement).getNodes()); deviceToTimeseriesSchemaInfo - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(measurementPath, Collections.singletonList(new TimeseriesSchemaInfo(new MeasurementSchemaInfo(measurement, deviceToMeasurementSchemaMap.get(devicePath).get(0), null, null)))); + .computeIfAbsent(devicePath, k -> new HashMap<>()) + .put( + measurementPath, + Collections.singletonList( + new TimeseriesSchemaInfo( + new MeasurementSchemaInfo( + measurement, + deviceToMeasurementSchemaMap.get(devicePath).get(0), + null, + null)))); } } } analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesSchemaInfo); // fetch Data partition - DataPartition dataPartition = fetchDataPartitionByDevices(deviceToAlignedMap.keySet().stream().map(PartialPath::getFullPath).collect(Collectors.toSet()), schemaTree, context); + DataPartition dataPartition = + fetchDataPartitionByDevices( + deviceToAlignedMap.keySet().stream() + .map(PartialPath::getFullPath) + .collect(Collectors.toSet()), + schemaTree, + context); analysis.setDataPartitionInfo(dataPartition); return true; } @@ -2934,7 +2963,11 @@ public Analysis visitShowTimeSeries( // If there is time condition in SHOW TIMESERIES, we need to scan the raw data boolean hasSchema = analyzeTimeseriesRegionScan( - showTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context, showTimeSeriesStatement.getAuthorityScope()); + showTimeSeriesStatement.getTimeCondition(), + patternTree, + analysis, + context, + showTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader()); return analysis; @@ -3018,14 +3051,17 @@ private void removeLogicViewMeasurement(ISchemaTree schemaTree) { private void analyzeDeviceRegionScan( WhereCondition timeCondition, - PathPatternTree patternTree, + PartialPath pattern, + PathPatternTree authorityScope, Analysis analysis, MPPQueryContext context) { // If there is time condition in SHOW DEVICES, we need to scan the raw data analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); - - ISchemaTree schemaTree = schemaFetcher.fetchSchemaInDeviceLevel(patternTree, context); + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(pattern); + ISchemaTree schemaTree = + schemaFetcher.fetchSchemaInDeviceLevel(patternTree, authorityScope, context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return; @@ -3054,14 +3090,17 @@ public Analysis visitShowDevices( Analysis analysis = new Analysis(); analysis.setStatement(showDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); - if (showDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - showDevicesStatement.getTimeCondition(), patternTree, analysis, context); + showDevicesStatement.getTimeCondition(), + showDevicesStatement.getPathPattern(), + showDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } @@ -3095,13 +3134,34 @@ public Analysis visitCountStorageGroup( } @Override - public Analysis visitSchemaFetch( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + public Analysis visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); - analysis.setStatement(schemaFetchStatement); + analysis.setStatement(seriesSchemaFetchStatement); SchemaPartition schemaPartition = - partitionFetcher.getSchemaPartition(schemaFetchStatement.getPatternTree()); + partitionFetcher.getSchemaPartition(seriesSchemaFetchStatement.getPatternTree()); + analysis.setSchemaPartitionInfo(schemaPartition); + + if (schemaPartition.isEmpty()) { + analysis.setFinishQueryAfterAnalyze(true); + } + + return analysis; + } + + @Override + public Analysis visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) { + Analysis analysis = new Analysis(); + analysis.setStatement(deviceSchemaFetchStatement); + + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath path : deviceSchemaFetchStatement.getPaths()) { + patternTree.appendPathPattern(path); + } + patternTree.constructTree(); + SchemaPartition schemaPartition = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartition); if (schemaPartition.isEmpty()) { @@ -3117,13 +3177,17 @@ public Analysis visitCountDevices( Analysis analysis = new Analysis(); analysis.setStatement(countDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); if (countDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - countDevicesStatement.getTimeCondition(), patternTree, analysis, context); + countDevicesStatement.getTimeCondition(), + countDevicesStatement.getPathPattern(), + countDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } @@ -3145,7 +3209,11 @@ public Analysis visitCountTimeSeries( try { boolean hasSchema = analyzeTimeseriesRegionScan( - countTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context, countTimeSeriesStatement.getAuthorityScope()); + countTimeSeriesStatement.getTimeCondition(), + patternTree, + analysis, + context, + countTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getCountTimeSeriesHeader()); return analysis; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java index 84b93546cdcf8..ce61926b8faf2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java @@ -34,7 +34,8 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.statement.Statement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SetThreadName; @@ -107,7 +108,7 @@ ClusterSchemaTree fetchSchemaOfFuzzyMatch( templateMap.putAll(templateManager.checkAllRelatedTemplate(pattern)); } return executeSchemaFetchQuery( - new SchemaFetchStatement(patternTree, templateMap, withTags, withTemplate), context); + new SeriesSchemaFetchStatement(patternTree, templateMap, withTags, withTemplate), context); } /** @@ -125,7 +126,7 @@ ClusterSchemaTree fetchSchemaOfPreciseMatchOrPreciseDeviceUsingTemplate( MPPQueryContext context) { ClusterSchemaTree schemaTree = executeSchemaFetchQuery( - new SchemaFetchStatement( + new SeriesSchemaFetchStatement( rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), context); if (!schemaTree.isEmpty()) { @@ -134,6 +135,12 @@ rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), return schemaTree; } + ClusterSchemaTree fetchDeviceLevelSchema( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return executeSchemaFetchQuery( + new DeviceSchemaFetchStatement(patternTree, authorityScope), context); + } + ClusterSchemaTree fetchSchemaOfOneDevice( PartialPath devicePath, String[] measurements, @@ -189,7 +196,7 @@ private ClusterSchemaTree fetchSchemaAndCacheResult( PathPatternTree patternTree, MPPQueryContext context) { ClusterSchemaTree schemaTree = executeSchemaFetchQuery( - new SchemaFetchStatement( + new SeriesSchemaFetchStatement( patternTree, analyzeTemplate(patternTree.getAllPathPatterns()), false, true), context); if (!schemaTree.isEmpty()) { @@ -207,11 +214,11 @@ private Map analyzeTemplate(List pathPatternList } private ClusterSchemaTree executeSchemaFetchQuery( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + Statement fetchStatement, MPPQueryContext context) { long queryId = SessionManager.getInstance().requestQueryId(); Throwable t = null; try { - ExecutionResult executionResult = executionStatement(queryId, schemaFetchStatement, context); + ExecutionResult executionResult = executionStatement(queryId, fetchStatement, context); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index d9fa5212e2e50..2c3aced4ba16c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -156,17 +156,17 @@ public ClusterSchemaTree fetchSchema( } @Override - public ISchemaTree fetchSchemaInDeviceLevel(PathPatternTree patternTree, MPPQueryContext context) { - patternTree.constructTree(); - - return ISchemaFetcher.super.fetchSchemaInDeviceLevel(patternTree, context); + public ISchemaTree fetchSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + authorityScope.constructTree(); + return clusterSchemaFetchExecutor.fetchDeviceLevelSchema(patternTree, authorityScope, context); } @Override public ClusterSchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { patternTree.constructTree(); - return clusterSchemaFetchExecutor.fetchSchemaOfFuzzyMatch( + return clusterSchemaFetchExecutor.fetchSchemaOfFuzzyMatch( patternTree, true, withTemplate, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java index b723b2847f325..080e93b63532a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java @@ -39,8 +39,9 @@ public interface ISchemaFetcher { /** - * Fetch all the schema of existing timeseries matched by the given patternTree. The result schema tree may contain extra measurement nodes if - * withTemplate is true. It recommends to use pattern to filter the result schema tree. + * Fetch all the schema of existing timeseries matched by the given patternTree. The result schema + * tree may contain extra measurement nodes if withTemplate is true. It recommends to use pattern + * to filter the result schema tree. * * @param patternTree used for matching the timeseries * @param withTemplate whether to include the template in the schema @@ -50,20 +51,12 @@ ISchemaTree fetchSchema( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context); /** - * TODO need to be implemented in schema engine - * - *

Fetch all the schema by the given patternTree in device level + * Fetch all the schema by the given patternTree in device level * * @return schemaTree without measurement nodes */ - default ISchemaTree fetchSchemaInDeviceLevel( - PathPatternTree patternTree, MPPQueryContext context) { - ISchemaTree schemaTree = fetchSchema(patternTree, false, context); - if (schemaTree.hasLogicalViewMeasurement()) { - schemaTree.removeLogicalView(); - } - return schemaTree; - } + ISchemaTree fetchSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context); /** * Fetch all the schema with tags of existing timeseries matched by the given patternTree diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 5e614e16e28ff..347826b8fd68e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -52,9 +53,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; @@ -1084,7 +1085,7 @@ public LogicalPlanBuilder planSchemaFetchMerge(List storageGroupList) { } @SuppressWarnings({"checkstyle:Indentation", "checkstyle:CommentsIndentation"}) - public LogicalPlanBuilder planSchemaFetchSource( + public LogicalPlanBuilder planSeriesSchemaFetchSource( List storageGroupList, PathPatternTree patternTree, Map templateMap, @@ -1102,7 +1103,7 @@ public LogicalPlanBuilder planSchemaFetchSource( overlappedPatternTree.appendFullPath(pathPattern); } this.root.addChild( - new SchemaFetchScanNode( + new SeriesSchemaFetchScanNode( context.getQueryId().genPlanNodeId(), storageGroupPath, overlappedPatternTree, @@ -1117,6 +1118,34 @@ public LogicalPlanBuilder planSchemaFetchSource( return this; } + @SuppressWarnings({"checkstyle:Indentation", "checkstyle:CommentsIndentation"}) + public LogicalPlanBuilder planDeviceSchemaFetchSource( + List storageGroupList, PathPatternTree patternTree, PathPatternTree authorityScope) { + PartialPath storageGroupPath; + for (String storageGroup : storageGroupList) { + try { + storageGroupPath = new PartialPath(storageGroup); + PathPatternTree overlappedPatternTree = new PathPatternTree(); + for (PartialPath pathPattern : + patternTree.getOverlappedPathPatterns( + storageGroupPath.concatNode(MULTI_LEVEL_PATH_WILDCARD))) { + // pathPattern has been deduplicated, no need to deduplicate again + overlappedPatternTree.appendFullPath(pathPattern); + } + this.root.addChild( + new DeviceSchemaFetchScanNode( + context.getQueryId().genPlanNodeId(), + storageGroupPath, + overlappedPatternTree, + authorityScope)); + } catch (IllegalPathException e) { + // definitely won't happen + throw new RuntimeException(e); + } + } + return this; + } + public LogicalPlanBuilder planCountMerge() { CountSchemaMergeNode countMergeNode = new CountSchemaMergeNode(context.getQueryId().genPlanNodeId()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 3bec19321d3c8..1a6bddb422a31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -61,10 +61,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountLevelTimeSeriesStatement; @@ -776,19 +777,34 @@ public PlanNode visitInsertRowsOfOneDevice( } @Override - public PlanNode visitSchemaFetch( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + public PlanNode visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) { LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); List storageGroupList = new ArrayList<>(analysis.getSchemaPartitionInfo().getSchemaPartitionMap().keySet()); return planBuilder .planSchemaFetchMerge(storageGroupList) - .planSchemaFetchSource( + .planSeriesSchemaFetchSource( storageGroupList, - schemaFetchStatement.getPatternTree(), - schemaFetchStatement.getTemplateMap(), - schemaFetchStatement.isWithTags(), - schemaFetchStatement.isWithTemplate()) + seriesSchemaFetchStatement.getPatternTree(), + seriesSchemaFetchStatement.getTemplateMap(), + seriesSchemaFetchStatement.isWithTags(), + seriesSchemaFetchStatement.isWithTemplate()) + .getRoot(); + } + + @Override + public PlanNode visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + List storageGroupList = + new ArrayList<>(analysis.getSchemaPartitionInfo().getSchemaPartitionMap().keySet()); + return planBuilder + .planSchemaFetchMerge(storageGroupList) + .planDeviceSchemaFetchSource( + storageGroupList, + deviceSchemaFetchStatement.getPatternTree(), + deviceSchemaFetchStatement.getAuthorityScope()) .getRoot(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 26a7ab2aa2766..abafd45ae9496 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -167,6 +167,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -177,10 +178,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; @@ -2593,8 +2594,8 @@ public Operator visitSchemaFetchMerge( } @Override - public Operator visitSchemaFetchScan( - SchemaFetchScanNode node, LocalExecutionPlanContext context) { + public Operator visitSeriesSchemaFetchScan( + SeriesSchemaFetchScanNode node, LocalExecutionPlanContext context) { OperatorContext operatorContext = context .getDriverContext() @@ -2602,7 +2603,7 @@ public Operator visitSchemaFetchScan( context.getNextOperatorId(), node.getPlanNodeId(), SchemaFetchScanOperator.class.getSimpleName()); - return new SchemaFetchScanOperator( + return SchemaFetchScanOperator.ofSeries( node.getPlanNodeId(), operatorContext, node.getPatternTree(), @@ -2612,6 +2613,24 @@ public Operator visitSchemaFetchScan( node.isWithTemplate()); } + @Override + public Operator visitDeviceSchemaFetchScan( + DeviceSchemaFetchScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaFetchScanOperator.class.getSimpleName()); + return SchemaFetchScanOperator.ofDevice( + node.getPlanNodeId(), + operatorContext, + node.getPatternTree(), + node.getAuthorityScope(), + ((SchemaDriverContext) (context.getDriverContext())).getSchemaRegion()); + } + @Override public Operator visitLastQueryScan(LastQueryScanNode node, LocalExecutionPlanContext context) { PartialPath seriesPath = node.getSeriesPath().transformToPartialPath(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index f261acfb65c26..e5983b6a18ce3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -28,11 +28,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.AbstractSchemaMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; @@ -158,7 +159,14 @@ public PlanNode visitSchemaQueryScan(SchemaQueryScanNode node, NodeGroupContext } @Override - public PlanNode visitSchemaFetchScan(SchemaFetchScanNode node, NodeGroupContext context) { + public PlanNode visitSeriesSchemaFetchScan( + SeriesSchemaFetchScanNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + + @Override + public PlanNode visitDeviceSchemaFetchScan( + DeviceSchemaFetchScanNode node, NodeGroupContext context) { return processNoChildSourceNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java index 63d59ccaec7ed..14d810514c893 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; @@ -894,7 +895,8 @@ public List visitSchemaFetchMerge( for (TRegionReplicaSet schemaRegion : storageGroupSchemaRegionMap.get( ((SchemaFetchScanNode) child).getStorageGroup().getFullPath())) { - SchemaFetchScanNode schemaFetchScanNode = (SchemaFetchScanNode) child.clone(); + SchemaFetchScanNode schemaFetchScanNode = + (SchemaFetchScanNode) child.clone(); schemaFetchScanNode.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId()); schemaFetchScanNode.setRegionReplicaSet(schemaRegion); root.addChild(schemaFetchScanNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index b546516be3880..c1432bf14512d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -31,9 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -142,7 +143,7 @@ public enum PlanNodeType { ALTER_TIME_SERIES((short) 21), CREATE_ALIGNED_TIME_SERIES((short) 22), TIME_SERIES_SCHEMA_SCAN((short) 23), - SCHEMA_FETCH_SCAN((short) 24), + SERIES_SCHEMA_FETCH_SCAN((short) 24), SCHEMA_QUERY_MERGE((short) 25), SCHEMA_QUERY_ORDER_BY_HEAT((short) 26), DEVICES_COUNT((short) 27), @@ -219,7 +220,9 @@ public enum PlanNodeType { DEVICE_REGION_SCAN((short) 93), TIMESERIES_REGION_SCAN((short) 94), - REGION_MERGE((short) 95); + REGION_MERGE((short) 95), + DEVICE_SCHEMA_FETCH_SCAN((short) 96), + ; public static final int BYTES = Short.BYTES; @@ -325,7 +328,7 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { case 23: return TimeSeriesSchemaScanNode.deserialize(buffer); case 24: - return SchemaFetchScanNode.deserialize(buffer); + return SeriesSchemaFetchScanNode.deserialize(buffer); case 25: return SchemaQueryMergeNode.deserialize(buffer); case 26: @@ -462,6 +465,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TimeseriesRegionScanNode.deserialize(buffer); case 95: return ActiveRegionScanMergeNode.deserialize(buffer); + case 96: + return DeviceSchemaFetchScanNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 81d7930dc9215..2654487da54cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -27,10 +28,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -385,7 +386,11 @@ public R visitSchemaFetchMerge(SchemaFetchMergeNode node, C context) { return visitPlan(node, context); } - public R visitSchemaFetchScan(SchemaFetchScanNode node, C context) { + public R visitSeriesSchemaFetchScan(SeriesSchemaFetchScanNode node, C context) { + return visitPlan(node, context); + } + + public R visitDeviceSchemaFetchScan(DeviceSchemaFetchScanNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java new file mode 100644 index 0000000000000..2e24512c8c2d8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class DeviceSchemaFetchScanNode extends SchemaFetchScanNode { + + private final PathPatternTree authorityScope; + + public DeviceSchemaFetchScanNode( + PlanNodeId id, + PartialPath storageGroup, + PathPatternTree patternTree, + PathPatternTree authorityScope) { + super(id, storageGroup, patternTree); + this.authorityScope = authorityScope; + this.authorityScope.constructTree(); + } + + + public PathPatternTree getAuthorityScope() { + return authorityScope; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN; + } + + @Override + public PlanNode clone() { + return new DeviceSchemaFetchScanNode( + getPlanNodeId(), storageGroup, patternTree, authorityScope); + } + + @Override + public String toString() { + return String.format( + "DeviceSchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", + this.getPlanNodeId(), + storageGroup, + PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN.serialize(byteBuffer); + storageGroup.serialize(byteBuffer); + patternTree.serialize(byteBuffer); + authorityScope.serialize(byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN.serialize(stream); + storageGroup.serialize(stream); + patternTree.serialize(stream); + authorityScope.serialize(stream); + } + + public static DeviceSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { + PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); + PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); + PathPatternTree authorityScope = PathPatternTree.deserialize(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new DeviceSchemaFetchScanNode(planNodeId, storageGroup, patternTree, authorityScope); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitDeviceSchemaFetchScan(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java index e68aebef1650d..d0598b735bc44 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java @@ -19,177 +19,74 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; +import com.google.common.collect.ImmutableList; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; -import org.apache.iotdb.db.schemaengine.template.Template; - -import com.google.common.collect.ImmutableList; -import org.apache.tsfile.utils.ReadWriteIOUtils; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -/** This class defines the scan task of schema fetcher. */ -public class SchemaFetchScanNode extends SourceNode { - - private final PartialPath storageGroup; - private final PathPatternTree patternTree; - private final Map templateMap; - private final boolean withTags; - private final boolean withTemplate; - private TRegionReplicaSet schemaRegionReplicaSet; - - public SchemaFetchScanNode( - PlanNodeId id, - PartialPath storageGroup, - PathPatternTree patternTree, - Map templateMap, - boolean withTags, - boolean withTemplate) { - super(id); - this.storageGroup = storageGroup; - this.patternTree = patternTree; - this.patternTree.constructTree(); - this.templateMap = templateMap; - this.withTags = withTags; - this.withTemplate = withTemplate; - } - - public PartialPath getStorageGroup() { - return storageGroup; - } - - public PathPatternTree getPatternTree() { - return patternTree; - } - - public Map getTemplateMap() { - return templateMap; - } - - @Override - public List getChildren() { - return Collections.emptyList(); - } - - @Override - public void addChild(PlanNode child) {} - - @Override - public PlanNodeType getType() { - return PlanNodeType.SCHEMA_FETCH_SCAN; - } - - @Override - public PlanNode clone() { - return new SchemaFetchScanNode( - getPlanNodeId(), storageGroup, patternTree, templateMap, withTags, withTemplate); - } - - @Override - public int allowedChildCount() { - return 0; - } - - @Override - public List getOutputColumnNames() { - return ImmutableList.of(); - } - - @Override - public String toString() { - return String.format( - "SchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", - this.getPlanNodeId(), - storageGroup, - PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); - } - - @Override - protected void serializeAttributes(ByteBuffer byteBuffer) { - PlanNodeType.SCHEMA_FETCH_SCAN.serialize(byteBuffer); - storageGroup.serialize(byteBuffer); - patternTree.serialize(byteBuffer); - - ReadWriteIOUtils.write(templateMap.size(), byteBuffer); - for (Template template : templateMap.values()) { - template.serialize(byteBuffer); + +public abstract class SchemaFetchScanNode extends SourceNode { + protected final PartialPath storageGroup; + protected final PathPatternTree patternTree; + protected TRegionReplicaSet schemaRegionReplicaSet; + + protected SchemaFetchScanNode( + PlanNodeId id, + PartialPath storageGroup, + PathPatternTree patternTree){ + super(id); + this.storageGroup = storageGroup; + this.patternTree = patternTree; + this.patternTree.constructTree(); + } + + + public PartialPath getStorageGroup() { + return storageGroup; + } + + public PathPatternTree getPatternTree() { + return patternTree; + } + + @Override + public List getChildren() { + return Collections.emptyList(); } - ReadWriteIOUtils.write(withTags, byteBuffer); - ReadWriteIOUtils.write(withTemplate, byteBuffer); - } - - @Override - protected void serializeAttributes(DataOutputStream stream) throws IOException { - PlanNodeType.SCHEMA_FETCH_SCAN.serialize(stream); - storageGroup.serialize(stream); - patternTree.serialize(stream); - ReadWriteIOUtils.write(templateMap.size(), stream); - for (Template template : templateMap.values()) { - template.serialize(stream); + + @Override + public void addChild(PlanNode child) {} + + @Override + public int allowedChildCount() { + return 0; } - ReadWriteIOUtils.write(withTags, stream); - ReadWriteIOUtils.write(withTemplate, stream); - } - - public static SchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { - PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); - PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); - - int templateNum = ReadWriteIOUtils.readInt(byteBuffer); - Map templateMap = new HashMap<>(); - Template template; - for (int i = 0; i < templateNum; i++) { - template = new Template(); - template.deserialize(byteBuffer); - templateMap.put(template.getId(), template); + + @Override + public List getOutputColumnNames() { + return ImmutableList.of(); } - boolean withTags = ReadWriteIOUtils.readBool(byteBuffer); - boolean withTemplate = ReadWriteIOUtils.readBool(byteBuffer); - PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); - return new SchemaFetchScanNode( - planNodeId, storageGroup, patternTree, templateMap, withTags, withTemplate); - } - - @Override - public void open() throws Exception {} - - @Override - public TRegionReplicaSet getRegionReplicaSet() { - return schemaRegionReplicaSet; - } - - @Override - public void setRegionReplicaSet(TRegionReplicaSet schemaRegionReplicaSet) { - this.schemaRegionReplicaSet = schemaRegionReplicaSet; - } - - @Override - public void close() throws Exception {} - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitSchemaFetchScan(this, context); - } - - public boolean isWithTags() { - return withTags; - } - - public boolean isWithTemplate() { - return withTemplate; - } + + + @Override + public void open() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public void setRegionReplicaSet(TRegionReplicaSet schemaRegionReplicaSet) { + this.schemaRegionReplicaSet = schemaRegionReplicaSet; + } + + @Override + public void close() throws Exception {} + } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java new file mode 100644 index 0000000000000..5884252cd60f9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; +import org.apache.iotdb.db.schemaengine.template.Template; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** This class defines the scan task of schema fetcher. */ +public class SeriesSchemaFetchScanNode extends SchemaFetchScanNode { + + private final Map templateMap; + private final boolean withTags; + private final boolean withTemplate; + + public SeriesSchemaFetchScanNode( + PlanNodeId id, + PartialPath storageGroup, + PathPatternTree patternTree, + Map templateMap, + boolean withTags, + boolean withTemplate) { + super(id, storageGroup, patternTree); + this.templateMap = templateMap; + this.withTags = withTags; + this.withTemplate = withTemplate; + } + + + public Map getTemplateMap() { + return templateMap; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.SERIES_SCHEMA_FETCH_SCAN; + } + + @Override + public PlanNode clone() { + return new SeriesSchemaFetchScanNode( + getPlanNodeId(), storageGroup, patternTree, templateMap, withTags, withTemplate); + } + + @Override + public String toString() { + return String.format( + "SeriesSchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", + this.getPlanNodeId(), + storageGroup, + PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.SERIES_SCHEMA_FETCH_SCAN.serialize(byteBuffer); + storageGroup.serialize(byteBuffer); + patternTree.serialize(byteBuffer); + + ReadWriteIOUtils.write(templateMap.size(), byteBuffer); + for (Template template : templateMap.values()) { + template.serialize(byteBuffer); + } + ReadWriteIOUtils.write(withTags, byteBuffer); + ReadWriteIOUtils.write(withTemplate, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.SERIES_SCHEMA_FETCH_SCAN.serialize(stream); + storageGroup.serialize(stream); + patternTree.serialize(stream); + ReadWriteIOUtils.write(templateMap.size(), stream); + for (Template template : templateMap.values()) { + template.serialize(stream); + } + ReadWriteIOUtils.write(withTags, stream); + ReadWriteIOUtils.write(withTemplate, stream); + } + + public static SeriesSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { + PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); + PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); + + int templateNum = ReadWriteIOUtils.readInt(byteBuffer); + Map templateMap = new HashMap<>(); + Template template; + for (int i = 0; i < templateNum; i++) { + template = new Template(); + template.deserialize(byteBuffer); + templateMap.put(template.getId(), template); + } + boolean withTags = ReadWriteIOUtils.readBool(byteBuffer); + boolean withTemplate = ReadWriteIOUtils.readBool(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new SeriesSchemaFetchScanNode( + planNodeId, storageGroup, patternTree, templateMap, withTags, withTemplate); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitSeriesSchemaFetchScan(this, context); + } + + public boolean isWithTags() { + return withTags; + } + + public boolean isWithTemplate() { + return withTemplate; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 248cf881de12b..c0ba988e38cc0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -28,10 +28,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; @@ -356,8 +357,14 @@ public R visitCountNodes(CountNodesStatement countStatement, C context) { return visitStatement(countStatement, context); } - public R visitSchemaFetch(SchemaFetchStatement schemaFetchStatement, C context) { - return visitStatement(schemaFetchStatement, context); + public R visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, C context) { + return visitStatement(seriesSchemaFetchStatement, context); + } + + public R visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, C context) { + return visitStatement(deviceSchemaFetchStatement, context); } public R visitShowChildPaths(ShowChildPathsStatement showChildPathsStatement, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java new file mode 100644 index 0000000000000..00271f1909786 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.internal; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.List; + +public class DeviceSchemaFetchStatement extends Statement { + private final PathPatternTree patternTree; + private final PathPatternTree authorityScope; + + public DeviceSchemaFetchStatement(PathPatternTree patternTree, PathPatternTree authorityScope) { + super(); + this.patternTree = patternTree; + this.authorityScope = authorityScope; + setType(StatementType.FETCH_SCHEMA); + } + + public PathPatternTree getPatternTree() { + return patternTree; + } + + public PathPatternTree getAuthorityScope() { + return authorityScope; + } + + @Override + public List getPaths() { + return patternTree.getAllPathPatterns(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitDeviceSchemaFetch(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java similarity index 93% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java index f9b3047e57006..be48b631bba95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java @@ -29,14 +29,14 @@ import java.util.List; import java.util.Map; -public class SchemaFetchStatement extends Statement { +public class SeriesSchemaFetchStatement extends Statement { private final PathPatternTree patternTree; private final Map templateMap; private final boolean withTags; private final boolean withTemplate; - public SchemaFetchStatement( + public SeriesSchemaFetchStatement( PathPatternTree patternTree, Map templateMap, boolean withTags, @@ -59,7 +59,7 @@ public Map getTemplateMap() { @Override public R accept(StatementVisitor visitor, C context) { - return visitor.visitSchemaFetch(this, context); + return visitor.visitSeriesSchemaFetch(this, context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 95e3524d32157..833e83e10b2e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -198,13 +198,24 @@ void checkSchemaQuota(PartialPath devicePath, int timeSeriesNum) MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws MetadataException; - ClusterSchemaTree fetchSchema( + ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, boolean withTemplate) throws MetadataException; + /** + * Fetch all the schema by the given patternTree in device level. Currently, devices with + * isAligned!=null will be filtered out. + * + * @param patternTree devices' pattern + * @param authorityScope the scope of the authority + * @return pattern tree with all leaves as device nodes + */ + ClusterSchemaTree fetchDeviceSchema(PathPatternTree patternTree, PathPatternTree authorityScope) + throws MetadataException; + // endregion // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index d5b7f58cfd4b7..12225a29dc011 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -949,7 +949,7 @@ public MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws Metadat } @Override - public ClusterSchemaTree fetchSchema( + public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, @@ -966,6 +966,12 @@ public ClusterSchemaTree fetchSchema( } } + @Override + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + return mtree.fetchDeviceSchema(patternTree, authorityScope); + } + // endregion // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index ad3ee7a223bd1..13da1fae890ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1027,7 +1027,7 @@ public MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws Metadat } @Override - public ClusterSchemaTree fetchSchema( + public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, @@ -1044,6 +1044,12 @@ public ClusterSchemaTree fetchSchema( } } + @Override + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + return mtree.fetchDeviceSchema(patternTree, authorityScope); + } + // endregion // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index e76c3931d6b48..be7adafe82e85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -764,6 +764,26 @@ protected Void collectMeasurement(IMeasurementMNode node) { return schemaTree; } + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + for (PartialPath pattern : patternTree.getAllPathPatterns()) { + try (EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, authorityScope) { + @Override + protected Void collectEntity(IDeviceMNode node) { + if (node.isAlignedNullable() != null) { + schemaTree.appendDevice(node.getPartialPath(), node.isAligned()); + } + return null; + } + }) { + collector.traverse(); + } + } + return schemaTree; + } + // endregion // region Interfaces and Implementation for MNode Query diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index d2ef703ee7f25..ab36b2abd2f23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -889,6 +889,26 @@ protected Void collectMeasurement(IMeasurementMNode node) { return schemaTree; } + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + for (PartialPath pattern : patternTree.getAllPathPatterns()) { + try (EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, authorityScope) { + @Override + protected Void collectEntity(IDeviceMNode node) { + if (node.isAlignedNullable() != null) { + schemaTree.appendDevice(node.getPartialPath(), node.isAligned()); + } + return null; + } + }) { + collector.traverse(); + } + } + return schemaTree; + } + // endregion // region Interfaces and Implementation for MNode Query diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 4064376cd96f0..18ebf38343b83 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -103,11 +103,11 @@ public void testFetchSchemaPerfomance() throws Exception { } } patternTree.constructTree(); - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); long startTime; startTime = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); } System.out.println("cost time: " + (System.currentTimeMillis() - startTime)); } @@ -153,7 +153,7 @@ public void testFetchSchema() throws Exception { patternTree.constructTree(); ; ClusterSchemaTree schemas = - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, true, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, true, true); List measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.*")).left; Assert.assertEquals(measurementPaths.size(), 2); @@ -182,7 +182,7 @@ public void testFetchSchema() throws Exception { patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.wf01.wt01.temp")); patternTree.constructTree(); - schemas = schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemas = schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.temp")).left; Assert.assertEquals(measurementPaths.size(), 1); @@ -368,7 +368,7 @@ public void testDeleteTimeseries() throws Exception { schemaRegion.deleteTimeseriesInBlackList(patternTree); List schemas = schemaRegion - .fetchSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, true) + .fetchSeriesSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, true) .searchMeasurementPaths(ALL_MATCH_PATTERN) .left; Assert.assertEquals(1, schemas.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index d85d203ec64d7..71b5e7dd38432 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -84,14 +84,14 @@ public void testRecoverWithAlignedTemplate() throws Exception { new PartialPath("root.sg.d1"), 2, templateId), template); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( + schemaRegion.fetchSeriesSchema( ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); simulateRestart(); schemaRegion = getSchemaRegion("root.sg", 0); schemaTree = - schemaRegion.fetchSchema( + schemaRegion.fetchSeriesSchema( ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java index 3aa6d9019fdbf..0747c19c72623 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java @@ -77,7 +77,7 @@ public void testFetchNestedTemplateDevice() throws Exception { new PartialPath("root.sg.d1.GPS"), 3, templateId), template); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( + schemaRegion.fetchSeriesSchema( ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); Assert.assertEquals(2, schemaTree.getAllDevices().size()); for (DeviceSchemaInfo deviceSchemaInfo : schemaTree.getAllDevices()) { @@ -213,7 +213,7 @@ public void testFetchSchemaWithTemplate() throws Exception { // check fetch schema ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema(ALL_MATCH_SCOPE, templateMap, true, true); + schemaRegion.fetchSeriesSchema(ALL_MATCH_SCOPE, templateMap, true, true); schemaTree.setTemplateMap(templateMap); List schemas = schemaTree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(expectedTimeseries.size(), schemas.size()); @@ -258,7 +258,7 @@ public void testDeleteSchemaWithTemplate() throws Exception { patternTree.appendFullPath(new PartialPath("root.db.d1.s1")); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( + schemaRegion.fetchSeriesSchema( patternTree, Collections.singletonMap(templateId, template), false, true); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( @@ -266,7 +266,7 @@ public void testDeleteSchemaWithTemplate() throws Exception { patternTree = new PathPatternTree(); patternTree.appendFullPath(new PartialPath("root.db.d1.s3")); schemaTree = - schemaRegion.fetchSchema( + schemaRegion.fetchSeriesSchema( patternTree, Collections.singletonMap(templateId, template), false, true); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index 52c5a71a46f05..7812e7cddedea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -832,7 +832,7 @@ public void SchemaFetchScanOperatorTest() { driverContext.addOperatorContext(1, planNodeId, SeriesScanOperator.class.getSimpleName()); SchemaFetchScanOperator operator = - new SchemaFetchScanOperator( + SchemaFetchScanOperator.ofSeries( planNodeId, driverContext.getOperatorContexts().get(0), null, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java index 8286a11c51a22..4b694d53fa92b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java @@ -59,7 +59,7 @@ public void testSchemaFetchResult() throws Exception { patternTree.constructTree(); SchemaFetchScanOperator schemaFetchScanOperator = - new SchemaFetchScanOperator( + SchemaFetchScanOperator.ofSeries( null, null, patternTree, Collections.emptyMap(), schemaRegion, false, true); Assert.assertTrue(schemaFetchScanOperator.hasNext()); @@ -151,7 +151,7 @@ private ISchemaRegion mockSchemaRegion() throws Exception { patternTree.appendPathPattern(new PartialPath("root.**.status")); patternTree.appendPathPattern(new PartialPath("root.**.s1")); patternTree.constructTree(); - Mockito.when(schemaRegion.fetchSchema(patternTree, Collections.emptyMap(), false, true)) + Mockito.when(schemaRegion.fetchSeriesSchema(patternTree, Collections.emptyMap(), false, true)) .thenReturn(clusterSchemaTree); return schemaRegion; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java new file mode 100644 index 0000000000000..ca52035bb5049 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node.metadata.read; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class DeviceSchemaFetchScanNodeTest { + + @Test + public void testSerialization() throws IllegalPathException { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); + DeviceSchemaFetchScanNode deviceSchemaFetchScanNodeTest = + new DeviceSchemaFetchScanNode( + new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, patternTree); + ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); + deviceSchemaFetchScanNodeTest.serialize(byteBuffer); + byteBuffer.flip(); + DeviceSchemaFetchScanNode recoveredNode = + (DeviceSchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); + Assert.assertEquals("root.sg", recoveredNode.getStorageGroup().getFullPath()); + Assert.assertEquals( + "root.sg.**.*", recoveredNode.getPatternTree().getAllPathPatterns().get(0).getFullPath()); + Assert.assertEquals( + "root.sg.**.*", + recoveredNode.getAuthorityScope().getAllPathPatterns().get(0).getFullPath()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java index c242778efab67..45a75fdf58fde 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.junit.Assert; import org.junit.Test; @@ -42,15 +42,15 @@ public void testSerialization() throws IllegalPathException { new SchemaFetchMergeNode(new PlanNodeId("0"), Arrays.asList("root.db1", "root.db2")); PathPatternTree patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); - SchemaFetchScanNode schemaFetchScanNode = - new SchemaFetchScanNode( + SeriesSchemaFetchScanNode seriesSchemaFetchScanNode = + new SeriesSchemaFetchScanNode( new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, Collections.emptyMap(), true, true); - schemaFetchMergeNode.addChild(schemaFetchScanNode); + schemaFetchMergeNode.addChild(seriesSchemaFetchScanNode); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); schemaFetchMergeNode.serialize(byteBuffer); byteBuffer.flip(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java similarity index 85% rename from iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java rename to iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java index 0d45a127f1766..ffa00cf8bdc90 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.junit.Assert; import org.junit.Test; @@ -32,14 +32,14 @@ import java.nio.ByteBuffer; import java.util.Collections; -public class SchemaFetchScanNodeTest { +public class SeriesSchemaFetchScanNodeTest { @Test public void testSerialization() throws IllegalPathException { PathPatternTree patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); - SchemaFetchScanNode schemaFetchScanNode = - new SchemaFetchScanNode( + SeriesSchemaFetchScanNode seriesSchemaFetchScanNode = + new SeriesSchemaFetchScanNode( new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, @@ -47,9 +47,10 @@ public void testSerialization() throws IllegalPathException { true, true); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); - schemaFetchScanNode.serialize(byteBuffer); + seriesSchemaFetchScanNode.serialize(byteBuffer); byteBuffer.flip(); - SchemaFetchScanNode recoveredNode = (SchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); + SeriesSchemaFetchScanNode recoveredNode = + (SeriesSchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); Assert.assertEquals("root.sg", recoveredNode.getStorageGroup().getFullPath()); Assert.assertEquals( "root.sg.**.*", recoveredNode.getPatternTree().getAllPathPatterns().get(0).getFullPath()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java index 2fa1b16f2663b..ac124fe67ccf9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java @@ -22,6 +22,11 @@ import org.apache.iotdb.commons.schema.SchemaConstant; public class PathPatternTreeUtils { + + private PathPatternTreeUtils() { + // hidden constructor + } + /** * Intersect the pattern tree with the full path prefix tree, and return the intersected pattern * tree. From 217bf7f212a62357d62ab6b9a24ba6eacad7197e Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 15:48:25 +0800 Subject: [PATCH 03/12] save --- .../db/queryengine/common/DeviceContext.java | 81 ++++++++++++ ...SchemaInfo.java => TimeseriesContext.java} | 12 +- .../common/schematree/ClusterSchemaTree.java | 35 +----- .../execution/driver/DataDriverContext.java | 19 +-- .../fragment/FragmentInstanceContext.java | 17 +-- .../schema/source/DeviceSchemaSource.java | 4 +- .../ActiveDeviceRegionScanOperator.java | 48 ++++++-- .../ActiveTimeSeriesRegionScanOperator.java | 12 +- .../source/RegionScanForActiveDeviceUtil.java | 3 +- .../RegionScanForActiveTimeSeriesUtil.java | 6 +- .../db/queryengine/plan/analyze/Analysis.java | 19 +-- .../plan/analyze/AnalyzeVisitor.java | 29 +++-- .../plan/planner/LocalExecutionPlanner.java | 12 +- .../plan/planner/LogicalPlanBuilder.java | 9 +- .../plan/planner/LogicalPlanVisitor.java | 4 +- .../plan/planner/OperatorTreeGenerator.java | 29 ++--- .../planner/distribution/SourceRewriter.java | 4 +- .../read/DeviceSchemaFetchScanNode.java | 7 -- .../metedata/read/SchemaFetchScanNode.java | 116 +++++++++--------- .../read/SeriesSchemaFetchScanNode.java | 6 - .../node/source/DeviceRegionScanNode.java | 46 +++---- .../node/source/TimeseriesRegionScanNode.java | 37 +++--- .../impl/mem/MTreeBelowSGMemoryImpl.java | 3 +- .../impl/pbtree/MTreeBelowSGCachedImpl.java | 3 +- .../storageengine/dataregion/DataRegion.java | 7 +- .../dataregion/IDataRegionForQuery.java | 3 +- .../dataregion/VirtualDataRegion.java | 3 +- .../dataregion/memtable/TsFileProcessor.java | 7 +- .../logical/RegionScanLogicalPlannerTest.java | 88 +++++++------ 29 files changed, 379 insertions(+), 290 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/{TimeseriesSchemaInfo.java => TimeseriesContext.java} (94%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java new file mode 100644 index 0000000000000..3c6663c680b11 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.common; + +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DeviceContext { + private boolean isAligned; + private final int templateId; + + public DeviceContext(DeviceSchemaInfo deviceSchemaInfo) { + this.isAligned = deviceSchemaInfo.isAligned(); + this.templateId = deviceSchemaInfo.getTemplateId(); + } + + public DeviceContext(boolean isAligned, int templateId) { + this.isAligned = isAligned; + this.templateId = templateId; + } + + public boolean isAligned() { + return isAligned; + } + + public int getTemplateId() { + return templateId; + } + + public void serializeAttributes(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(isAligned, byteBuffer); + ReadWriteIOUtils.write(templateId, byteBuffer); + } + + public void serializeAttributes(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(isAligned, stream); + ReadWriteIOUtils.write(templateId, stream); + } + + public static DeviceContext deserialize(ByteBuffer buffer) { + boolean isAligned = ReadWriteIOUtils.readBool(buffer); + int templateId = ReadWriteIOUtils.readInt(buffer); + return new DeviceContext(isAligned, templateId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeviceContext that = (DeviceContext) o; + return isAligned == that.isAligned && templateId == that.templateId; + } + + @Override + public int hashCode() { + return Objects.hash(isAligned, templateId); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java similarity index 94% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 56dd0fa6a8078..6450c036128ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -32,7 +32,7 @@ import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; -public class TimeseriesSchemaInfo { +public class TimeseriesContext { private final String dataType; private final String encoding; private final String compression; @@ -45,7 +45,7 @@ public class TimeseriesSchemaInfo { private final String deadband; private final String deadbandParameters; - public TimeseriesSchemaInfo(IMeasurementSchemaInfo schemaInfo) { + public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { this.dataType = schemaInfo.getSchema().getType().toString(); this.encoding = schemaInfo.getSchema().getEncodingType().toString(); this.compression = schemaInfo.getSchema().getCompressor().toString(); @@ -85,7 +85,7 @@ public String getDeadband() { return deadband; } - public TimeseriesSchemaInfo( + public TimeseriesContext( String dataType, String alias, String encoding, @@ -122,7 +122,7 @@ public void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(deadbandParameters, stream); } - public static TimeseriesSchemaInfo deserialize(ByteBuffer buffer) { + public static TimeseriesContext deserialize(ByteBuffer buffer) { String dataType = ReadWriteIOUtils.readString(buffer); String alias = ReadWriteIOUtils.readString(buffer); String encoding = ReadWriteIOUtils.readString(buffer); @@ -130,7 +130,7 @@ public static TimeseriesSchemaInfo deserialize(ByteBuffer buffer) { String tags = ReadWriteIOUtils.readString(buffer); String deadband = ReadWriteIOUtils.readString(buffer); String deadbandParameters = ReadWriteIOUtils.readString(buffer); - return new TimeseriesSchemaInfo( + return new TimeseriesContext( dataType, alias, encoding, compression, tags, deadband, deadbandParameters); } @@ -142,7 +142,7 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - TimeseriesSchemaInfo that = (TimeseriesSchemaInfo) obj; + TimeseriesContext that = (TimeseriesContext) obj; return Objects.equals(dataType, that.dataType) && Objects.equals(alias, that.alias) && encoding.equals(that.encoding) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index 336afb0274569..f1d90875ac537 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -311,39 +311,8 @@ public void appendTemplateDevice( entityNode.setTemplateId(templateId); cur.replaceChild(deviceName, entityNode); } - templateMap.putIfAbsent(templateId, template); - } - - /** - * Append a template device to the schema tree. - * - * @param devicePath device path - * @param isAligned whether the device is aligned - */ - public void appendDevice(PartialPath devicePath, Boolean isAligned) { - String[] nodes = devicePath.getNodes(); - SchemaNode cur = root; - SchemaNode child; - for (int i = 1; i < nodes.length - 1; i++) { - child = cur.getChild(nodes[i]); - if (child == null) { - child = new SchemaInternalNode(nodes[i]); - cur.addChild(nodes[i], child); - } - cur = child; - } - String deviceName = nodes[nodes.length - 1]; - child = cur.getChild(deviceName); - if (child == null) { - SchemaEntityNode entityNode = new SchemaEntityNode(deviceName); - entityNode.setAligned(isAligned); - cur.addChild(deviceName, entityNode); - } else if (child.isEntity()) { - child.getAsEntityNode().setAligned(isAligned); - } else { - SchemaEntityNode entityNode = new SchemaEntityNode(deviceName); - entityNode.setAligned(isAligned); - cur.replaceChild(deviceName, entityNode); + if (template != null) { + templateMap.putIfAbsent(templateId, template); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java index c30c11549102a..3fc9598ef8af8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.operator.source.DataSourceOperator; import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery; @@ -39,7 +40,7 @@ public class DataDriverContext extends DriverContext { // it will be set to null, after being merged into Parent FIContext private List paths; private QueryDataSourceType queryDataSourceType = null; - private Map deviceIDToAligned; + private Map deviceIDToContext; // it will be set to null, after QueryDataSource being inited private List sourceOperators; @@ -47,27 +48,27 @@ public DataDriverContext(FragmentInstanceContext fragmentInstanceContext, int pi super(fragmentInstanceContext, pipelineId); this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); - this.deviceIDToAligned = null; + this.deviceIDToContext = null; } public DataDriverContext(DataDriverContext parentContext, int pipelineId) { super(parentContext.getFragmentInstanceContext(), pipelineId); this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); - this.deviceIDToAligned = null; + this.deviceIDToContext = null; } public void setQueryDataSourceType(QueryDataSourceType queryDataSourceType) { this.queryDataSourceType = queryDataSourceType; } - public void setDeviceIDToAligned(Map deviceIDToAligned) { - this.deviceIDToAligned = deviceIDToAligned; + public void setDeviceIDToContext(Map deviceIDToContext) { + this.deviceIDToContext = deviceIDToContext; } - public void clearDeviceIDToAligned() { + public void clearDeviceIDToContext() { // friendly for gc - deviceIDToAligned = null; + deviceIDToContext = null; } public void addPath(PartialPath path) { @@ -82,8 +83,8 @@ public List getPaths() { return paths; } - public Map getDeviceIDToAligned() { - return deviceIDToAligned; + public Map getDeviceIDToContext() { + return deviceIDToContext; } public Optional getQueryDataSourceType() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 72425e4df363a..3748372c4f61b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -68,7 +69,7 @@ public class FragmentInstanceContext extends QueryContext { // it will only be used once, after sharedQueryDataSource being inited, it will be set to null private List sourcePaths; // Used for region scan. - private Map devicePathsToAligned; + private Map devicePathsToContext; // Shared by all scan operators in this fragment instance to avoid memory problem private IQueryDataSource sharedQueryDataSource; @@ -356,8 +357,8 @@ public void setSourcePaths(List sourcePaths) { this.sourcePaths = sourcePaths; } - public void setDevicePathsToAligned(Map devicePathsToAligned) { - this.devicePathsToAligned = devicePathsToAligned; + public void setDevicePathsToContext(Map devicePathsToContext) { + this.devicePathsToContext = devicePathsToContext; } public void initQueryDataSource(List sourcePaths) throws QueryProcessException { @@ -399,17 +400,17 @@ public void initQueryDataSource(List sourcePaths) throws QueryProce } } - public void initRegionScanQueryDataSource(Map devicePathToAligned) + public void initRegionScanQueryDataSource(Map devicePathsToContext) throws QueryProcessException { long startTime = System.nanoTime(); - if (devicePathsToAligned == null) { + if (devicePathsToContext == null) { return; } dataRegion.readLock(); try { this.sharedQueryDataSource = dataRegion.queryForDeviceRegionScan( - devicePathToAligned, + devicePathsToContext, this, globalTimeFilter != null ? globalTimeFilter.copy() : null, timePartitions); @@ -460,8 +461,8 @@ public synchronized IQueryDataSource getSharedQueryDataSource() throws QueryProc sourcePaths = null; break; case DEVICE_REGION_SCAN: - initRegionScanQueryDataSource(devicePathsToAligned); - devicePathsToAligned = null; + initRegionScanQueryDataSource(devicePathsToContext); + devicePathsToContext = null; break; case TIME_SERIES_REGION_SCAN: initRegionScanQueryDataSource(sourcePaths); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java index 37a9d63865c70..d1f6029f04c58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java @@ -109,7 +109,7 @@ public void transformToTsBlockColumns( builder .getColumnBuilder(2) .writeBinary(new Binary(String.valueOf(device.isAligned()), TSFileConfig.STRING_CHARSET)); - if (templateId != -1) { + if (templateId != SchemaConstant.NON_TEMPLATE) { builder .getColumnBuilder(3) .writeBinary( @@ -125,7 +125,7 @@ public void transformToTsBlockColumns( builder .getColumnBuilder(1) .writeBinary(new Binary(String.valueOf(device.isAligned()), TSFileConfig.STRING_CHARSET)); - if (templateId != -1) { + if (templateId != SchemaConstant.NON_TEMPLATE) { builder .getColumnBuilder(2) .writeBinary( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java index 873352b8b7412..1932129c2f576 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java @@ -19,16 +19,22 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; @@ -41,7 +47,7 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSourceOperator { // The devices which need to be checked. - private final Map deviceToAlignedMap; + private final Map deviceContextMap; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ActiveDeviceRegionScanOperator.class) @@ -50,24 +56,24 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSource public ActiveDeviceRegionScanOperator( OperatorContext operatorContext, PlanNodeId sourceId, - Map deviceToAlignedMap, + Map deviceContextMap, Filter timeFilter, boolean outputCount) { this.outputCount = outputCount; this.sourceId = sourceId; this.operatorContext = operatorContext; - this.deviceToAlignedMap = deviceToAlignedMap; + this.deviceContextMap = deviceContextMap; this.regionScanUtil = new RegionScanForActiveDeviceUtil(timeFilter); } @Override protected boolean getNextTsFileHandle() throws IOException, IllegalPathException { - return ((RegionScanForActiveDeviceUtil) regionScanUtil).nextTsFileHandle(deviceToAlignedMap); + return ((RegionScanForActiveDeviceUtil) regionScanUtil).nextTsFileHandle(deviceContextMap); } @Override protected boolean isAllDataChecked() { - return deviceToAlignedMap.isEmpty(); + return deviceContextMap.isEmpty(); } @Override @@ -77,20 +83,42 @@ protected void updateActiveData() { if (this.outputCount) { count += activeDevices.size(); - activeDevices.forEach(deviceToAlignedMap.keySet()::remove); + activeDevices.forEach(deviceContextMap.keySet()::remove); } else { TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); for (IDeviceID deviceID : activeDevices) { + DeviceContext deviceContext = deviceContextMap.get(deviceID); + int templateId = deviceContext.getTemplateId(); + // TODO: use IDeviceID interface to get ttl + long ttl; + try { + ttl = DataNodeTTLCache.getInstance().getTTL(((PlainDeviceID) deviceID).toStringID()); + } catch (IllegalPathException e) { + ttl = Long.MAX_VALUE; + } + // TODO: make it more readable, like "30 days" or "10 hours" + String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl); + timeColumnBuilder.writeLong(-1); columnBuilders[0].writeBinary(new Binary(deviceID.getBytes())); columnBuilders[1].writeBinary( new Binary( - String.valueOf(deviceToAlignedMap.get(deviceID)), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].appendNull(); - columnBuilders[3].appendNull(); + String.valueOf(deviceContextMap.get(deviceID).isAligned()), + TSFileConfig.STRING_CHARSET)); + + if (templateId != SchemaConstant.NON_TEMPLATE) { + columnBuilders[2].writeBinary( + new Binary( + String.valueOf( + ClusterTemplateManager.getInstance().getTemplate(templateId).getName()), + TSFileConfig.STRING_CHARSET)); + } else { + columnBuilders[2].appendNull(); + } + columnBuilders[3].writeBinary(new Binary(ttlStr, TSFileConfig.STRING_CHARSET)); resultTsBlockBuilder.declarePosition(); - deviceToAlignedMap.remove(deviceID); + deviceContextMap.remove(deviceID); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java index e3385f92ecd05..6be648fb6bc4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -42,7 +42,7 @@ public class ActiveTimeSeriesRegionScanOperator extends AbstractRegionScanDataSourceOperator { // Timeseries which need to be checked. - private final Map> timeSeriesToSchemasInfo; + private final Map> timeSeriesToSchemasInfo; private static final Binary VIEW_TYPE = new Binary("BASE".getBytes()); private final Binary dataBaseName; private static final long INSTANCE_SIZE = @@ -53,7 +53,7 @@ public class ActiveTimeSeriesRegionScanOperator extends AbstractRegionScanDataSo public ActiveTimeSeriesRegionScanOperator( OperatorContext operatorContext, PlanNodeId sourceId, - Map> timeSeriesToSchemasInfo, + Map> timeSeriesToSchemasInfo, Filter timeFilter, boolean isOutputCount) { this.outputCount = isOutputCount; @@ -111,9 +111,9 @@ protected void updateActiveData() { IDeviceID deviceID = entry.getKey(); String deviceStr = ((PlainDeviceID) deviceID).toStringID(); List timeSeriesList = entry.getValue(); - Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); + Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { - TimeseriesSchemaInfo schemaInfo = timeSeriesInfo.get(timeSeries); + TimeseriesContext schemaInfo = timeSeriesInfo.get(timeSeries); timeColumnBuilder.writeLong(-1); columnBuilders[0].writeBinary( new Binary(contactDeviceAndMeasurement(deviceStr, timeSeries))); @@ -135,7 +135,7 @@ protected void updateActiveData() { } private void removeTimeseriesListFromDevice(IDeviceID deviceID, List timeSeriesList) { - Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); + Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { timeSeriesInfo.remove(timeSeries); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java index 8ca5e165363f9..11faa55f3941c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime; @@ -60,7 +61,7 @@ public boolean isCurrentTsFileFinished() { return deviceSetForCurrentTsFile.isEmpty(); } - public boolean nextTsFileHandle(Map targetDevices) + public boolean nextTsFileHandle(Map targetDevices) throws IOException, IllegalPathException { if (!queryDataSource.hasNext()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java index ff6711232bbac..61c7028ab211a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime; @@ -55,8 +55,8 @@ public RegionScanForActiveTimeSeriesUtil(Filter timeFilter) { this.activeTimeSeries = new HashMap<>(); } - public boolean nextTsFileHandle( - Map> targetTimeseries) throws IOException { + public boolean nextTsFileHandle(Map> targetTimeseries) + throws IOException { if (!queryDataSource.hasNext()) { // There is no more TsFileHandles to be scanned. return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 5897f78b0dddf..f2f70adebf45c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -28,9 +28,10 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.NodeRef; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.execution.memory.StatementMemorySource; @@ -301,23 +302,23 @@ aggregation results last_value(temperature) and last_value(status), whereas buck private List measurementSchemaList; // Used for regionScan - private Map devicePathToAlignedStatus; - private Map>> deviceToTimeseriesSchemas; + private Map devicePathToContextMap; + private Map>> deviceToTimeseriesSchemas; - public void setDevicePathToAlignedStatus(Map devicePathToAlignedStatus) { - this.devicePathToAlignedStatus = devicePathToAlignedStatus; + public void setDevicePathToContextMap(Map devicePathToContextMap) { + this.devicePathToContextMap = devicePathToContextMap; } - public Map getDevicePathToAlignedStatus() { - return devicePathToAlignedStatus; + public Map getDevicePathToContextMap() { + return devicePathToContextMap; } public void setDeviceToTimeseriesSchemas( - Map>> deviceToTimeseriesSchemas) { + Map>> deviceToTimeseriesSchemas) { this.deviceToTimeseriesSchemas = deviceToTimeseriesSchemas; } - public Map>> + public Map>> getDeviceToTimeseriesSchemas() { return deviceToTimeseriesSchemas; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 894881dc6cd8c..d02cb3f1e001e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -49,8 +49,9 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; @@ -2879,7 +2880,7 @@ private boolean analyzeTimeseriesRegionScan( Map> deviceToMeasurementMap = new HashMap<>(); Map deviceToAlignedMap = new HashMap<>(); Map> deviceToMeasurementSchemaMap = new HashMap<>(); - Map> deviceToTimeseriesSchemaMap = new HashMap<>(); + Map> deviceToTimeseriesSchemaMap = new HashMap<>(); for (PartialPath pattern : patternTree.getAllPathPatterns()) { List measurementPathList = schemaTree.searchMeasurementPaths(pattern).left; for (MeasurementPath measurementPath : measurementPathList) { @@ -2895,7 +2896,7 @@ private boolean analyzeTimeseriesRegionScan( deviceToTimeseriesSchemaMap .computeIfAbsent(devicePath, k -> new ArrayList<>()) .add( - new TimeseriesSchemaInfo( + new TimeseriesContext( new MeasurementSchemaInfo( measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), @@ -2904,19 +2905,18 @@ private boolean analyzeTimeseriesRegionScan( } } - Map>> deviceToTimeseriesSchemaInfo = + Map>> deviceToTimeseriesSchemaInfo = new HashMap<>(); for (PartialPath devicePath : deviceToAlignedMap.keySet()) { if (deviceToAlignedMap.get(devicePath)) { List measurementList = deviceToMeasurementMap.get(devicePath); List schemaList = deviceToMeasurementSchemaMap.get(devicePath); - List timeseriesSchemaInfoList = - deviceToTimeseriesSchemaMap.get(devicePath); + List timeseriesContextList = deviceToTimeseriesSchemaMap.get(devicePath); AlignedPath alignedPath = new AlignedPath(devicePath.getNodes(), measurementList, schemaList); deviceToTimeseriesSchemaInfo .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(alignedPath, timeseriesSchemaInfoList); + .put(alignedPath, timeseriesContextList); } else { for (String measurement : deviceToMeasurementMap.get(devicePath)) { MeasurementPath measurementPath = @@ -2926,7 +2926,7 @@ private boolean analyzeTimeseriesRegionScan( .put( measurementPath, Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( new MeasurementSchemaInfo( measurement, deviceToMeasurementSchemaMap.get(devicePath).get(0), @@ -3068,15 +3068,14 @@ private void analyzeDeviceRegionScan( } // fetch Data partition - List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); - Map devicePathsToAlignedStatus = new HashMap<>(); - for (DeviceSchemaInfo deviceSchema : deviceSchemaInfoList) { - devicePathsToAlignedStatus.put(deviceSchema.getDevicePath(), deviceSchema.isAligned()); - } - analysis.setDevicePathToAlignedStatus(devicePathsToAlignedStatus); + + Map devicePathsToInfoMap = + schemaTree.getMatchedDevices(pattern).stream() + .collect(Collectors.toMap(DeviceSchemaInfo::getDevicePath, DeviceContext::new)); + analysis.setDevicePathToContextMap(devicePathsToInfoMap); DataPartition dataPartition = fetchDataPartitionByDevices( - devicePathsToAlignedStatus.keySet().stream() + devicePathsToInfoMap.keySet().stream() .map(PartialPath::getFullPath) .collect(Collectors.toSet()), schemaTree, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index 4789834f35eb7..4ad583bec1410 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException; import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; @@ -97,7 +98,7 @@ public List plan( context.addPipelineDriverFactory(root, context.getDriverContext(), estimatedMemorySize); instanceContext.setSourcePaths(collectSourcePaths(context)); - instanceContext.setDevicePathsToAligned(collectDevicePathsToAligned(context)); + instanceContext.setDevicePathsToContext(collectDevicePathsToContext(context)); instanceContext.setQueryDataSourceType( getQueryDataSourceType((DataDriverContext) context.getDriverContext())); @@ -190,11 +191,12 @@ private QueryDataSourceType getQueryDataSourceType(DataDriverContext dataDriverC return dataDriverContext.getQueryDataSourceType().orElse(QueryDataSourceType.SERIES_SCAN); } - private Map collectDevicePathsToAligned(LocalExecutionPlanContext context) { + private Map collectDevicePathsToContext( + LocalExecutionPlanContext context) { DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); - Map deviceToAlignedMap = dataDriverContext.getDeviceIDToAligned(); - dataDriverContext.clearDeviceIDToAligned(); - return deviceToAlignedMap; + Map deviceContextMap = dataDriverContext.getDeviceIDToContext(); + dataDriverContext.clearDeviceIDToContext(); + return deviceContextMap; } private List collectSourcePaths(LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 347826b8fd68e..3ff06fb338a94 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -28,8 +28,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; @@ -1376,15 +1377,15 @@ public LogicalPlanBuilder planEndTimeColumnInject( } public LogicalPlanBuilder planDeviceRegionScan( - Map devicePathToAlignedStatus, boolean outputCount) { + Map devicePathToContextMap, boolean outputCount) { this.root = new DeviceRegionScanNode( - context.getQueryId().genPlanNodeId(), devicePathToAlignedStatus, outputCount, null); + context.getQueryId().genPlanNodeId(), devicePathToContextMap, outputCount, null); return this; } public LogicalPlanBuilder planTimeseriesRegionScan( - Map>> deviceToTimeseriesSchemaInfo, + Map>> deviceToTimeseriesSchemaInfo, boolean outputCount) { TimeseriesRegionScanNode timeseriesRegionScanNode = new TimeseriesRegionScanNode(context.getQueryId().genPlanNodeId(), outputCount, null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 1a6bddb422a31..96307945e462b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -579,7 +579,7 @@ public PlanNode visitShowDevices( if (showDevicesStatement.hasTimeCondition()) { planBuilder = planBuilder - .planDeviceRegionScan(analysis.getDevicePathToAlignedStatus(), false) + .planDeviceRegionScan(analysis.getDevicePathToContextMap(), false) .planLimit(showDevicesStatement.getLimit()) .planOffset(showDevicesStatement.getOffset()); return planBuilder.getRoot(); @@ -625,7 +625,7 @@ public PlanNode visitCountDevices( LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); if (countDevicesStatement.hasTimeCondition()) { - planBuilder = planBuilder.planDeviceRegionScan(analysis.getDevicePathToAlignedStatus(), true); + planBuilder = planBuilder.planDeviceRegionScan(analysis.getDevicePathToContextMap(), true); return planBuilder.getRoot(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index abafd45ae9496..1127cbd1683ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -26,9 +26,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.NodeRef; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.execution.aggregation.Accumulator; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; import org.apache.iotdb.db.queryengine.execution.aggregation.Aggregator; @@ -3553,17 +3554,18 @@ public Operator visitDeviceRegionScan( node.getPlanNodeId(), ActiveDeviceRegionScanOperator.class.getSimpleName()); Filter filter = context.getGlobalTimeFilter(); - Map deviceIDToAligned = new HashMap<>(); - for (Map.Entry entry : node.getDevicePathsToAligned().entrySet()) { - deviceIDToAligned.put(new PlainDeviceID(entry.getKey().getFullPath()), entry.getValue()); + Map deviceIDToContext = new HashMap<>(); + for (Map.Entry entry : + node.getDevicePathToContextMap().entrySet()) { + deviceIDToContext.put(new PlainDeviceID(entry.getKey().getFullPath()), entry.getValue()); } ActiveDeviceRegionScanOperator regionScanOperator = new ActiveDeviceRegionScanOperator( - operatorContext, node.getPlanNodeId(), deviceIDToAligned, filter, node.isOutputCount()); + operatorContext, node.getPlanNodeId(), deviceIDToContext, filter, node.isOutputCount()); DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); dataDriverContext.addSourceOperator(regionScanOperator); - dataDriverContext.setDeviceIDToAligned(deviceIDToAligned); + dataDriverContext.setDeviceIDToContext(deviceIDToContext); dataDriverContext.setQueryDataSourceType(QueryDataSourceType.DEVICE_REGION_SCAN); return regionScanOperator; @@ -3582,10 +3584,10 @@ public Operator visitTimeSeriesRegionScan( Filter filter = context.getGlobalTimeFilter(); DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); - Map> timeseriesToSchemaInfo = new HashMap<>(); - for (Map.Entry>> entryMap : + Map> timeseriesToSchemaInfo = new HashMap<>(); + for (Map.Entry>> entryMap : node.getDeviceToTimeseriesSchemaInfo().entrySet()) { - Map timeseriesSchemaInfoMap = + Map timeseriesSchemaInfoMap = getTimeseriesSchemaInfoMap(entryMap, dataDriverContext); timeseriesToSchemaInfo.put( new PlainDeviceID(entryMap.getKey().getFullPath()), timeseriesSchemaInfoMap); @@ -3604,12 +3606,11 @@ public Operator visitTimeSeriesRegionScan( return regionScanOperator; } - private static Map getTimeseriesSchemaInfoMap( - Map.Entry>> entryMap, + private static Map getTimeseriesSchemaInfoMap( + Map.Entry>> entryMap, DataDriverContext context) { - Map timeseriesSchemaInfoMap = new HashMap<>(); - for (Map.Entry> entry : - entryMap.getValue().entrySet()) { + Map timeseriesSchemaInfoMap = new HashMap<>(); + for (Map.Entry> entry : entryMap.getValue().entrySet()) { PartialPath path = entry.getKey(); context.addPath(path); if (path instanceof MeasurementPath) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java index 14d810514c893..63d59ccaec7ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java @@ -39,7 +39,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; @@ -895,8 +894,7 @@ public List visitSchemaFetchMerge( for (TRegionReplicaSet schemaRegion : storageGroupSchemaRegionMap.get( ((SchemaFetchScanNode) child).getStorageGroup().getFullPath())) { - SchemaFetchScanNode schemaFetchScanNode = - (SchemaFetchScanNode) child.clone(); + SchemaFetchScanNode schemaFetchScanNode = (SchemaFetchScanNode) child.clone(); schemaFetchScanNode.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId()); schemaFetchScanNode.setRegionReplicaSet(schemaRegion); root.addChild(schemaFetchScanNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java index 2e24512c8c2d8..5b1935e1c4c05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; -import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; @@ -28,15 +27,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; - -import com.google.common.collect.ImmutableList; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; public class DeviceSchemaFetchScanNode extends SchemaFetchScanNode { @@ -52,7 +46,6 @@ public DeviceSchemaFetchScanNode( this.authorityScope.constructTree(); } - public PathPatternTree getAuthorityScope() { return authorityScope; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java index d0598b735bc44..56d2d5cf65740 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; -import com.google.common.collect.ImmutableList; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; @@ -27,66 +26,63 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; +import com.google.common.collect.ImmutableList; + import java.util.Collections; import java.util.List; -public abstract class SchemaFetchScanNode extends SourceNode { - protected final PartialPath storageGroup; - protected final PathPatternTree patternTree; - protected TRegionReplicaSet schemaRegionReplicaSet; - - protected SchemaFetchScanNode( - PlanNodeId id, - PartialPath storageGroup, - PathPatternTree patternTree){ - super(id); - this.storageGroup = storageGroup; - this.patternTree = patternTree; - this.patternTree.constructTree(); - } - - - public PartialPath getStorageGroup() { - return storageGroup; - } - - public PathPatternTree getPatternTree() { - return patternTree; - } - - @Override - public List getChildren() { - return Collections.emptyList(); - } - - @Override - public void addChild(PlanNode child) {} - - @Override - public int allowedChildCount() { - return 0; - } - - @Override - public List getOutputColumnNames() { - return ImmutableList.of(); - } - - - @Override - public void open() throws Exception {} - - @Override - public TRegionReplicaSet getRegionReplicaSet() { - return schemaRegionReplicaSet; - } - - @Override - public void setRegionReplicaSet(TRegionReplicaSet schemaRegionReplicaSet) { - this.schemaRegionReplicaSet = schemaRegionReplicaSet; - } - - @Override - public void close() throws Exception {} - +public abstract class SchemaFetchScanNode extends SourceNode { + protected final PartialPath storageGroup; + protected final PathPatternTree patternTree; + protected TRegionReplicaSet schemaRegionReplicaSet; + + protected SchemaFetchScanNode( + PlanNodeId id, PartialPath storageGroup, PathPatternTree patternTree) { + super(id); + this.storageGroup = storageGroup; + this.patternTree = patternTree; + this.patternTree.constructTree(); + } + + public PartialPath getStorageGroup() { + return storageGroup; + } + + public PathPatternTree getPatternTree() { + return patternTree; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return ImmutableList.of(); + } + + @Override + public void open() throws Exception {} + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return schemaRegionReplicaSet; + } + + @Override + public void setRegionReplicaSet(TRegionReplicaSet schemaRegionReplicaSet) { + this.schemaRegionReplicaSet = schemaRegionReplicaSet; + } + + @Override + public void close() throws Exception {} } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java index 5884252cd60f9..6f6d5ee491a13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; -import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; @@ -28,18 +27,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; import org.apache.iotdb.db.schemaengine.template.Template; -import com.google.common.collect.ImmutableList; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; /** This class defines the scan task of schema fetcher. */ @@ -62,7 +57,6 @@ public SeriesSchemaFetchScanNode( this.withTemplate = withTemplate; } - public Map getTemplateMap() { return templateMap; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java index 8671c52ddc0cb..a4db348342bf1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -45,21 +46,21 @@ import java.util.stream.Collectors; public class DeviceRegionScanNode extends RegionScanNode { - private Map devicePathsToAligned; + private Map devicePathToContextMap; public DeviceRegionScanNode( PlanNodeId planNodeId, - Map devicePathsToAligned, + Map devicePathToContextMap, boolean outputCount, TRegionReplicaSet regionReplicaSet) { super(planNodeId); - this.devicePathsToAligned = devicePathsToAligned; + this.devicePathToContextMap = devicePathToContextMap; this.regionReplicaSet = regionReplicaSet; this.outputCount = outputCount; } - public Map getDevicePathsToAligned() { - return devicePathsToAligned; + public Map getDevicePathToContextMap() { + return devicePathToContextMap; } @Override @@ -75,7 +76,7 @@ public void addChild(PlanNode child) { @Override public PlanNode clone() { return new DeviceRegionScanNode( - getPlanNodeId(), getDevicePathsToAligned(), isOutputCount(), getRegionReplicaSet()); + getPlanNodeId(), getDevicePathToContextMap(), isOutputCount(), getRegionReplicaSet()); } @Override @@ -101,24 +102,23 @@ public int allowedChildCount() { public static PlanNode deserialize(ByteBuffer buffer) { int size = ReadWriteIOUtils.readInt(buffer); - Map devicePathsToAligned = new HashMap<>(); + Map devicePathToContextMap = new HashMap<>(); for (int i = 0; i < size; i++) { PartialPath path = (PartialPath) PathDeserializeUtil.deserialize(buffer); - boolean aligned = ReadWriteIOUtils.readBool(buffer); - devicePathsToAligned.put(path, aligned); + devicePathToContextMap.put(path, DeviceContext.deserialize(buffer)); } boolean outputCount = ReadWriteIOUtils.readBool(buffer); PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); - return new DeviceRegionScanNode(planNodeId, devicePathsToAligned, outputCount, null); + return new DeviceRegionScanNode(planNodeId, devicePathToContextMap, outputCount, null); } @Override protected void serializeAttributes(ByteBuffer byteBuffer) { PlanNodeType.DEVICE_REGION_SCAN.serialize(byteBuffer); - ReadWriteIOUtils.write(devicePathsToAligned.size(), byteBuffer); - for (Map.Entry entry : devicePathsToAligned.entrySet()) { + ReadWriteIOUtils.write(devicePathToContextMap.size(), byteBuffer); + for (Map.Entry entry : devicePathToContextMap.entrySet()) { entry.getKey().serialize(byteBuffer); - ReadWriteIOUtils.write(entry.getValue(), byteBuffer); + entry.getValue().serializeAttributes(byteBuffer); } ReadWriteIOUtils.write(outputCount, byteBuffer); } @@ -126,10 +126,10 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.DEVICE_REGION_SCAN.serialize(stream); - ReadWriteIOUtils.write(devicePathsToAligned.size(), stream); - for (Map.Entry entry : devicePathsToAligned.entrySet()) { + ReadWriteIOUtils.write(devicePathToContextMap.size(), stream); + for (Map.Entry entry : devicePathToContextMap.entrySet()) { entry.getKey().serialize(stream); - ReadWriteIOUtils.write(entry.getValue(), stream); + entry.getValue().serializeAttributes(stream); } ReadWriteIOUtils.write(outputCount, stream); } @@ -145,23 +145,23 @@ public String toString() { @Override public Set getDevicePaths() { - return new HashSet<>(devicePathsToAligned.keySet()); + return new HashSet<>(devicePathToContextMap.keySet()); } @Override public void addDevicePath(PartialPath devicePath, RegionScanNode node) { - this.devicePathsToAligned.put( - devicePath, ((DeviceRegionScanNode) node).devicePathsToAligned.get(devicePath)); + this.devicePathToContextMap.put( + devicePath, ((DeviceRegionScanNode) node).devicePathToContextMap.get(devicePath)); } @Override public void clearPath() { - this.devicePathsToAligned = new HashMap<>(); + this.devicePathToContextMap = new HashMap<>(); } @Override public long getSize() { - return devicePathsToAligned.size(); + return devicePathToContextMap.size(); } @Override @@ -170,12 +170,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; DeviceRegionScanNode that = (DeviceRegionScanNode) o; - return devicePathsToAligned.equals(that.devicePathsToAligned) + return devicePathToContextMap.equals(that.devicePathToContextMap) && outputCount == that.isOutputCount(); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), devicePathsToAligned, outputCount); + return Objects.hash(super.hashCode(), devicePathToContextMap, outputCount); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java index 2bbe27d2230d5..5a5abed1aa020 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java @@ -26,7 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathType; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -55,8 +55,7 @@ public class TimeseriesRegionScanNode extends RegionScanNode { // IDeviceID -> (MeasurementPath -> TimeseriesSchemaInfo) - private Map>> - deviceToTimeseriesSchemaInfo; + private Map>> deviceToTimeseriesSchemaInfo; public TimeseriesRegionScanNode( PlanNodeId planNodeId, boolean outputCount, TRegionReplicaSet regionReplicaSet) { @@ -67,7 +66,7 @@ public TimeseriesRegionScanNode( public TimeseriesRegionScanNode( PlanNodeId planNodeId, - Map>> deviceToTimeseriesSchemaInfo, + Map>> deviceToTimeseriesSchemaInfo, boolean outputCount, TRegionReplicaSet regionReplicaSet) { super(planNodeId); @@ -77,11 +76,11 @@ public TimeseriesRegionScanNode( } public void setDeviceToTimeseriesSchemaInfo( - Map>> deviceToTimeseriesSchemaInfo) { + Map>> deviceToTimeseriesSchemaInfo) { this.deviceToTimeseriesSchemaInfo = deviceToTimeseriesSchemaInfo; } - public Map>> + public Map>> getDeviceToTimeseriesSchemaInfo() { return deviceToTimeseriesSchemaInfo; } @@ -125,7 +124,7 @@ public int allowedChildCount() { public static PlanNode deserialize(ByteBuffer buffer) { int size = ReadWriteIOUtils.readInt(buffer); - Map>> deviceToTimeseriesSchemaInfo = + Map>> deviceToTimeseriesSchemaInfo = new HashMap<>(); for (int i = 0; i < size; i++) { @@ -137,13 +136,13 @@ public static PlanNode deserialize(ByteBuffer buffer) { PartialPath devicePath = new PartialPath(nodes); int pathSize = ReadWriteIOUtils.readInt(buffer); - Map> measurementToSchemaInfo = new HashMap<>(); + Map> measurementToSchemaInfo = new HashMap<>(); for (int j = 0; j < pathSize; j++) { PartialPath path = deserializePartialPath(nodes, buffer); int schemaSize = ReadWriteIOUtils.readInt(buffer); - List schemaInfos = new ArrayList<>(); + List schemaInfos = new ArrayList<>(); for (int k = 0; k < schemaSize; k++) { - schemaInfos.add(TimeseriesSchemaInfo.deserialize(buffer)); + schemaInfos.add(TimeseriesContext.deserialize(buffer)); } measurementToSchemaInfo.put(path, schemaInfos); } @@ -197,7 +196,7 @@ public String toString() { "%s -> %s", entry1.getKey().getFullPath(), entry1.getValue().stream() - .map(TimeseriesSchemaInfo::toString) + .map(TimeseriesContext::toString) .collect(Collectors.joining(", ")))) .collect(Collectors.joining(", ")))) .collect(Collectors.joining(", "))); @@ -247,7 +246,7 @@ public int hashCode() { protected void serializeAttributes(ByteBuffer byteBuffer) { PlanNodeType.TIMESERIES_REGION_SCAN.serialize(byteBuffer); ReadWriteIOUtils.write(deviceToTimeseriesSchemaInfo.size(), byteBuffer); - for (Map.Entry>> entry : + for (Map.Entry>> entry : deviceToTimeseriesSchemaInfo.entrySet()) { int size = entry.getKey().getNodeLength(); @@ -258,12 +257,12 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { } ReadWriteIOUtils.write(entry.getValue().size(), byteBuffer); - for (Map.Entry> timseriesEntry : + for (Map.Entry> timseriesEntry : entry.getValue().entrySet()) { serializeMeasurements(timseriesEntry.getKey(), byteBuffer); ReadWriteIOUtils.write(timseriesEntry.getValue().size(), byteBuffer); - for (TimeseriesSchemaInfo timeseriesSchemaInfo : timseriesEntry.getValue()) { - timeseriesSchemaInfo.serializeAttributes(byteBuffer); + for (TimeseriesContext timeseriesContext : timseriesEntry.getValue()) { + timeseriesContext.serializeAttributes(byteBuffer); } } } @@ -274,7 +273,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.TIMESERIES_REGION_SCAN.serialize(stream); ReadWriteIOUtils.write(deviceToTimeseriesSchemaInfo.size(), stream); - for (Map.Entry>> entry : + for (Map.Entry>> entry : deviceToTimeseriesSchemaInfo.entrySet()) { int size = entry.getKey().getNodeLength(); @@ -285,12 +284,12 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } ReadWriteIOUtils.write(entry.getValue().size(), stream); - for (Map.Entry> timseriesEntry : + for (Map.Entry> timseriesEntry : entry.getValue().entrySet()) { serializeMeasurements(timseriesEntry.getKey(), stream); ReadWriteIOUtils.write(timseriesEntry.getValue().size(), stream); - for (TimeseriesSchemaInfo timeseriesSchemaInfo : timseriesEntry.getValue()) { - timeseriesSchemaInfo.serializeAttributes(stream); + for (TimeseriesContext timeseriesContext : timseriesEntry.getValue()) { + timeseriesContext.serializeAttributes(stream); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index be7adafe82e85..97cc5f8828aeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -773,7 +773,8 @@ public ClusterSchemaTree fetchDeviceSchema( @Override protected Void collectEntity(IDeviceMNode node) { if (node.isAlignedNullable() != null) { - schemaTree.appendDevice(node.getPartialPath(), node.isAligned()); + schemaTree.appendTemplateDevice( + node.getPartialPath(), node.isAligned(), node.getSchemaTemplateId(), null); } return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index ab36b2abd2f23..ceeebf1247aeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -898,7 +898,8 @@ public ClusterSchemaTree fetchDeviceSchema( @Override protected Void collectEntity(IDeviceMNode node) { if (node.isAlignedNullable() != null) { - schemaTree.appendDevice(node.getPartialPath(), node.isAligned()); + schemaTree.appendTemplateDevice( + node.getPartialPath(), node.isAligned(), node.getSchemaTemplateId(), null); } return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 5a58b3e1dbfce..8c473652c9393 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.quota.ExceedQuotaException; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.execution.load.LoadTsFileRateLimiter; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; @@ -1871,7 +1872,7 @@ private List getFileHandleListForQuery( @Override public IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathToAligned, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) @@ -1906,7 +1907,7 @@ public IQueryDataSource queryForDeviceRegionScan( private List getFileHandleListForQuery( Collection tsFileResources, - Map devicePathToAligned, + Map devicePathsToContext, QueryContext context, Filter globalTimeFilter, boolean isSeq) @@ -1924,7 +1925,7 @@ private List getFileHandleListForQuery( } else { tsFileResource .getProcessor() - .queryForDeviceRegionScan(devicePathToAligned, context, fileScanHandles); + .queryForDeviceRegionScan(devicePathsToContext, context, fileScanHandles); } } finally { closeQueryLock.readLock().unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java index 1ad4de9abf857..c18262145e533 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.dataregion.read.IQueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; @@ -49,7 +50,7 @@ QueryDataSource query( /** Get satisfied QueryDataSource from DataRegion for regionScan */ IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java index b18dbb01eb7ea..42c5a766886c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.dataregion.read.IQueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; @@ -73,7 +74,7 @@ public QueryDataSource query( @Override public IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 74aed59ab887e..5a964c33600d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; @@ -1883,7 +1884,7 @@ public void queryForSeriesRegionScan( * get the related ChunkMetadata of data on disk. */ public void queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, List fileScanHandlesForQuery) { long startTime = System.nanoTime(); @@ -1893,9 +1894,9 @@ public void queryForDeviceRegionScan( new HashMap<>(); flushQueryLock.readLock().lock(); try { - for (Map.Entry entry : devicePathToAligned.entrySet()) { + for (Map.Entry entry : devicePathsToContext.entrySet()) { IDeviceID deviceID = entry.getKey(); - boolean isAligned = entry.getValue(); + boolean isAligned = entry.getValue().isAligned(); long timeLowerBound = getQueryTimeLowerBound( PathUtils.splitPathToDetachedNodes(((PlainDeviceID) deviceID).toStringID())); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java index 452e6dce7983b..ba24a39d64dc1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java @@ -23,8 +23,10 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.QueryId; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -49,10 +51,10 @@ public class RegionScanLogicalPlannerTest { - private static Map>> + private static Map>> deviceToTimeseriesSchemaInfoMap; - private static Map>> + private static Map>> getDeviceToTimeseriesSchemaInfoMap() throws IllegalPathException { if (deviceToTimeseriesSchemaInfoMap != null) { @@ -60,40 +62,40 @@ public class RegionScanLogicalPlannerTest { } deviceToTimeseriesSchemaInfoMap = new HashMap<>(); - Map> timeseriesSchemaInfoMap = new HashMap<>(); + Map> timeseriesSchemaInfoMap = new HashMap<>(); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s1", TSDataType.INT32), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s2", TSDataType.DOUBLE), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s3", TSDataType.BOOLEAN), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "BOOLEAN", null, "PLAIN", "LZ4", "{\"key1\":\"value2\"}", null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d1")), timeseriesSchemaInfoMap); - Map> timeseriesSchemaInfoMap2 = new HashMap<>(); + Map> timeseriesSchemaInfoMap2 = new HashMap<>(); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s1", TSDataType.INT32), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s2", TSDataType.DOUBLE), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s4", TSDataType.TEXT), Collections.singletonList( - new TimeseriesSchemaInfo( + new TimeseriesContext( "TEXT", null, "PLAIN", "LZ4", "{\"key2\":\"value1\"}", null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d2")), timeseriesSchemaInfoMap2); @@ -101,17 +103,15 @@ public class RegionScanLogicalPlannerTest { List schemas = new ArrayList<>(); schemas.add("s1"); schemas.add("s2"); - List timeseriesSchemaInfoList = new ArrayList<>(); - Map> timeseriesSchemaInfoMap3 = new HashMap<>(); - timeseriesSchemaInfoList.add( - new TimeseriesSchemaInfo( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); - timeseriesSchemaInfoList.add( - new TimeseriesSchemaInfo( + List timeseriesContextList = new ArrayList<>(); + Map> timeseriesSchemaInfoMap3 = new HashMap<>(); + timeseriesContextList.add( + new TimeseriesContext("INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); + timeseriesContextList.add( + new TimeseriesContext( "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); timeseriesSchemaInfoMap3.put( - new AlignedPath("root.sg.d2.a", schemas, Collections.emptyList()), - timeseriesSchemaInfoList); + new AlignedPath("root.sg.d2.a", schemas, Collections.emptyList()), timeseriesContextList); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d2.a")), timeseriesSchemaInfoMap3); @@ -126,13 +126,19 @@ public void testShowDevicesWithTimeCondition() throws IllegalPathException { // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, false, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, false, null); PlanNode actualPlan = parseSQLToPlanNode(sql); Assert.assertEquals(actualPlan, regionScanNode); @@ -146,13 +152,19 @@ public void testShowDevicesWithTimeConditionWithLimitOffset() throws IllegalPath // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, false, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, false, null); LimitNode limitNode = new LimitNode(queryId.genPlanNodeId(), 20); limitNode.addChild(regionScanNode); @@ -171,13 +183,19 @@ public void testCountDevicesWithTimeConditionWithLimitOffset() throws IllegalPat // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, true, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, true, null); PlanNode actualPlan = parseSQLToPlanNode(sql); Assert.assertEquals(actualPlan, regionScanNode); From 843cbcc96299583dfa19a102d522fd907a3c9ec4 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 16:00:49 +0800 Subject: [PATCH 04/12] save --- .../db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java index 718d10ef8156a..62ccf5e9ef551 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java @@ -53,6 +53,13 @@ public ClusterSchemaTree fetchSchema( return schemaTree; } + @Override + public ISchemaTree fetchSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + schemaTree.setDatabases(Collections.singleton("root.sg")); + return schemaTree; + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { From 52d3b7ec2c7df7d09b6583115e439a2624483c14 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 16:04:51 +0800 Subject: [PATCH 05/12] save --- .../db/queryengine/plan/planner/distribution/Util.java | 6 ++++++ .../db/queryengine/plan/planner/distribution/Util2.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java index 59cd36a2f8ae5..65b614ff48495 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java @@ -305,6 +305,12 @@ public ISchemaTree fetchSchema( return ANALYSIS.getSchemaTree(); } + @Override + public ISchemaTree fetchSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java index eb802edd9e541..a84e002df2e41 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java @@ -199,6 +199,12 @@ public ISchemaTree fetchSchema( return ANALYSIS.getSchemaTree(); } + @Override + public ISchemaTree fetchSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { From d77cf622d8dc58688bab4b56ae003a08a7a51e8a Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 16:07:52 +0800 Subject: [PATCH 06/12] fix compile error --- .../source/ActiveDeviceRegionScanOperator.java | 9 ++------- .../plan/analyze/cache/schema/DataNodeTTLCache.java | 12 ++++++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java index 1932129c2f576..6437fe673faf7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java @@ -34,7 +34,6 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; @@ -91,12 +90,8 @@ protected void updateActiveData() { DeviceContext deviceContext = deviceContextMap.get(deviceID); int templateId = deviceContext.getTemplateId(); // TODO: use IDeviceID interface to get ttl - long ttl; - try { - ttl = DataNodeTTLCache.getInstance().getTTL(((PlainDeviceID) deviceID).toStringID()); - } catch (IllegalPathException e) { - ttl = Long.MAX_VALUE; - } + long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID); + // TODO: make it more readable, like "30 days" or "10 hours" String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 07b835896d9da..0c87b51b74918 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@ -23,6 +23,9 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; + import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -71,6 +74,15 @@ public void unsetTTL(String[] path) { } } + // TODO: support real implementation + public long getTTL(IDeviceID deviceID) { + try { + return getTTL(((PlainDeviceID) deviceID).toStringID()); + } catch (IllegalPathException e) { + return Long.MAX_VALUE; + } + } + public long getTTL(String path) throws IllegalPathException { lock.readLock().lock(); try { From 54f0e0729797351ca00d52bcd91c5d1bd769d225 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Wed, 12 Jun 2024 17:53:40 +0800 Subject: [PATCH 07/12] try fix UT --- .../queryengine/common/TimeseriesContext.java | 16 ++++++++------- .../visitor/SchemaTreeMeasurementVisitor.java | 4 +++- .../plan/analyze/AnalyzeVisitor.java | 20 ++++++++----------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 6450c036128ce..2bf55ec5baf11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -143,13 +143,15 @@ public boolean equals(Object obj) { return false; } TimeseriesContext that = (TimeseriesContext) obj; - return Objects.equals(dataType, that.dataType) - && Objects.equals(alias, that.alias) - && encoding.equals(that.encoding) - && Objects.equals(compression, that.compression) - && Objects.equals(tags, that.tags) - && Objects.equals(deadband, that.deadband) - && Objects.equals(deadbandParameters, that.deadbandParameters); + boolean res = + Objects.equals(dataType, that.dataType) + && Objects.equals(alias, that.alias) + && encoding.equals(that.encoding) + && Objects.equals(compression, that.compression) + && Objects.equals(tags, that.tags) + && Objects.equals(deadband, that.deadband) + && Objects.equals(deadbandParameters, that.deadbandParameters); + return res; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java index ca33a519b01dd..19f061f5f5089 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java @@ -26,6 +26,8 @@ import org.apache.iotdb.commons.path.fa.IFATransition; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; +import org.apache.commons.lang3.StringUtils; + import java.util.Map; public class SchemaTreeMeasurementVisitor extends SchemaTreeVisitor { @@ -128,7 +130,7 @@ protected MeasurementPath generateResult(SchemaNode nextMatchedNode) { result.setTagMap(nextMatchedNode.getAsMeasurementNode().getTagMap()); result.setUnderAlignedEntity(getParentOfNextMatchedNode().getAsEntityNode().isAligned()); String alias = nextMatchedNode.getAsMeasurementNode().getAlias(); - if (tailNode.equals(alias)) { + if (alias != null && !StringUtils.isEmpty(alias)) { result.setMeasurementAlias(alias); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index d02cb3f1e001e..4b1a8cff7a093 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2900,7 +2900,9 @@ private boolean analyzeTimeseriesRegionScan( new MeasurementSchemaInfo( measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), - measurementPath.getMeasurementAlias(), + measurementPath.isMeasurementAliasExists() + ? measurementPath.getMeasurementAlias() + : null, measurementPath.getTagMap()))); } } @@ -2918,20 +2920,14 @@ private boolean analyzeTimeseriesRegionScan( .computeIfAbsent(devicePath, k -> new HashMap<>()) .put(alignedPath, timeseriesContextList); } else { - for (String measurement : deviceToMeasurementMap.get(devicePath)) { + List measurementList = deviceToMeasurementMap.get(devicePath); + List timeseriesContextList = deviceToTimeseriesSchemaMap.get(devicePath); + for (int i = 0; i < measurementList.size(); i++) { MeasurementPath measurementPath = - new MeasurementPath(devicePath.concatNode(measurement).getNodes()); + new MeasurementPath(devicePath.concatNode(measurementList.get(i)).getNodes()); deviceToTimeseriesSchemaInfo .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put( - measurementPath, - Collections.singletonList( - new TimeseriesContext( - new MeasurementSchemaInfo( - measurement, - deviceToMeasurementSchemaMap.get(devicePath).get(0), - null, - null)))); + .put(measurementPath, Collections.singletonList(timeseriesContextList.get(i))); } } } From 408815323283bcb6cfe0ad84512377a916cf3c0e Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 13 Jun 2024 22:48:14 +0800 Subject: [PATCH 08/12] fix ci --- .../queryengine/common/TimeseriesContext.java | 13 ++ .../common/schematree/ClusterSchemaTree.java | 12 ++ .../common/schematree/ISchemaTree.java | 11 ++ .../visitor/SchemaTreeMeasurementVisitor.java | 4 +- .../SchemaTreeTimeseriesContextVisitor.java | 136 ++++++++++++++++++ .../visitor/SchemaTreeVisitorFactory.java | 5 + .../plan/analyze/AnalyzeVisitor.java | 25 ++-- 7 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 2bf55ec5baf11..638716f832d2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -24,10 +24,12 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import java.util.Objects; import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; @@ -57,6 +59,17 @@ public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { this.deadbandParameters = deadbandInfo.right; } + public TimeseriesContext(IMeasurementSchema schema, String alias, Map tagMap) { + this.dataType = schema.getType().toString(); + this.alias = alias; + this.encoding = schema.getEncodingType().toString(); + this.compression = schema.getCompressor().toString(); + this.tags = mapToString(tagMap); + Pair deadbandInfo = MetaUtils.parseDeadbandInfo(schema.getProps()); + this.deadband = deadbandInfo.left; + this.deadbandParameters = deadbandInfo.right; + } + public String getDataType() { return dataType; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index f1d90875ac537..9b5257baa395e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -33,12 +33,14 @@ import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeDeviceUsingTemplateVisitor; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeDeviceVisitor; +import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeTimeseriesContextVisitor; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorWithLimitOffsetWrapper; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.commons.lang3.tuple.Triple; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -118,6 +120,16 @@ public Pair, Integer> searchMeasurementPaths(PartialPath p return searchMeasurementPaths(pathPattern, 0, 0, false); } + @Override + public List> searchMeasurementSchemaInfo( + PartialPath pathPattern) { + try (SchemaTreeTimeseriesContextVisitor visitor = + SchemaTreeVisitorFactory.createTimeseriesContextVisitor( + root, pathPattern, authorityScope)) { + return visitor.getAllResult(); + } + } + public List getAllDevices() { return getMatchedDevices(ALL_MATCH_PATTERN); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java index 0288033b6955c..fdfad14ab87b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.commons.lang3.tuple.Triple; import org.apache.tsfile.utils.Pair; import java.util.List; @@ -49,6 +50,16 @@ Pair, Integer> searchMeasurementPaths( */ Pair, Integer> searchMeasurementPaths(PartialPath pathPattern); + /** + * Search all timeseries schema info that match the given path pattern. Only use for active schema + * query. Only non-view timeseries will be returned. + * + * @param pathPattern pattern + * @return list of triple [path, isAligned, schema] + */ + List> searchMeasurementSchemaInfo( + PartialPath pathPattern); + /** * Get all device matching the path pattern. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java index 19f061f5f5089..ca33a519b01dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeMeasurementVisitor.java @@ -26,8 +26,6 @@ import org.apache.iotdb.commons.path.fa.IFATransition; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; -import org.apache.commons.lang3.StringUtils; - import java.util.Map; public class SchemaTreeMeasurementVisitor extends SchemaTreeVisitor { @@ -130,7 +128,7 @@ protected MeasurementPath generateResult(SchemaNode nextMatchedNode) { result.setTagMap(nextMatchedNode.getAsMeasurementNode().getTagMap()); result.setUnderAlignedEntity(getParentOfNextMatchedNode().getAsEntityNode().isAligned()); String alias = nextMatchedNode.getAsMeasurementNode().getAlias(); - if (alias != null && !StringUtils.isEmpty(alias)) { + if (tailNode.equals(alias)) { result.setMeasurementAlias(alias); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java new file mode 100644 index 0000000000000..777d4fa26cddc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.common.schematree.visitor; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.path.fa.IFAState; +import org.apache.iotdb.commons.path.fa.IFATransition; +import org.apache.iotdb.db.queryengine.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaMeasurementNode; +import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; + +import org.apache.commons.lang3.tuple.Triple; + +import java.util.Map; + +public class SchemaTreeTimeseriesContextVisitor + extends SchemaTreeVisitor> { + public SchemaTreeTimeseriesContextVisitor( + SchemaNode root, PartialPath pathPattern, boolean isPrefixMatch) { + super(root, pathPattern, isPrefixMatch); + } + + public SchemaTreeTimeseriesContextVisitor( + SchemaNode root, PartialPath pathPattern, boolean isPrefixMatch, PathPatternTree scope) { + super(root, pathPattern, isPrefixMatch, scope); + } + + @Override + protected boolean mayTargetNodeType(SchemaNode node) { + return node.isMeasurement(); + } + + @Override + protected IFAState tryGetNextState( + SchemaNode node, IFAState sourceState, Map preciseMatchTransitionMap) { + IFATransition transition; + IFAState state; + if (node.isMeasurement()) { + String alias = node.getAsMeasurementNode().getAlias(); + if (alias != null) { + transition = preciseMatchTransitionMap.get(alias); + if (transition != null) { + state = patternFA.getNextState(sourceState, transition); + if (state.isFinal()) { + return state; + } + } + } + transition = preciseMatchTransitionMap.get(node.getName()); + if (transition != null) { + state = patternFA.getNextState(sourceState, transition); + if (state.isFinal()) { + return state; + } + } + return null; + } + + transition = preciseMatchTransitionMap.get(node.getName()); + if (transition == null) { + return null; + } + return patternFA.getNextState(sourceState, transition); + } + + @Override + protected IFAState tryGetNextState( + SchemaNode node, IFAState sourceState, IFATransition transition) { + IFAState state; + if (node.isMeasurement()) { + String alias = node.getAsMeasurementNode().getAlias(); + if (alias != null && transition.isMatch(alias)) { + state = patternFA.getNextState(sourceState, transition); + if (state.isFinal()) { + return state; + } + } + if (transition.isMatch(node.getName())) { + state = patternFA.getNextState(sourceState, transition); + if (state.isFinal()) { + return state; + } + } + return null; + } + + if (transition.isMatch(node.getName())) { + return patternFA.getNextState(sourceState, transition); + } + return null; + } + + @Override + protected boolean acceptInternalMatchedNode(SchemaNode node) { + return false; + } + + @Override + protected boolean acceptFullMatchedNode(SchemaNode node) { + return node.isMeasurement() && !node.getAsMeasurementNode().isLogicalView(); + } + + @Override + protected Triple generateResult( + SchemaNode nextMatchedNode) { + SchemaMeasurementNode measurementNode = nextMatchedNode.getAsMeasurementNode(); + MeasurementSchemaInfo measurementSchemaInfo = + new MeasurementSchemaInfo( + measurementNode.getName(), + measurementNode.getSchema(), + measurementNode.getAlias(), + measurementNode.getTagMap()); + return Triple.of( + getPartialPathFromRootToNode(nextMatchedNode), + getParentOfNextMatchedNode().getAsEntityNode().isAligned(), + measurementSchemaInfo); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java index 8b4c722af5fd2..0e9a465911ea3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java @@ -56,6 +56,11 @@ public static SchemaTreeMeasurementVisitor createSchemaTreeMeasurementVisitor( return new SchemaTreeMeasurementVisitor(root, pathPattern, false, scope); } + public static SchemaTreeTimeseriesContextVisitor createTimeseriesContextVisitor( + SchemaNode root, PartialPath pathPattern, PathPatternTree scope) { + return new SchemaTreeTimeseriesContextVisitor(root, pathPattern, false, scope); + } + public static SchemaTreeVisitorWithLimitOffsetWrapper createSchemaTreeMeasurementVisitor( SchemaNode root, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 4b1a8cff7a093..127ae8b922b6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -150,6 +150,7 @@ import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.commons.lang3.tuple.Triple; import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.TimeRange; @@ -2882,28 +2883,20 @@ private boolean analyzeTimeseriesRegionScan( Map> deviceToMeasurementSchemaMap = new HashMap<>(); Map> deviceToTimeseriesSchemaMap = new HashMap<>(); for (PartialPath pattern : patternTree.getAllPathPatterns()) { - List measurementPathList = schemaTree.searchMeasurementPaths(pattern).left; - for (MeasurementPath measurementPath : measurementPathList) { - boolean isAligned = measurementPath.isUnderAlignedEntity(); - PartialPath devicePath = measurementPath.getDevicePath(); - deviceToAlignedMap.put(devicePath, isAligned); + List> contextList = + schemaTree.searchMeasurementSchemaInfo(pattern); + for (Triple item : contextList) { + PartialPath devicePath = item.getLeft().getDevicePath(); + deviceToAlignedMap.put(devicePath, item.getMiddle()); deviceToMeasurementMap .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(measurementPath.getMeasurement()); + .add(item.getLeft().getMeasurement()); deviceToMeasurementSchemaMap .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(measurementPath.getMeasurementSchema()); + .add(item.getRight().getSchema()); deviceToTimeseriesSchemaMap .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add( - new TimeseriesContext( - new MeasurementSchemaInfo( - measurementPath.getMeasurement(), - measurementPath.getMeasurementSchema(), - measurementPath.isMeasurementAliasExists() - ? measurementPath.getMeasurementAlias() - : null, - measurementPath.getTagMap()))); + .add(new TimeseriesContext(item.getRight())); } } From 88223a4ad2c6082475c28401edc6ddefbfb1afd4 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Fri, 14 Jun 2024 14:26:22 +0800 Subject: [PATCH 09/12] add attributes interface --- .../queryengine/common/TimeseriesContext.java | 33 ++--- .../common/schematree/ClusterSchemaTree.java | 21 +-- .../schematree/IMeasurementSchemaInfo.java | 2 + .../common/schematree/ISchemaTree.java | 11 -- .../schematree/MeasurementSchemaInfo.java | 13 +- .../node/SchemaMeasurementNode.java | 14 +- .../visitor/SchemaTreeDeviceVisitor.java | 3 +- .../SchemaTreeTimeseriesContextVisitor.java | 136 ------------------ .../visitor/SchemaTreeVisitorFactory.java | 5 - .../schema/SchemaFetchScanOperator.java | 28 +++- .../ActiveTimeSeriesRegionScanOperator.java | 2 +- .../plan/analyze/AnalyzeVisitor.java | 83 ++++------- .../DeviceUsingTemplateSchemaCache.java | 5 + .../cache/schema/SchemaCacheEntry.java | 5 + .../cache/schema/TimeSeriesSchemaCache.java | 2 + .../schema/AutoCreateSchemaExecutor.java | 2 + .../schema/ClusterSchemaFetchExecutor.java | 31 +++- .../analyze/schema/ClusterSchemaFetcher.java | 13 +- .../plan/analyze/schema/ISchemaFetcher.java | 10 +- .../plan/planner/LogicalPlanBuilder.java | 8 +- .../plan/planner/LogicalPlanVisitor.java | 4 +- .../plan/planner/OperatorTreeGenerator.java | 4 +- .../read/SeriesSchemaFetchScanNode.java | 40 +++++- .../internal/SeriesSchemaFetchStatement.java | 16 ++- .../schemaregion/ISchemaRegion.java | 4 +- .../impl/SchemaRegionMemoryImpl.java | 15 +- .../impl/SchemaRegionPBTreeImpl.java | 4 +- .../impl/mem/MTreeBelowSGMemoryImpl.java | 43 ++++-- .../schemaregion/tag/TagManager.java | 15 ++ .../cache/DataNodeSchemaCacheTest.java | 6 + .../schemaRegion/SchemaRegionBasicTest.java | 13 +- .../SchemaRegionSimpleRecoverTest.java | 14 +- .../SchemaRegionTemplateTest.java | 13 +- .../schematree/ClusterSchemaTreeTest.java | 2 + .../operator/OperatorMemoryTest.java | 2 + .../schema/SchemaFetchScanOperatorTest.java | 14 +- .../plan/analyze/FakeSchemaFetcherImpl.java | 9 +- .../plan/planner/distribution/Util.java | 8 +- .../plan/planner/distribution/Util2.java | 8 +- .../logical/RegionScanLogicalPlannerTest.java | 17 +-- .../read/SchemaFetchMergeNodeTest.java | 4 +- .../read/SeriesSchemaFetchScanNodeTest.java | 4 +- 42 files changed, 380 insertions(+), 306 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 638716f832d2e..9225b97fc70ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -24,12 +24,10 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; import java.util.Objects; import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; @@ -40,9 +38,7 @@ public class TimeseriesContext { private final String compression; private final String tags; private final String alias; - - // TODO: Currently we can't get attributes from fetchSeriesSchema in query - // private final String attributes; + private final String attributes; private final String deadband; private final String deadbandParameters; @@ -53,23 +49,13 @@ public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { this.compression = schemaInfo.getSchema().getCompressor().toString(); this.alias = schemaInfo.getAlias(); this.tags = mapToString(schemaInfo.getTagMap()); + this.attributes = mapToString(schemaInfo.getAttributeMap()); Pair deadbandInfo = MetaUtils.parseDeadbandInfo(schemaInfo.getSchema().getProps()); this.deadband = deadbandInfo.left; this.deadbandParameters = deadbandInfo.right; } - public TimeseriesContext(IMeasurementSchema schema, String alias, Map tagMap) { - this.dataType = schema.getType().toString(); - this.alias = alias; - this.encoding = schema.getEncodingType().toString(); - this.compression = schema.getCompressor().toString(); - this.tags = mapToString(tagMap); - Pair deadbandInfo = MetaUtils.parseDeadbandInfo(schema.getProps()); - this.deadband = deadbandInfo.left; - this.deadbandParameters = deadbandInfo.right; - } - public String getDataType() { return dataType; } @@ -90,6 +76,10 @@ public String getTags() { return tags; } + public String getAttributes() { + return attributes; + } + public String getDeadbandParameters() { return deadbandParameters; } @@ -104,6 +94,7 @@ public TimeseriesContext( String encoding, String compression, String tags, + String attributes, String deadband, String deadbandParameters) { this.dataType = dataType; @@ -111,6 +102,7 @@ public TimeseriesContext( this.encoding = encoding; this.compression = compression; this.tags = tags; + this.attributes = attributes; this.deadband = deadband; this.deadbandParameters = deadbandParameters; } @@ -121,6 +113,7 @@ public void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(encoding, byteBuffer); ReadWriteIOUtils.write(compression, byteBuffer); ReadWriteIOUtils.write(tags, byteBuffer); + ReadWriteIOUtils.write(attributes, byteBuffer); ReadWriteIOUtils.write(deadband, byteBuffer); ReadWriteIOUtils.write(deadbandParameters, byteBuffer); } @@ -131,6 +124,7 @@ public void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(encoding, stream); ReadWriteIOUtils.write(compression, stream); ReadWriteIOUtils.write(tags, stream); + ReadWriteIOUtils.write(attributes, stream); ReadWriteIOUtils.write(deadband, stream); ReadWriteIOUtils.write(deadbandParameters, stream); } @@ -141,10 +135,11 @@ public static TimeseriesContext deserialize(ByteBuffer buffer) { String encoding = ReadWriteIOUtils.readString(buffer); String compression = ReadWriteIOUtils.readString(buffer); String tags = ReadWriteIOUtils.readString(buffer); + String attributes = ReadWriteIOUtils.readString(buffer); String deadband = ReadWriteIOUtils.readString(buffer); String deadbandParameters = ReadWriteIOUtils.readString(buffer); return new TimeseriesContext( - dataType, alias, encoding, compression, tags, deadband, deadbandParameters); + dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); } @Override @@ -162,6 +157,7 @@ public boolean equals(Object obj) { && encoding.equals(that.encoding) && Objects.equals(compression, that.compression) && Objects.equals(tags, that.tags) + && Objects.equals(attributes, that.attributes) && Objects.equals(deadband, that.deadband) && Objects.equals(deadbandParameters, that.deadbandParameters); return res; @@ -169,6 +165,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(dataType, alias, encoding, compression, tags, deadband, deadbandParameters); + return Objects.hash( + dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index 9b5257baa395e..66dcb77fd08c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -33,14 +33,12 @@ import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeDeviceUsingTemplateVisitor; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeDeviceVisitor; -import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeTimeseriesContextVisitor; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorWithLimitOffsetWrapper; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; -import org.apache.commons.lang3.tuple.Triple; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -120,16 +118,6 @@ public Pair, Integer> searchMeasurementPaths(PartialPath p return searchMeasurementPaths(pathPattern, 0, 0, false); } - @Override - public List> searchMeasurementSchemaInfo( - PartialPath pathPattern) { - try (SchemaTreeTimeseriesContextVisitor visitor = - SchemaTreeVisitorFactory.createTimeseriesContextVisitor( - root, pathPattern, authorityScope)) { - return visitor.getAllResult(); - } - } - public List getAllDevices() { return getMatchedDevices(ALL_MATCH_PATTERN); } @@ -167,7 +155,8 @@ private IMeasurementSchemaInfo getMeasurementSchemaInfo( } else { Template template = templateMap.get(entityNode.getTemplateId()); if (template != null && template.getSchemaMap().containsKey(measurement)) { - return new MeasurementSchemaInfo(measurement, template.getSchema(measurement), null, null); + return new MeasurementSchemaInfo( + measurement, template.getSchema(measurement), null, null, null); } } return null; @@ -279,7 +268,8 @@ public void computeSourceOfLogicalView( measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), null, - measurementPath.getTagMap()), + measurementPath.getTagMap(), + null), measurementPath.isUnderAlignedEntity()); } } @@ -339,6 +329,7 @@ public void appendSingleMeasurementPath(MeasurementPath measurementPath) { measurementPath, measurementPath.getMeasurementSchema(), measurementPath.getTagMap(), + null, measurementPath.isMeasurementAliasExists() ? measurementPath.getMeasurementAlias() : null, measurementPath.isUnderAlignedEntity()); } @@ -347,6 +338,7 @@ public void appendSingleMeasurement( PartialPath path, IMeasurementSchema schema, Map tagMap, + Map attributeMap, String alias, boolean isAligned) { String[] nodes = path.getNodes(); @@ -362,6 +354,7 @@ public void appendSingleMeasurement( cur.getAsEntityNode().addAliasChild(alias, measurementNode); } measurementNode.setTagMap(tagMap); + measurementNode.setAttributeMap(attributeMap); child = measurementNode; if (schema.isLogicalView()) { this.hasLogicalMeasurementPath = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java index 7200c9a5e5f21..5636363f800f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java @@ -43,6 +43,8 @@ public interface IMeasurementSchemaInfo { Map getTagMap(); + Map getAttributeMap(); + String getAlias(); boolean isLogicalView(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java index fdfad14ab87b0..0288033b6955c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.schemaengine.template.Template; -import org.apache.commons.lang3.tuple.Triple; import org.apache.tsfile.utils.Pair; import java.util.List; @@ -50,16 +49,6 @@ Pair, Integer> searchMeasurementPaths( */ Pair, Integer> searchMeasurementPaths(PartialPath pathPattern); - /** - * Search all timeseries schema info that match the given path pattern. Only use for active schema - * query. Only non-view timeseries will be returned. - * - * @param pathPattern pattern - * @return list of triple [path, isAligned, schema] - */ - List> searchMeasurementSchemaInfo( - PartialPath pathPattern); - /** * Get all device matching the path pattern. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java index a8225dbcb0ad7..9db3ebadbfded 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java @@ -37,14 +37,20 @@ public class MeasurementSchemaInfo implements IMeasurementSchemaInfo { private final String name; private final String alias; private Map tagMap; + private Map attributeMap; private final IMeasurementSchema schema; public MeasurementSchemaInfo( - String name, IMeasurementSchema schema, String alias, Map tagMap) { + String name, + IMeasurementSchema schema, + String alias, + Map tagMap, + Map attributeMap) { this.name = name; this.schema = schema; this.alias = alias; this.tagMap = tagMap; + this.attributeMap = attributeMap; } public String getName() { @@ -68,6 +74,11 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return attributeMap; + } + @Override public LogicalViewSchema getSchemaAsLogicalViewSchema() { if (this.isLogicalView()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java index 55657a59700be..b1eaa7ff5c100 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java @@ -36,8 +36,8 @@ public class SchemaMeasurementNode extends SchemaNode implements IMeasurementSch private String alias; private IMeasurementSchema schema; - private Map tagMap; + private Map attributeMap; public SchemaMeasurementNode(String name, IMeasurementSchema schema) { super(name); @@ -82,6 +82,15 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return attributeMap; + } + + public void setAttributeMap(Map attributeMap) { + this.attributeMap = attributeMap; + } + @Override public SchemaNode getChild(String name) { return null; @@ -144,6 +153,7 @@ public void serialize(OutputStream outputStream) throws IOException { schema.serializeTo(outputStream); ReadWriteIOUtils.write(tagMap, outputStream); + ReadWriteIOUtils.write(attributeMap, outputStream); } public static SchemaMeasurementNode deserialize(InputStream inputStream) throws IOException { @@ -161,10 +171,12 @@ public static SchemaMeasurementNode deserialize(InputStream inputStream) throws } Map tagMap = ReadWriteIOUtils.readMap(inputStream); + Map attributeMap = ReadWriteIOUtils.readMap(inputStream); SchemaMeasurementNode measurementNode = new SchemaMeasurementNode(name, schema); measurementNode.setAlias(alias); measurementNode.setTagMap(tagMap); + measurementNode.setAttributeMap(attributeMap); return measurementNode; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java index 42fe1b15db19a..684035cce16f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java @@ -67,7 +67,8 @@ protected DeviceSchemaInfo generateResult(SchemaNode nextMatchedNode) { measurementNode.getName(), measurementNode.getSchema(), measurementNode.getAlias(), - measurementNode.getTagMap())); + measurementNode.getTagMap(), + measurementNode.getAttributeMap())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java deleted file mode 100644 index 777d4fa26cddc..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeTimeseriesContextVisitor.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.common.schematree.visitor; - -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.path.fa.IFAState; -import org.apache.iotdb.commons.path.fa.IFATransition; -import org.apache.iotdb.db.queryengine.common.schematree.MeasurementSchemaInfo; -import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaMeasurementNode; -import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; - -import org.apache.commons.lang3.tuple.Triple; - -import java.util.Map; - -public class SchemaTreeTimeseriesContextVisitor - extends SchemaTreeVisitor> { - public SchemaTreeTimeseriesContextVisitor( - SchemaNode root, PartialPath pathPattern, boolean isPrefixMatch) { - super(root, pathPattern, isPrefixMatch); - } - - public SchemaTreeTimeseriesContextVisitor( - SchemaNode root, PartialPath pathPattern, boolean isPrefixMatch, PathPatternTree scope) { - super(root, pathPattern, isPrefixMatch, scope); - } - - @Override - protected boolean mayTargetNodeType(SchemaNode node) { - return node.isMeasurement(); - } - - @Override - protected IFAState tryGetNextState( - SchemaNode node, IFAState sourceState, Map preciseMatchTransitionMap) { - IFATransition transition; - IFAState state; - if (node.isMeasurement()) { - String alias = node.getAsMeasurementNode().getAlias(); - if (alias != null) { - transition = preciseMatchTransitionMap.get(alias); - if (transition != null) { - state = patternFA.getNextState(sourceState, transition); - if (state.isFinal()) { - return state; - } - } - } - transition = preciseMatchTransitionMap.get(node.getName()); - if (transition != null) { - state = patternFA.getNextState(sourceState, transition); - if (state.isFinal()) { - return state; - } - } - return null; - } - - transition = preciseMatchTransitionMap.get(node.getName()); - if (transition == null) { - return null; - } - return patternFA.getNextState(sourceState, transition); - } - - @Override - protected IFAState tryGetNextState( - SchemaNode node, IFAState sourceState, IFATransition transition) { - IFAState state; - if (node.isMeasurement()) { - String alias = node.getAsMeasurementNode().getAlias(); - if (alias != null && transition.isMatch(alias)) { - state = patternFA.getNextState(sourceState, transition); - if (state.isFinal()) { - return state; - } - } - if (transition.isMatch(node.getName())) { - state = patternFA.getNextState(sourceState, transition); - if (state.isFinal()) { - return state; - } - } - return null; - } - - if (transition.isMatch(node.getName())) { - return patternFA.getNextState(sourceState, transition); - } - return null; - } - - @Override - protected boolean acceptInternalMatchedNode(SchemaNode node) { - return false; - } - - @Override - protected boolean acceptFullMatchedNode(SchemaNode node) { - return node.isMeasurement() && !node.getAsMeasurementNode().isLogicalView(); - } - - @Override - protected Triple generateResult( - SchemaNode nextMatchedNode) { - SchemaMeasurementNode measurementNode = nextMatchedNode.getAsMeasurementNode(); - MeasurementSchemaInfo measurementSchemaInfo = - new MeasurementSchemaInfo( - measurementNode.getName(), - measurementNode.getSchema(), - measurementNode.getAlias(), - measurementNode.getTagMap()); - return Triple.of( - getPartialPathFromRootToNode(nextMatchedNode), - getParentOfNextMatchedNode().getAsEntityNode().isAligned(), - measurementSchemaInfo); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java index 0e9a465911ea3..8b4c722af5fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorFactory.java @@ -56,11 +56,6 @@ public static SchemaTreeMeasurementVisitor createSchemaTreeMeasurementVisitor( return new SchemaTreeMeasurementVisitor(root, pathPattern, false, scope); } - public static SchemaTreeTimeseriesContextVisitor createTimeseriesContextVisitor( - SchemaNode root, PartialPath pathPattern, PathPatternTree scope) { - return new SchemaTreeTimeseriesContextVisitor(root, pathPattern, false, scope); - } - public static SchemaTreeVisitorWithLimitOffsetWrapper createSchemaTreeMeasurementVisitor( SchemaNode root, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java index 47a80841a4bfc..f4b40c69b79e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java @@ -54,7 +54,10 @@ public class SchemaFetchScanOperator implements SourceOperator { private final ISchemaRegion schemaRegion; private final boolean withTags; + private final boolean withAttributes; private final boolean withTemplate; + private final boolean withAliasForce; + private final boolean fetchDevice; private boolean isFinished = false; private final PathPatternTree authorityScope; @@ -72,9 +75,19 @@ public static SchemaFetchScanOperator ofSeries( Map templateMap, ISchemaRegion schemaRegion, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { return new SchemaFetchScanOperator( - planNodeId, context, patternTree, templateMap, schemaRegion, withTags, withTemplate); + planNodeId, + context, + patternTree, + templateMap, + schemaRegion, + withTags, + withAttributes, + withTemplate, + withAliasForce); } public static SchemaFetchScanOperator ofDevice( @@ -94,14 +107,18 @@ private SchemaFetchScanOperator( Map templateMap, ISchemaRegion schemaRegion, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { this.sourceId = planNodeId; this.operatorContext = context; this.patternTree = patternTree; this.schemaRegion = schemaRegion; this.templateMap = templateMap; this.withTags = withTags; + this.withAttributes = withAttributes; this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; this.fetchDevice = false; this.authorityScope = SchemaConstant.ALL_MATCH_SCOPE; } @@ -118,7 +135,9 @@ private SchemaFetchScanOperator( this.schemaRegion = schemaRegion; this.templateMap = Collections.emptyMap(); this.withTags = false; + this.withAttributes = false; this.withTemplate = false; + this.withAliasForce = false; this.fetchDevice = true; this.authorityScope = authorityScope; } @@ -165,7 +184,8 @@ private TsBlock fetchSchema() throws MetadataException { ClusterSchemaTree schemaTree = fetchDevice ? schemaRegion.fetchDeviceSchema(patternTree, authorityScope) - : schemaRegion.fetchSeriesSchema(patternTree, templateMap, withTags, withTemplate); + : schemaRegion.fetchSeriesSchema( + patternTree, templateMap, withTags, withAttributes, withTemplate, withAliasForce); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java index 6be648fb6bc4e..b947bfcee89e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java @@ -124,7 +124,7 @@ protected void updateActiveData() { checkAndAppend(schemaInfo.getEncoding(), columnBuilders[4]); // Encoding checkAndAppend(schemaInfo.getCompression(), columnBuilders[5]); // Compression checkAndAppend(schemaInfo.getTags(), columnBuilders[6]); // Tags - columnBuilders[7].appendNull(); // Attributes + checkAndAppend(schemaInfo.getAttributes(), columnBuilders[7]); // Attributes checkAndAppend(schemaInfo.getDeadband(), columnBuilders[8]); // Description checkAndAppend(schemaInfo.getDeadbandParameters(), columnBuilders[9]); // DeadbandParameters columnBuilders[10].writeBinary(VIEW_TYPE); // ViewType diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 127ae8b922b6c..720b365f4e98d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -34,7 +34,6 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -57,8 +56,8 @@ import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; -import org.apache.iotdb.db.queryengine.common.schematree.MeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.execution.operator.window.WindowType; import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; @@ -150,7 +149,6 @@ import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.commons.lang3.tuple.Triple; import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.TimeRange; @@ -2867,73 +2865,54 @@ private boolean analyzeTimeseriesRegionScan( context.generateGlobalTimeFilter(analysis); patternTree.constructTree(); ISchemaTree schemaTree = - schemaFetcher.fetchSchemaWithTags( - PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), - false, - context); + schemaFetcher.fetchRawSchemaInMeasurementLevel(patternTree, authorityScope, context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return false; } removeLogicViewMeasurement(schemaTree); - - Map> deviceToMeasurementMap = new HashMap<>(); - Map deviceToAlignedMap = new HashMap<>(); - Map> deviceToMeasurementSchemaMap = new HashMap<>(); - Map> deviceToTimeseriesSchemaMap = new HashMap<>(); - for (PartialPath pattern : patternTree.getAllPathPatterns()) { - List> contextList = - schemaTree.searchMeasurementSchemaInfo(pattern); - for (Triple item : contextList) { - PartialPath devicePath = item.getLeft().getDevicePath(); - deviceToAlignedMap.put(devicePath, item.getMiddle()); - deviceToMeasurementMap - .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(item.getLeft().getMeasurement()); - deviceToMeasurementSchemaMap - .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(item.getRight().getSchema()); - deviceToTimeseriesSchemaMap - .computeIfAbsent(devicePath, k -> new ArrayList<>()) - .add(new TimeseriesContext(item.getRight())); - } - } - - Map>> deviceToTimeseriesSchemaInfo = + Map>> deviceToTimeseriesContext = new HashMap<>(); - for (PartialPath devicePath : deviceToAlignedMap.keySet()) { - if (deviceToAlignedMap.get(devicePath)) { - List measurementList = deviceToMeasurementMap.get(devicePath); - List schemaList = deviceToMeasurementSchemaMap.get(devicePath); - List timeseriesContextList = deviceToTimeseriesSchemaMap.get(devicePath); + List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); + Set deviceSet = new HashSet<>(); + for (DeviceSchemaInfo deviceSchemaInfo : deviceSchemaInfoList) { + boolean isAligned = deviceSchemaInfo.isAligned(); + PartialPath devicePath = deviceSchemaInfo.getDevicePath(); + deviceSet.add(devicePath.getFullPath()); + if (isAligned) { + List measurementList = new ArrayList<>(); + List schemaList = new ArrayList<>(); + List timeseriesContextList = new ArrayList<>(); + for (IMeasurementSchemaInfo measurementSchemaInfo : + deviceSchemaInfo.getMeasurementSchemaInfoList()) { + schemaList.add(measurementSchemaInfo.getSchema()); + measurementList.add(measurementSchemaInfo.getName()); + timeseriesContextList.add(new TimeseriesContext(measurementSchemaInfo)); + } AlignedPath alignedPath = new AlignedPath(devicePath.getNodes(), measurementList, schemaList); - deviceToTimeseriesSchemaInfo + deviceToTimeseriesContext .computeIfAbsent(devicePath, k -> new HashMap<>()) .put(alignedPath, timeseriesContextList); } else { - List measurementList = deviceToMeasurementMap.get(devicePath); - List timeseriesContextList = deviceToTimeseriesSchemaMap.get(devicePath); - for (int i = 0; i < measurementList.size(); i++) { + for (IMeasurementSchemaInfo measurementSchemaInfo : + deviceSchemaInfo.getMeasurementSchemaInfoList()) { MeasurementPath measurementPath = - new MeasurementPath(devicePath.concatNode(measurementList.get(i)).getNodes()); - deviceToTimeseriesSchemaInfo + new MeasurementPath( + devicePath.concatNode(measurementSchemaInfo.getName()).getNodes()); + deviceToTimeseriesContext .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(measurementPath, Collections.singletonList(timeseriesContextList.get(i))); + .put( + measurementPath, + Collections.singletonList(new TimeseriesContext(measurementSchemaInfo))); } } } - analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesSchemaInfo); + analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesContext); // fetch Data partition - DataPartition dataPartition = - fetchDataPartitionByDevices( - deviceToAlignedMap.keySet().stream() - .map(PartialPath::getFullPath) - .collect(Collectors.toSet()), - schemaTree, - context); + DataPartition dataPartition = fetchDataPartitionByDevices(deviceSet, schemaTree, context); analysis.setDataPartitionInfo(dataPartition); return true; } @@ -3050,7 +3029,7 @@ private void analyzeDeviceRegionScan( PathPatternTree patternTree = new PathPatternTree(); patternTree.appendPathPattern(pattern); ISchemaTree schemaTree = - schemaFetcher.fetchSchemaInDeviceLevel(patternTree, authorityScope, context); + schemaFetcher.fetchRawSchemaInDeviceLevel(patternTree, authorityScope, context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java index 1ea59da13554c..167242ff20cc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java @@ -168,6 +168,11 @@ public Map getTagMap() { return null; } + @Override + public Map getAttributeMap() { + return null; + } + @Override public String getAlias() { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java index d3d99c2548be8..bf6d5c31a86c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java @@ -70,6 +70,11 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return null; + } + public TSDataType getTsDataType() { return iMeasurementSchema.getType(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java index c912a7c221f7b..1909e1558e7b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java @@ -111,6 +111,7 @@ public void computeValue(int index, SchemaCacheEntry value) { value.getIMeasurementSchema(), value.getTagMap(), null, + null, value.isAligned()); storageGroupSet.add(value.getStorageGroup()); } @@ -130,6 +131,7 @@ public ClusterSchemaTree get(PartialPath fullPath) { schemaCacheEntry.getIMeasurementSchema(), schemaCacheEntry.getTagMap(), null, + null, schemaCacheEntry.isAligned()); schemaTree.setDatabases(Collections.singleton(schemaCacheEntry.getStorageGroup())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java index 630ca1f8cce28..3d2cae267fc23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java @@ -493,6 +493,7 @@ private void internalCreateTimeSeries( measurements.get(i), tsDataTypes.get(i), encodings.get(i), compressors.get(i)), null, null, + null, isAligned); } } @@ -620,6 +621,7 @@ private void internalCreateTimeSeries( measurementGroup.getCompressors().get(i)), null, null, + null, entry.getValue().left); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java index ce61926b8faf2..64fe751c37785 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java @@ -108,7 +108,9 @@ ClusterSchemaTree fetchSchemaOfFuzzyMatch( templateMap.putAll(templateManager.checkAllRelatedTemplate(pattern)); } return executeSchemaFetchQuery( - new SeriesSchemaFetchStatement(patternTree, templateMap, withTags, withTemplate), context); + new SeriesSchemaFetchStatement( + patternTree, templateMap, withTags, false, withTemplate, false), + context); } /** @@ -127,7 +129,12 @@ ClusterSchemaTree fetchSchemaOfPreciseMatchOrPreciseDeviceUsingTemplate( ClusterSchemaTree schemaTree = executeSchemaFetchQuery( new SeriesSchemaFetchStatement( - rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), + rawPatternTree, + analyzeTemplate(fullPathList), + false, + withTemplate, + withTemplate, + withTemplate), context); if (!schemaTree.isEmpty()) { schemaCacheUpdater.accept(schemaTree); @@ -135,12 +142,23 @@ rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), return schemaTree; } - ClusterSchemaTree fetchDeviceLevelSchema( + ClusterSchemaTree fetchDeviceLevelRawSchema( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { return executeSchemaFetchQuery( new DeviceSchemaFetchStatement(patternTree, authorityScope), context); } + ClusterSchemaTree fetchMeasurementLevelRawSchema( + PathPatternTree patternTree, MPPQueryContext context) { + Map templateMap = new HashMap<>(); + List pathPatternList = patternTree.getAllPathPatterns(); + for (PartialPath pattern : pathPatternList) { + templateMap.putAll(templateManager.checkAllRelatedTemplate(pattern)); + } + return executeSchemaFetchQuery( + new SeriesSchemaFetchStatement(patternTree, templateMap, true, true, false, true), context); + } + ClusterSchemaTree fetchSchemaOfOneDevice( PartialPath devicePath, String[] measurements, @@ -197,7 +215,12 @@ private ClusterSchemaTree fetchSchemaAndCacheResult( ClusterSchemaTree schemaTree = executeSchemaFetchQuery( new SeriesSchemaFetchStatement( - patternTree, analyzeTemplate(patternTree.getAllPathPatterns()), false, true), + patternTree, + analyzeTemplate(patternTree.getAllPathPatterns()), + false, + false, + true, + false), context); if (!schemaTree.isEmpty()) { schemaCacheUpdater.accept(schemaTree); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index 2c3aced4ba16c..11c0979a2b7da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -156,10 +157,18 @@ public ClusterSchemaTree fetchSchema( } @Override - public ISchemaTree fetchSchemaInDeviceLevel( + public ISchemaTree fetchRawSchemaInDeviceLevel( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { authorityScope.constructTree(); - return clusterSchemaFetchExecutor.fetchDeviceLevelSchema(patternTree, authorityScope, context); + return clusterSchemaFetchExecutor.fetchDeviceLevelRawSchema( + patternTree, authorityScope, context); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return clusterSchemaFetchExecutor.fetchMeasurementLevelRawSchema( + PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java index 080e93b63532a..56a7670ad9d9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java @@ -55,7 +55,15 @@ ISchemaTree fetchSchema( * * @return schemaTree without measurement nodes */ - ISchemaTree fetchSchemaInDeviceLevel( + ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context); + + /** + * Fetch all the schema by the given patternTree in device level + * + * @return schemaTree without measurement nodes + */ + ISchemaTree fetchRawSchemaInMeasurementLevel( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context); /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 3ff06fb338a94..1af2387b0d51d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -1091,7 +1091,9 @@ public LogicalPlanBuilder planSeriesSchemaFetchSource( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { PartialPath storageGroupPath; for (String storageGroup : storageGroupList) { try { @@ -1110,7 +1112,9 @@ public LogicalPlanBuilder planSeriesSchemaFetchSource( overlappedPatternTree, templateMap, withTags, - withTemplate)); + withAttributes, + withTemplate, + withAliasForce)); } catch (IllegalPathException e) { // definitely won't happen throw new RuntimeException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 96307945e462b..f3c3c26561e69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -789,7 +789,9 @@ public PlanNode visitSeriesSchemaFetch( seriesSchemaFetchStatement.getPatternTree(), seriesSchemaFetchStatement.getTemplateMap(), seriesSchemaFetchStatement.isWithTags(), - seriesSchemaFetchStatement.isWithTemplate()) + seriesSchemaFetchStatement.isWithAttributes(), + seriesSchemaFetchStatement.isWithTemplate(), + seriesSchemaFetchStatement.isWithAliasForce()) .getRoot(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 1127cbd1683ba..647d443a0c3ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2611,7 +2611,9 @@ public Operator visitSeriesSchemaFetchScan( node.getTemplateMap(), ((SchemaDriverContext) (context.getDriverContext())).getSchemaRegion(), node.isWithTags(), - node.isWithTemplate()); + node.isWithAttributes(), + node.isWithTemplate(), + node.isWithAliasForce()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java index 6f6d5ee491a13..8f3bab5e9cc10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java @@ -42,7 +42,9 @@ public class SeriesSchemaFetchScanNode extends SchemaFetchScanNode { private final Map templateMap; private final boolean withTags; + private final boolean withAttributes; private final boolean withTemplate; + private final boolean withAliasForce; public SeriesSchemaFetchScanNode( PlanNodeId id, @@ -50,11 +52,15 @@ public SeriesSchemaFetchScanNode( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { super(id, storageGroup, patternTree); this.templateMap = templateMap; this.withTags = withTags; + this.withAttributes = withAttributes; this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; } public Map getTemplateMap() { @@ -69,7 +75,14 @@ public PlanNodeType getType() { @Override public PlanNode clone() { return new SeriesSchemaFetchScanNode( - getPlanNodeId(), storageGroup, patternTree, templateMap, withTags, withTemplate); + getPlanNodeId(), + storageGroup, + patternTree, + templateMap, + withTags, + withAttributes, + withTemplate, + withAliasForce); } @Override @@ -92,7 +105,9 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { template.serialize(byteBuffer); } ReadWriteIOUtils.write(withTags, byteBuffer); + ReadWriteIOUtils.write(withAttributes, byteBuffer); ReadWriteIOUtils.write(withTemplate, byteBuffer); + ReadWriteIOUtils.write(withAliasForce, byteBuffer); } @Override @@ -105,7 +120,9 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { template.serialize(stream); } ReadWriteIOUtils.write(withTags, stream); + ReadWriteIOUtils.write(withAttributes, stream); ReadWriteIOUtils.write(withTemplate, stream); + ReadWriteIOUtils.write(withAliasForce, stream); } public static SeriesSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { @@ -121,10 +138,19 @@ public static SeriesSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { templateMap.put(template.getId(), template); } boolean withTags = ReadWriteIOUtils.readBool(byteBuffer); + boolean withAttributes = ReadWriteIOUtils.readBool(byteBuffer); boolean withTemplate = ReadWriteIOUtils.readBool(byteBuffer); + boolean withAliasForce = ReadWriteIOUtils.readBool(byteBuffer); PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); return new SeriesSchemaFetchScanNode( - planNodeId, storageGroup, patternTree, templateMap, withTags, withTemplate); + planNodeId, + storageGroup, + patternTree, + templateMap, + withTags, + withAttributes, + withTemplate, + withAliasForce); } @Override @@ -139,4 +165,12 @@ public boolean isWithTags() { public boolean isWithTemplate() { return withTemplate; } + + public boolean isWithAttributes() { + return withAttributes; + } + + public boolean isWithAliasForce() { + return withAliasForce; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java index be48b631bba95..f0b393c817f81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java @@ -34,18 +34,24 @@ public class SeriesSchemaFetchStatement extends Statement { private final PathPatternTree patternTree; private final Map templateMap; private final boolean withTags; + private final boolean withAttributes; private final boolean withTemplate; + private final boolean withAliasForce; public SeriesSchemaFetchStatement( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { super(); this.patternTree = patternTree; this.templateMap = templateMap; this.withTags = withTags; + this.withAttributes = withAttributes; this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; setType(StatementType.FETCH_SCHEMA); } @@ -74,4 +80,12 @@ public boolean isWithTags() { public boolean isWithTemplate() { return withTemplate; } + + public boolean isWithAttributes() { + return withAttributes; + } + + public boolean isWithAliasForce() { + return withAliasForce; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 833e83e10b2e9..61ef64c1d2638 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -202,7 +202,9 @@ ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException; /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 12225a29dc011..e9173cc5888a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -214,6 +214,7 @@ public synchronized void init() throws MetadataException { new MTreeBelowSGMemoryImpl( new PartialPath(storageGroupFullPath), tagManager::readTags, + tagManager::readAttributes, regionStatistics, metric); @@ -510,7 +511,8 @@ public void loadSnapshot(File latestSnapshotRootDir) { regionStatistics.activateTemplate(deviceMNode.getSchemaTemplateId()); } }, - tagManager::readTags); + tagManager::readTags, + tagManager::readAttributes); logger.info( "MTree snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, @@ -953,16 +955,21 @@ public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { if (patternTree.isContainWildcard()) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); for (PartialPath path : patternTree.getAllPathPatterns()) { - schemaTree.mergeSchemaTree(mtree.fetchSchema(path, templateMap, withTags, withTemplate)); + schemaTree.mergeSchemaTree( + mtree.fetchSchema( + path, templateMap, withTags, withAttributes, withTemplate, withAliasForce)); } return schemaTree; } else { - return mtree.fetchSchemaWithoutWildcard(patternTree, templateMap, withTags, withTemplate); + return mtree.fetchSchemaWithoutWildcard( + patternTree, templateMap, withTags, withAttributes, withTemplate); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 13da1fae890ae..e48ba2e01a8e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1031,7 +1031,9 @@ public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForcete) throws MetadataException { if (patternTree.isContainWildcard()) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 97cc5f8828aeb..c1a977d0300d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -122,6 +122,7 @@ public class MTreeBelowSGMemoryImpl { private final IMemMNode rootNode; private final Function, Map> tagGetter; + private final Function, Map> attributeGetter; private final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); private final int levelOfSG; @@ -131,6 +132,7 @@ public class MTreeBelowSGMemoryImpl { public MTreeBelowSGMemoryImpl( PartialPath storageGroupPath, Function, Map> tagGetter, + Function, Map> attributeGetter, MemSchemaRegionStatistics regionStatistics, SchemaRegionMemMetric metric) { store = new MemMTreeStore(storageGroupPath, regionStatistics, metric); @@ -139,12 +141,14 @@ public MTreeBelowSGMemoryImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; } private MTreeBelowSGMemoryImpl( PartialPath storageGroupPath, MemMTreeStore store, Function, Map> tagGetter, + Function, Map> attributeGetter, MemSchemaRegionStatistics regionStatistics) { this.store = store; this.regionStatistics = regionStatistics; @@ -152,6 +156,7 @@ private MTreeBelowSGMemoryImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; } public void clear() { @@ -170,13 +175,15 @@ public static MTreeBelowSGMemoryImpl loadFromSnapshot( SchemaRegionMemMetric metric, Consumer> measurementProcess, Consumer> deviceProcess, - Function, Map> tagGetter) + Function, Map> tagGetter, + Function, Map> attributeGetter) throws IOException, IllegalPathException { return new MTreeBelowSGMemoryImpl( new PartialPath(storageGroupFullPath), MemMTreeStore.loadFromSnapshot( snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric), tagGetter, + attributeGetter, regionStatistics); } @@ -695,7 +702,9 @@ public ClusterSchemaTree fetchSchema( PartialPath pathPattern, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); try (MeasurementCollector collector = @@ -710,14 +719,15 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - if (nodes[nodes.length - 1].equals(node.getAlias())) { - // only when user query with alias, the alias in path will be set - path.setMeasurementAlias(node.getAlias()); - } - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + withAliasForce || nodes[nodes.length - 1].equals(node.getAlias()) + ? node.getAlias() + : null, + deviceMNode.isAligned()); } return null; } @@ -733,6 +743,7 @@ public ClusterSchemaTree fetchSchemaWithoutWildcard( PathPatternTree patternTree, Map templateMap, boolean withTags, + boolean withAttributes, boolean withTemplate) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); @@ -748,11 +759,13 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - path.setMeasurementAlias(node.getAlias()); - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + node.getAlias(), + deviceMNode.isAligned()); } return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java index 2643635731b3a..d184c4647f1d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java @@ -739,6 +739,21 @@ public Map readTags(IMeasurementMNode node) { } } + /** + * Read the attributes of this node. + * + * @param node the node to query. + * @return the attribute key-value map. + * @throws RuntimeException If any IOException happens. + */ + public Map readAttributes(IMeasurementMNode node) { + try { + return readTagFile(node.getOffset()).getRight(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public void clear() throws IOException { this.tagIndex.clear(); if (tagLogFile != null) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index f264bd4ba3b19..fc1e2f6ff6943 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -206,18 +206,21 @@ private ISchemaTree generateSchemaTree1() throws IllegalPathException { new MeasurementSchema("s1", TSDataType.INT32), s1TagMap, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s2"), new MeasurementSchema("s2", TSDataType.FLOAT), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s3"), new MeasurementSchema("s3", TSDataType.BOOLEAN), null, null, + null, false); schemaTree.setDatabases(Collections.singleton("root.sg1")); return schemaTree; @@ -231,18 +234,21 @@ private ISchemaTree generateSchemaTree2() throws IllegalPathException { new MeasurementSchema("s3", TSDataType.BOOLEAN), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s4"), new MeasurementSchema("s4", TSDataType.TEXT), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s5"), new MeasurementSchema("s5", TSDataType.INT64), null, null, + null, false); schemaTree.setDatabases(Collections.singleton("root.sg1")); return schemaTree; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 18ebf38343b83..6dcb9200c059a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -103,11 +103,11 @@ public void testFetchSchemaPerfomance() throws Exception { } } patternTree.constructTree(); - schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, false, true, false); long startTime; startTime = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { - schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, false, true, false); } System.out.println("cost time: " + (System.currentTimeMillis() - startTime)); } @@ -153,7 +153,8 @@ public void testFetchSchema() throws Exception { patternTree.constructTree(); ; ClusterSchemaTree schemas = - schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, true, true); + schemaRegion.fetchSeriesSchema( + patternTree, Collections.EMPTY_MAP, true, false, true, false); List measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.*")).left; Assert.assertEquals(measurementPaths.size(), 2); @@ -182,7 +183,9 @@ public void testFetchSchema() throws Exception { patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.wf01.wt01.temp")); patternTree.constructTree(); - schemas = schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemas = + schemaRegion.fetchSeriesSchema( + patternTree, Collections.EMPTY_MAP, false, false, true, false); measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.temp")).left; Assert.assertEquals(measurementPaths.size(), 1); @@ -368,7 +371,7 @@ public void testDeleteTimeseries() throws Exception { schemaRegion.deleteTimeseriesInBlackList(patternTree); List schemas = schemaRegion - .fetchSeriesSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, true) + .fetchSeriesSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, false, true, false) .searchMeasurementPaths(ALL_MATCH_PATTERN) .left; Assert.assertEquals(1, schemas.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index 71b5e7dd38432..0811e3c35efdc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -85,14 +85,24 @@ public void testRecoverWithAlignedTemplate() throws Exception { template); ClusterSchemaTree schemaTree = schemaRegion.fetchSeriesSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); simulateRestart(); schemaRegion = getSchemaRegion("root.sg", 0); schemaTree = schemaRegion.fetchSeriesSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java index 0747c19c72623..555a081f00151 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java @@ -78,7 +78,12 @@ public void testFetchNestedTemplateDevice() throws Exception { template); ClusterSchemaTree schemaTree = schemaRegion.fetchSeriesSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertEquals(2, schemaTree.getAllDevices().size()); for (DeviceSchemaInfo deviceSchemaInfo : schemaTree.getAllDevices()) { Assert.assertEquals(templateId, deviceSchemaInfo.getTemplateId()); @@ -213,7 +218,7 @@ public void testFetchSchemaWithTemplate() throws Exception { // check fetch schema ClusterSchemaTree schemaTree = - schemaRegion.fetchSeriesSchema(ALL_MATCH_SCOPE, templateMap, true, true); + schemaRegion.fetchSeriesSchema(ALL_MATCH_SCOPE, templateMap, true, false, true, false); schemaTree.setTemplateMap(templateMap); List schemas = schemaTree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(expectedTimeseries.size(), schemas.size()); @@ -259,7 +264,7 @@ public void testDeleteSchemaWithTemplate() throws Exception { ClusterSchemaTree schemaTree = schemaRegion.fetchSeriesSchema( - patternTree, Collections.singletonMap(templateId, template), false, true); + patternTree, Collections.singletonMap(templateId, template), false, false, true, false); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( 1, schemaTree.searchMeasurementPaths(new PartialPath("root.db.d1.s1")).left.size()); @@ -267,7 +272,7 @@ public void testDeleteSchemaWithTemplate() throws Exception { patternTree.appendFullPath(new PartialPath("root.db.d1.s3")); schemaTree = schemaRegion.fetchSeriesSchema( - patternTree, Collections.singletonMap(templateId, template), false, true); + patternTree, Collections.singletonMap(templateId, template), false, false, true, false); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( 0, schemaTree.searchMeasurementPaths(new PartialPath("root.db.d1.s3")).left.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java index e221fd7a6d113..b1b0cae80697d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java @@ -1030,6 +1030,7 @@ public void testHasView() throws IllegalPathException { new MeasurementSchema("s1", TSDataType.INT32), null, null, + null, false); Assert.assertFalse(schemaTree.hasLogicalViewMeasurement()); schemaTree.appendSingleMeasurement( @@ -1037,6 +1038,7 @@ public void testHasView() throws IllegalPathException { new LogicalViewSchema("s1", new TimeSeriesViewOperand("root.db.d.s1")), null, null, + null, false); Assert.assertTrue(schemaTree.hasLogicalViewMeasurement()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index 7812e7cddedea..335dd31c74aa2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -839,6 +839,8 @@ public void SchemaFetchScanOperatorTest() { null, null, false, + false, + false, false); assertEquals(DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES, operator.calculateMaxPeekMemory()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java index 4b694d53fa92b..50249bc55d528 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java @@ -60,7 +60,15 @@ public void testSchemaFetchResult() throws Exception { SchemaFetchScanOperator schemaFetchScanOperator = SchemaFetchScanOperator.ofSeries( - null, null, patternTree, Collections.emptyMap(), schemaRegion, false, true); + null, + null, + patternTree, + Collections.emptyMap(), + schemaRegion, + false, + false, + true, + false); Assert.assertTrue(schemaFetchScanOperator.hasNext()); @@ -151,7 +159,9 @@ private ISchemaRegion mockSchemaRegion() throws Exception { patternTree.appendPathPattern(new PartialPath("root.**.status")); patternTree.appendPathPattern(new PartialPath("root.**.s1")); patternTree.constructTree(); - Mockito.when(schemaRegion.fetchSeriesSchema(patternTree, Collections.emptyMap(), false, true)) + Mockito.when( + schemaRegion.fetchSeriesSchema( + patternTree, Collections.emptyMap(), false, false, true, false)) .thenReturn(clusterSchemaTree); return schemaRegion; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java index 62ccf5e9ef551..1e6042e05dfee 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java @@ -54,7 +54,14 @@ public ClusterSchemaTree fetchSchema( } @Override - public ISchemaTree fetchSchemaInDeviceLevel( + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + schemaTree.setDatabases(Collections.singleton("root.sg")); + return schemaTree; + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { schemaTree.setDatabases(Collections.singleton("root.sg")); return schemaTree; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java index 65b614ff48495..e08bbb074ea2a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java @@ -306,7 +306,13 @@ public ISchemaTree fetchSchema( } @Override - public ISchemaTree fetchSchemaInDeviceLevel( + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { return ANALYSIS.getSchemaTree(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java index a84e002df2e41..3d5e66aea49b2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java @@ -200,7 +200,13 @@ public ISchemaTree fetchSchema( } @Override - public ISchemaTree fetchSchemaInDeviceLevel( + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { return ANALYSIS.getSchemaTree(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java index ba24a39d64dc1..4bee7e70c27c6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java @@ -67,17 +67,17 @@ public class RegionScanLogicalPlannerTest { new MeasurementPath("root.sg.d1.s1", TSDataType.INT32), Collections.singletonList( new TimeseriesContext( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s2", TSDataType.DOUBLE), Collections.singletonList( new TimeseriesContext( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s3", TSDataType.BOOLEAN), Collections.singletonList( new TimeseriesContext( - "BOOLEAN", null, "PLAIN", "LZ4", "{\"key1\":\"value2\"}", null, null))); + "BOOLEAN", null, "PLAIN", "LZ4", "{\"key1\":\"value2\"}", null, null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d1")), timeseriesSchemaInfoMap); @@ -86,17 +86,17 @@ public class RegionScanLogicalPlannerTest { new MeasurementPath("root.sg.d2.s1", TSDataType.INT32), Collections.singletonList( new TimeseriesContext( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s2", TSDataType.DOUBLE), Collections.singletonList( new TimeseriesContext( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s4", TSDataType.TEXT), Collections.singletonList( new TimeseriesContext( - "TEXT", null, "PLAIN", "LZ4", "{\"key2\":\"value1\"}", null, null))); + "TEXT", null, "PLAIN", "LZ4", "{\"key2\":\"value1\"}", null, null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d2")), timeseriesSchemaInfoMap2); @@ -106,10 +106,11 @@ public class RegionScanLogicalPlannerTest { List timeseriesContextList = new ArrayList<>(); Map> timeseriesSchemaInfoMap3 = new HashMap<>(); timeseriesContextList.add( - new TimeseriesContext("INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); + new TimeseriesContext( + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null)); timeseriesContextList.add( new TimeseriesContext( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null)); timeseriesSchemaInfoMap3.put( new AlignedPath("root.sg.d2.a", schemas, Collections.emptyList()), timeseriesContextList); deviceToTimeseriesSchemaInfoMap.put( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java index 45a75fdf58fde..e31d886a4f921 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java @@ -49,7 +49,9 @@ public void testSerialization() throws IllegalPathException { patternTree, Collections.emptyMap(), true, - true); + false, + true, + false); schemaFetchMergeNode.addChild(seriesSchemaFetchScanNode); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); schemaFetchMergeNode.serialize(byteBuffer); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java index ffa00cf8bdc90..0ad52409c09da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java @@ -45,7 +45,9 @@ public void testSerialization() throws IllegalPathException { patternTree, Collections.emptyMap(), true, - true); + false, + true, + false); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); seriesSchemaFetchScanNode.serialize(byteBuffer); byteBuffer.flip(); From e950a18b053c4db419992fea20eb8301f748ae2c Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Fri, 14 Jun 2024 14:30:35 +0800 Subject: [PATCH 10/12] delete useless --- .../common/schematree/ClusterSchemaTree.java | 10 +--------- .../db/queryengine/plan/analyze/AnalyzeVisitor.java | 5 +++++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index 66dcb77fd08c4..dc9776ab54ab6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -56,7 +55,6 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; -import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; import static org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode.SCHEMA_ENTITY_NODE; import static org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode.SCHEMA_MEASUREMENT_NODE; @@ -75,8 +73,6 @@ public class ClusterSchemaTree implements ISchemaTree { /** used to judge schema tree type */ private boolean hasNormalTimeSeries = false; - private PathPatternTree authorityScope = ALL_MATCH_SCOPE; - private Map templateMap = new HashMap<>(); public ClusterSchemaTree() { @@ -91,10 +87,6 @@ public void setTemplateMap(Map templateMap) { this.templateMap = templateMap; } - public void setAuthorityScope(PathPatternTree authorityScope) { - this.authorityScope = authorityScope; - } - /** * Return all measurement paths for given path pattern and filter the result by slimit and offset. * @@ -107,7 +99,7 @@ public Pair, Integer> searchMeasurementPaths( PartialPath pathPattern, int slimit, int soffset, boolean isPrefixMatch) { try (SchemaTreeVisitorWithLimitOffsetWrapper visitor = SchemaTreeVisitorFactory.createSchemaTreeMeasurementVisitor( - root, pathPattern, isPrefixMatch, slimit, soffset, authorityScope)) { + root, pathPattern, isPrefixMatch, slimit, soffset)) { visitor.setTemplateMap(templateMap); return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 720b365f4e98d..a51b6fc5ef619 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2874,6 +2874,11 @@ private boolean analyzeTimeseriesRegionScan( removeLogicViewMeasurement(schemaTree); Map>> deviceToTimeseriesContext = new HashMap<>(); + /** + * Since we fetch raw time series schema without template(The template sequence will be treated + * as a normal node, not a device+templateId. This means that all nodes are what we need.). We + * can use ALL_MATCH_PATTERN to get result. + */ List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); Set deviceSet = new HashSet<>(); for (DeviceSchemaInfo deviceSchemaInfo : deviceSchemaInfoList) { From d9bcddcd1a6f28b94e39a08231e6d52e273e5449 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Mon, 17 Jun 2024 17:42:03 +0800 Subject: [PATCH 11/12] Add IT --- .../regionscan/IoTDBActiveRegionScanIT.java | 12 +- .../regionscan/IoTDBActiveRegionScanIT2.java | 9 +- .../IoTDBActiveRegionScanWithModsIT.java | 26 +- .../IoTDBActiveRegionScanWithModsIT2.java | 9 +- .../regionscan/IoTDBActiveSchemaQueryIT.java | 363 ++++++++++++++++++ .../apache/iotdb/util/AbstractSchemaIT.java | 8 + .../metadata/CountDevicesStatement.java | 27 ++ .../metadata/CountTimeSeriesStatement.java | 27 ++ .../metadata/ShowDevicesStatement.java | 27 ++ .../metadata/ShowTimeSeriesStatement.java | 27 ++ .../impl/SchemaRegionPBTreeImpl.java | 9 +- .../impl/pbtree/MTreeBelowSGCachedImpl.java | 41 +- 12 files changed, 544 insertions(+), 41 deletions(-) rename integration-test/src/test/java/org/apache/iotdb/db/it/{ => schema}/regionscan/IoTDBActiveRegionScanIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/db/it/{ => schema}/regionscan/IoTDBActiveRegionScanIT2.java (91%) rename integration-test/src/test/java/org/apache/iotdb/db/it/{ => schema}/regionscan/IoTDBActiveRegionScanWithModsIT.java (85%) rename integration-test/src/test/java/org/apache/iotdb/db/it/{ => schema}/regionscan/IoTDBActiveRegionScanWithModsIT2.java (91%) create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java index c608bf3e24c0e..aa568b583c1bd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java @@ -17,18 +17,17 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.ResultSet; @@ -45,9 +44,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBActiveRegionScanIT { +public class IoTDBActiveRegionScanIT extends AbstractSchemaIT { // Data can be viewed in // https://docs.google.com/spreadsheets/d/11tNRIaHmNdFWc0RC4yAloO6JJbmo9S5qxwAqWxeVY6c/edit#gid=0 public static final String[] common_insert_sqls = @@ -145,6 +143,10 @@ public class IoTDBActiveRegionScanIT { public static final String COUNT_TIMESERIES_COLUMN_NAMES = "count(timeseries)"; public static final String COUNT_DEVICES_COLUMN_NAMES = "count(devices)"; + public IoTDBActiveRegionScanIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + public static void insertData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java index be4751495de4b..da56d3078b01e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java @@ -17,21 +17,22 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBActiveRegionScanIT2 extends IoTDBActiveRegionScanIT { + public IoTDBActiveRegionScanIT2(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv() diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java similarity index 85% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java index d977a6d0132e6..0c10f507aa214 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java @@ -17,34 +17,32 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.Statement; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.COUNT_DEVICES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.COUNT_TIMESERIES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.SHOW_DEVICES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.SHOW_TIMESERIES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.basicCountActiveDeviceTest; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.basicShowActiveDeviceTest; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.common_insert_sqls; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.COUNT_DEVICES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.COUNT_TIMESERIES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.SHOW_DEVICES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.SHOW_TIMESERIES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.basicCountActiveDeviceTest; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.basicShowActiveDeviceTest; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.common_insert_sqls; import static org.junit.Assert.fail; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBActiveRegionScanWithModsIT { +public class IoTDBActiveRegionScanWithModsIT extends AbstractSchemaIT { private static String[] delete_sqls = new String[] { @@ -57,6 +55,10 @@ public class IoTDBActiveRegionScanWithModsIT { "delete from root.sg.unaligned.d2.* where time > 40 and time <= 43" }; + public IoTDBActiveRegionScanWithModsIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + public static void insertData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java index a87240341e019..5adccd1fd6980 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java @@ -17,21 +17,22 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBActiveRegionScanWithModsIT2 extends IoTDBActiveRegionScanWithModsIT { + public IoTDBActiveRegionScanWithModsIT2(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv() diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java new file mode 100644 index 0000000000000..853ef3c87ae00 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.schema.regionscan; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBActiveSchemaQueryIT extends AbstractSchemaIT { + + private static String[] sqls = + new String[] { + "create aligned timeseries root.sg.d1(s1(alias1) int32 tags('tag1'='v1', 'tag2'='v2'), s2 double attributes('attr3'='v3'))", + "CREATE TIMESERIES root.sg.d0.s0 WITH DATATYPE=INT32", + "CREATE TIMESERIES root.sg.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT,COMPDEV=2", + "CREATE TIMESERIES root.sg.d0.s2 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15", + "CREATE DEVICE TEMPLATE t1 (s1 INT64 encoding=RLE compressor=SNAPPY, s2 INT32)", + "CREATE DEVICE TEMPLATE t2 aligned(s1 INT64 encoding=RLE compressor=SNAPPY, s2 INT32)", + "SET DEVICE TEMPLATE t1 to root.sg.d2", + "SET DEVICE TEMPLATE t2 to root.sg.d3", + "CREATE TIMESERIES OF DEVICE TEMPLATE ON root.sg.d2", + "CREATE TIMESERIES OF DEVICE TEMPLATE ON root.sg.d3", + "set ttl to root.sg.d3 60000", + }; + + public IoTDBActiveSchemaQueryIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + + @Parameterized.BeforeParam + public static void before() throws Exception { + SchemaTestMode schemaTestMode = setUpEnvironment(); + if (schemaTestMode.equals(SchemaTestMode.PBTree)) { + allocateMemoryForSchemaRegion(10000); + } + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Parameterized.AfterParam + public static void after() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + tearDownEnvironment(); + } + + @Before + public void setUp() throws Exception { + createSchema(); + } + + @After + public void tearDown() throws Exception { + clearSchema(); + } + + private void createSchema() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testShowTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // 0. Check environment + Set expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.**", expected); + // 1. Without any data, result set should be empty + try (ResultSet resultSet = + statement.executeQuery("show timeseries root.sg.** where time>0")) { + Assert.assertFalse(resultSet.next()); + } + // 2. Insert data and check again. + // - root.db.d0.* time=1 + // - root.db.d1.* time=2 + // - root.db.d2.* time=3 + // - root.db.d3.* time=now() + statement.execute("insert into root.sg.d0(timestamp,s0, s1, s2) values(1,1,1,1)"); + statement.execute("insert into root.sg.d1(timestamp,s1,s2) values(2,2,2)"); + statement.execute("insert into root.sg.d2(timestamp,s1,s2) values(3,3,3)"); + statement.execute("insert into root.sg.d3(timestamp,s1,s2) values(now(),4,4)"); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time>0", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("9,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time>1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time>1", + new HashSet<>(Collections.singletonList("6,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time!=1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time!=1", + new HashSet<>(Collections.singletonList("6,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time<2", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time<2", + new HashSet<>(Collections.singletonList("3,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time=1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time=1", + new HashSet<>(Collections.singletonList("3,"))); + // 3. Check non-root user + statement.execute("CREATE USER user1 'password'"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.1 Without read data permission, result set should be empty + try (ResultSet resultSet = + userStmt.executeQuery("show timeseries root.sg.** where time>0")) { + Assert.assertFalse(resultSet.next()); + } + checkResultSet( + userStmt, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("0,"))); + } + statement.execute("GRANT READ_DATA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_DATA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.2 With read data permission, result set should not be empty + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,")); + checkResultSet(userStmt, "show timeseries root.sg.** where time>0", expected); + checkResultSet( + userStmt, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("2,"))); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testShowDevices() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // 0. Check environment + Set expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0,false,null,INF,", + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.*", expected); + // 1. Without any data, result set should be empty + try (ResultSet resultSet = statement.executeQuery("show devices root.sg.* where time>0")) { + Assert.assertFalse(resultSet.next()); + } + // 2. Insert data and check again. + // - root.sg.d0.* time=1 + // - root.sg.d1.* time=2 + // - root.sg.d2.* time=3 + // - root.sg.d3.* time=now() + statement.execute("insert into root.sg.d0(timestamp,s0, s1, s2) values(1,1,1,1)"); + statement.execute("insert into root.sg.d1(timestamp,s1,s2) values(2,2,2)"); + statement.execute("insert into root.sg.d2(timestamp,s1,s2) values(3,3,3)"); + statement.execute("insert into root.sg.d3(timestamp,s1,s2) values(now(),4,4)"); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0,false,null,INF,", + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time>0", expected); + checkResultSet( + statement, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("4,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time>1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time>1", + new HashSet<>(Collections.singletonList("3,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time!=1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time!=1", + new HashSet<>(Collections.singletonList("3,"))); + expected = new HashSet<>(Collections.singletonList("root.sg.d0,false,null,INF,")); + checkResultSet(statement, "show devices root.sg.* where time<2", expected); + checkResultSet( + statement, + "count devices root.sg.* where time<2", + new HashSet<>(Collections.singletonList("1,"))); + expected = new HashSet<>(Collections.singletonList("root.sg.d0,false,null,INF,")); + checkResultSet(statement, "show devices root.sg.* where time=1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time=1", + new HashSet<>(Collections.singletonList("1,"))); + // 3. Check non-root user + statement.execute("CREATE USER user1 'password'"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.1 Without read data permission, result set should be empty + try (ResultSet resultSet = userStmt.executeQuery("show devices root.sg.* where time>0")) { + Assert.assertFalse(resultSet.next()); + } + checkResultSet( + userStmt, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("0,"))); + } + statement.execute("GRANT READ_DATA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_DATA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.2 With read data permission, result set should not be empty + expected = + new HashSet<>(Arrays.asList("root.sg.d0,false,null,INF,", "root.sg.d3,true,t2,60000,")); + checkResultSet(userStmt, "show devices root.sg.* where time>0", expected); + checkResultSet( + userStmt, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("2,"))); + } + + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + private void checkResultSet(Statement statement, String sql, Set expected) + throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + if (!expected.contains(builder.toString())) { + System.out.println(builder.toString()); + } + Assert.assertTrue(expected.contains(builder.toString())); + expected.remove(builder.toString()); + } + } + Assert.assertTrue(expected.isEmpty()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java index deefd0cccc90e..1c3f3708ae374 100644 --- a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java @@ -100,6 +100,14 @@ protected static void clearSchema() throws Exception { "DROP DEVICE TEMPLATE " + resultSet.getString(ColumnHeaderConstant.TEMPLATE_NAME)); } } + // drop all users + try (ResultSet resultSet = statement.executeQuery("LIST USER")) { + while (resultSet.next()) { + if (!resultSet.getString(ColumnHeaderConstant.USER).equals("root")) { + statement.execute("DROP USER " + resultSet.getString(ColumnHeaderConstant.USER)); + } + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java index 06d32f7c9b53d..9b73e1f9b0a5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java @@ -19,9 +19,15 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; public class CountDevicesStatement extends CountStatement { @@ -43,6 +49,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitCountDevices(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java index c7c259030a9cf..c0910185421bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; public class CountTimeSeriesStatement extends CountStatement { @@ -54,6 +60,27 @@ public WhereCondition getTimeCondition() { return timeCondition; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitCountTimeSeries(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java index 75f33c7b4a1cb..b0b7f8bba3178 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; import java.util.Collections; import java.util.List; @@ -78,6 +84,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitShowDevices(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java index 82a389d622499..7d615cac33648 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; import java.util.Collections; import java.util.List; @@ -77,6 +83,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public List getPaths() { return Collections.singletonList(pathPattern); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index e48ba2e01a8e1..4dbfaf5c3bf67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -211,6 +211,7 @@ public synchronized void init() throws MetadataException { new MTreeBelowSGCachedImpl( new PartialPath(storageGroupFullPath), tagManager::readTags, + tagManager::readAttributes, this::flushCallback, measurementInitProcess(), deviceInitProcess(), @@ -549,6 +550,7 @@ public void loadSnapshot(File latestSnapshotRootDir) { measurementInitProcess(), deviceInitProcess(), tagManager::readTags, + tagManager::readAttributes, this::flushCallback); logger.info( "MTree snapshot loading of schemaRegion {} costs {}ms.", @@ -1038,11 +1040,14 @@ public ClusterSchemaTree fetchSeriesSchema( if (patternTree.isContainWildcard()) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); for (PartialPath path : patternTree.getAllPathPatterns()) { - schemaTree.mergeSchemaTree(mtree.fetchSchema(path, templateMap, withTags, withTemplate)); + schemaTree.mergeSchemaTree( + mtree.fetchSchema( + path, templateMap, withTags, withAttributes, withTemplate, withAliasForcete)); } return schemaTree; } else { - return mtree.fetchSchemaWithoutWildcard(patternTree, templateMap, withTags, withTemplate); + return mtree.fetchSchemaWithoutWildcard( + patternTree, templateMap, withTags, withAttributes, withTemplate); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index ceeebf1247aeb..12af832364c62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -121,6 +121,7 @@ public class MTreeBelowSGCachedImpl { private final ICachedMNode rootNode; private final Function, Map> tagGetter; + private final Function, Map> attributeGetter; private final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getCachedMNodeIMNodeFactory(); private final int levelOfSG; @@ -130,6 +131,7 @@ public class MTreeBelowSGCachedImpl { public MTreeBelowSGCachedImpl( PartialPath storageGroupPath, Function, Map> tagGetter, + Function, Map> attributeGetter, Runnable flushCallback, Consumer> measurementProcess, Consumer> deviceProcess, @@ -138,6 +140,7 @@ public MTreeBelowSGCachedImpl( SchemaRegionCachedMetric metric) throws MetadataException, IOException { this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; this.regionStatistics = regionStatistics; store = PBTreeFactory.getInstance() @@ -177,6 +180,7 @@ private MTreeBelowSGCachedImpl( Consumer> measurementProcess, Consumer> deviceProcess, Function, Map> tagGetter, + Function, Map> attributeGetter, CachedSchemaRegionStatistics regionStatistics) throws MetadataException { this.store = store; @@ -185,6 +189,7 @@ private MTreeBelowSGCachedImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupMNode.getPartialPath().getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; // recover MNode try (MNodeCollector collector = @@ -226,6 +231,7 @@ public static MTreeBelowSGCachedImpl loadFromSnapshot( Consumer> measurementProcess, Consumer> deviceProcess, Function, Map> tagGetter, + Function, Map> attributeGetter, Runnable flushCallback) throws IOException, MetadataException { return new MTreeBelowSGCachedImpl( @@ -241,6 +247,7 @@ public static MTreeBelowSGCachedImpl loadFromSnapshot( measurementProcess, deviceProcess, tagGetter, + attributeGetter, regionStatistics); } @@ -818,7 +825,9 @@ public ClusterSchemaTree fetchSchema( PartialPath pathPattern, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); try (MeasurementCollector collector = @@ -834,14 +843,15 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - if (nodes[nodes.length - 1].equals(node.getAlias())) { - // only when user query with alias, the alias in path will be set - path.setMeasurementAlias(node.getAlias()); - } - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + withAliasForce || nodes[nodes.length - 1].equals(node.getAlias()) + ? node.getAlias() + : null, + deviceMNode.isAligned()); } return null; } @@ -857,6 +867,7 @@ public ClusterSchemaTree fetchSchemaWithoutWildcard( PathPatternTree patternTree, Map templateMap, boolean withTags, + boolean withAttributes, boolean withTemplate) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); @@ -873,11 +884,13 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - path.setMeasurementAlias(node.getAlias()); - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + node.getAlias(), + deviceMNode.isAligned()); } return null; } From ad64f698d8cff1b6b71a676ed58183b8b5f2114c Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Tue, 18 Jun 2024 15:53:32 +0800 Subject: [PATCH 12/12] remove --- iotdb-core/datanode/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 8e59e2d863b79..cabc80a5aa195 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -8,7 +8,7 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at -gi + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing,