From d0e455e51c590830a49ac8601837dbcc27c36d6a Mon Sep 17 00:00:00 2001 From: choubenson <2027178562@qq.com> Date: Tue, 11 Jun 2024 18:39:54 +0800 Subject: [PATCH 1/3] support show ttl on pathPattern --- .mvn/develocity.xml | 4 +- .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 ++- .../request/read/ttl/ShowTTLPlan.java | 13 ++++ .../confignode/manager/ConfigManager.java | 4 +- .../iotdb/confignode/manager/IManager.java | 2 +- .../iotdb/confignode/manager/TTLManager.java | 4 +- .../iotdb/confignode/persistence/TTLInfo.java | 21 +++++-- .../executor/ConfigPlanExecutor.java | 3 +- .../schema/CNPhysicalPlanGenerator.java | 11 ++-- .../thrift/ConfigNodeRPCServiceProcessor.java | 7 ++- .../confignode/persistence/TTLInfoTest.java | 39 ++++++------ .../db/protocol/client/ConfigNodeClient.java | 5 +- .../cache/schema/DataNodeTTLCache.java | 11 ---- .../executor/ClusterConfigTaskExecutor.java | 8 ++- .../queryengine/plan/parser/ASTVisitor.java | 12 +++- .../statement/metadata/ShowTTLStatement.java | 11 +--- .../execute/utils/CompactionPathUtils.java | 6 +- .../iotdb/commons/schema/ttl/TTLCache.java | 60 +++++++++---------- .../src/main/thrift/common.thrift | 5 ++ .../src/main/thrift/confignode.thrift | 4 +- 20 files changed, 132 insertions(+), 105 deletions(-) diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml index 1e82305ef93c5..b505d1a366647 100644 --- a/.mvn/develocity.xml +++ b/.mvn/develocity.xml @@ -26,9 +26,7 @@ #{isFalse(env['GITHUB_ACTIONS'])} - - - + #{{'0.0.0.0'}} diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index c2086e8de8919..8a3be205d08f2 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -47,7 +47,7 @@ ddlStatement | setSchemaTemplate | unsetSchemaTemplate | alterSchemaTemplate // TTL - | setTTL | unsetTTL | showAllTTL + | setTTL | unsetTTL | showTTL | showAllTTL // Function | createFunction | dropFunction | showFunctions // Trigger @@ -325,6 +325,11 @@ unsetTTL : UNSET TTL TO path=prefixPath ; +// ---- Show TTL +showTTL + : SHOW TTL ON prefixPath (COMMA prefixPath)* + ; + // ---- Show All TTL showAllTTL : SHOW ALL TTL diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java index c37c93a907659..9dd9eded31eb8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ttl/ShowTTLPlan.java @@ -20,14 +20,27 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.db.utils.constant.SqlConstant; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; public class ShowTTLPlan extends ConfigPhysicalPlan { + private String[] pathPattern; + + public String[] getPathPattern() { + return pathPattern; + } + public ShowTTLPlan() { super(ConfigPhysicalPlanType.ShowTTL); + this.pathPattern = SqlConstant.getSingleRootArray(); + } + + public ShowTTLPlan(String[] pathPattern) { + super(ConfigPhysicalPlanType.ShowTTL); + this.pathPattern = pathPattern; } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 8238598e651b7..daf07ae6107be 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -620,10 +620,10 @@ public DataSet getMatchedDatabaseSchemas(GetDatabasePlan getDatabaseReq) { } @Override - public DataSet showAllTTL(ShowTTLPlan showTTLPlan) { + public DataSet showTTL(ShowTTLPlan showTTLPlan) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return ttlManager.showAllTTL(showTTLPlan); + return ttlManager.showTTL(showTTLPlan); } else { ShowTTLResp resp = new ShowTTLResp(); resp.setStatus(status); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 3ac0ee6ade41f..1daaf13cbacaa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -302,7 +302,7 @@ public interface IManager { TSStatus setTTL(SetTTLPlan configRequest); - DataSet showAllTTL(ShowTTLPlan showTTLPlan); + DataSet showTTL(ShowTTLPlan showTTLPlan); TSStatus setSchemaReplicationFactor(SetSchemaReplicationFactorPlan configPhysicalPlan); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java index af4fb49a0cb09..6bebe7c686474 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java @@ -119,7 +119,7 @@ public TSStatus unsetTTL(SetTTLPlan setTTLPlan, final boolean isGeneratedByPipe) return configManager.getProcedureManager().setTTL(setTTLPlan, isGeneratedByPipe); } - public DataSet showAllTTL(ShowTTLPlan showTTLPlan) { + public DataSet showTTL(ShowTTLPlan showTTLPlan) { try { return configManager.getConsensusManager().read(showTTLPlan); } catch (ConsensusException e) { @@ -133,7 +133,7 @@ public DataSet showAllTTL(ShowTTLPlan showTTLPlan) { } public Map getAllTTL() { - return ((ShowTTLResp) showAllTTL(new ShowTTLPlan())).getPathTTLMap(); + return ((ShowTTLResp) showTTL(new ShowTTLPlan())).getPathTTLMap(); } public int getTTLCount() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java index 539a6ac0ffe3b..c9b85953f15fb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.response.ttl.ShowTTLResp; import org.apache.iotdb.rpc.RpcUtils; @@ -43,6 +44,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -118,12 +120,19 @@ public TSStatus unsetTTL(SetTTLPlan plan) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - public ShowTTLResp showAllTTL() { + public ShowTTLResp showTTL(ShowTTLPlan plan) { ShowTTLResp resp = new ShowTTLResp(); - Map pathTTLMap; + Map pathTTLMap = new HashMap<>(); lock.readLock().lock(); try { - pathTTLMap = ttlCache.getAllPathTTL(); + PartialPath pathPattern = new PartialPath(plan.getPathPattern()); + for (Map.Entry entry : ttlCache.getAllTTLs().entrySet()) { + if (pathPattern.matchFullPath(new PartialPath(entry.getKey()))) { + pathTTLMap.put( + String.join(String.valueOf(IoTDBConstant.PATH_SEPARATOR), entry.getKey()), + entry.getValue()); + } + } } finally { lock.readLock().unlock(); } @@ -175,6 +184,8 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { ttlCache.clear(); ttlCache.deserialize(bufferedInputStream); + } catch (IllegalPathException e) { + throw new IOException(e); } finally { lock.writeLock().unlock(); } @@ -190,7 +201,9 @@ public boolean equals(Object o) { } TTLInfo other = (TTLInfo) o; return this.getTTLCount() == other.getTTLCount() - && this.showAllTTL().getPathTTLMap().equals(other.showAllTTL().getPathTTLMap()); + && this.showTTL(new ShowTTLPlan()) + .getPathTTLMap() + .equals(other.showTTL(new ShowTTLPlan()).getPathTTLMap()); } @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 5e5db43a5af4c..dc1ffbde72034 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -48,6 +48,7 @@ import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerJarPlan; import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerLocationPlan; import org.apache.iotdb.confignode.consensus.request.read.trigger.GetTriggerTablePlan; +import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; @@ -310,7 +311,7 @@ public DataSet executeQueryPlan(ConfigPhysicalPlan req) case ShowPipeV2: return pipeInfo.getPipeTaskInfo().showPipes(); case ShowTTL: - return ttlInfo.showAllTTL(); + return ttlInfo.showTTL((ShowTTLPlan) req); case ShowTopic: return subscriptionInfo.showTopics(); case ShowSubscription: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java index 1658de401ce64..f8895158afce6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.utils.AuthUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; @@ -38,7 +39,6 @@ import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.commons.io.IOUtils; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; @@ -60,7 +60,6 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; import java.util.Stack; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; @@ -277,13 +276,11 @@ private void generateSetTTLPlan() { while (size > 0) { String path = ReadWriteIOUtils.readString(ttlInputStream); long ttl = ReadWriteIOUtils.readLong(ttlInputStream); - planDeque.add( - new SetTTLPlan( - Objects.requireNonNull(path).split(TsFileConstant.PATH_SEPARATER_NO_REGEX), ttl)); + planDeque.add(new SetTTLPlan(PathUtils.splitPathToDetachedNodes(path), ttl)); size--; } - } catch (IOException e) { - logger.error("Got IOException when deserializing ttl file", e); + } catch (IOException | IllegalPathException e) { + logger.error("Got exception when deserializing ttl file", e); latestException = e; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 82cb2672752c2..35ad847a979dc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowTTLReq; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; @@ -490,8 +491,10 @@ public TDatabaseSchemaResp getMatchedDatabaseSchemas(TGetDatabaseReq req) { } @Override - public TShowTTLResp showAllTTL() { - ShowTTLResp showTTLResp = (ShowTTLResp) configManager.showAllTTL(new ShowTTLPlan()); + public TShowTTLResp showTTL(TShowTTLReq req) { + ShowTTLResp showTTLResp = + (ShowTTLResp) + configManager.showTTL(new ShowTTLPlan(req.getPathPattern().toArray(new String[0]))); return showTTLResp.convertToRPCTShowTTLResp(); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java index 80dc2ed4d31ce..2468914ea2b58 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.response.ttl.ShowTTLResp; @@ -68,7 +69,7 @@ public void tearDown() throws IOException { @Test public void testSetAndUnsetTTL() throws IllegalPathException { - ShowTTLResp resp = ttlInfo.showAllTTL(); + ShowTTLResp resp = ttlInfo.showTTL(new ShowTTLPlan()); Map ttlMap = resp.getPathTTLMap(); Map resultMap = new HashMap<>(); @@ -80,125 +81,125 @@ public void testSetAndUnsetTTL() throws IllegalPathException { PartialPath path = new PartialPath("root.test.db1.**"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 121322323L)); resultMap.put(path.getFullPath(), 121322323L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(2, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test.db1.group1.group1.d1"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 2222L)); resultMap.put(path.getFullPath(), 2222L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(3, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test.db1.group1.group2.d1"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 1)); resultMap.put(path.getFullPath(), 1L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(4, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test1.db1.group1.group2.d1"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 599722L)); resultMap.put(path.getFullPath(), 599722L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(5, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test1.db1.group1.group2.d1.**"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 9999999L)); resultMap.put(path.getFullPath(), 9999999L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(6, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test1.db1.group1.group2.d1.d2.**"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 888888L)); resultMap.put(path.getFullPath(), 888888L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(7, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test1.db1.group1.group2.d1.d2"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 9898989898L)); resultMap.put(path.getFullPath(), 9898989898L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(8, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test.db1.group1.group2.d1"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 11111222L)); resultMap.put(path.getFullPath(), 11111222L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(8, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.test.db1.group1"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), Long.MAX_VALUE)); resultMap.put(path.getFullPath(), Long.MAX_VALUE); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(9, ttlInfo.getTTLCount()); // set ttl path = new PartialPath("root.**"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 222222L)); resultMap.put(path.getFullPath(), 222222L); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(9, ttlInfo.getTTLCount()); // set ttl, not support negative ttl path = new PartialPath("root.test.db1.group1.group2"); ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(9, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test.db1.group1.group2"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(9, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test.db1.group1"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(8, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.**"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.put(path.getFullPath(), ttl); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(8, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test1.db1.group1.group2.d1.d2"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(7, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test.db1.**"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(6, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test1.db1.group1.group2.d1"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(5, ttlInfo.getTTLCount()); // unset ttl path = new PartialPath("root.test1.db1.group1.group2.d1.d2.**"); ttlInfo.unsetTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), -1)); resultMap.remove(path.getFullPath()); - Assert.assertEquals(resultMap, ttlInfo.showAllTTL().getPathTTLMap()); + Assert.assertEquals(resultMap, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap()); Assert.assertEquals(4, ttlInfo.getTTLCount()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index c49b7e0b4451e..806a5b5cbf4bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -27,6 +27,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowTTLReq; import org.apache.iotdb.commons.client.ClientManager; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.factory.ThriftClientFactory; @@ -479,9 +480,9 @@ public TDatabaseSchemaResp getMatchedDatabaseSchemas(TGetDatabaseReq req) throws } @Override - public TShowTTLResp showAllTTL() throws TException { + public TShowTTLResp showTTL(TShowTTLReq req) throws TException { return executeRemoteCallWithRetry( - () -> client.showAllTTL(), resp -> !updateConfigNodeLeader(resp.status)); + () -> client.showTTL(req), resp -> !updateConfigNodeLeader(resp.status)); } public TSStatus callSpecialProcedure(TTestOperation operation) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 07b835896d9da..bc51cae2c8640 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; -import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; public class DataNodeTTLCache { @@ -89,16 +88,6 @@ public long getTTL(String[] path) { } } - /** Get all ttl map under path node. */ - public Map getTTLUnderOneNode(String path) throws IllegalPathException { - lock.readLock().lock(); - try { - return ttlCache.getAllTTLUnderOneNode(PathUtils.splitPathToDetachedNodes(path)); - } finally { - lock.readLock().unlock(); - } - } - /** Get ttl of one specific path node. If this node does not set ttl, then return -1. */ public long getNodeTTL(String path) throws IllegalPathException { lock.readLock().lock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index cb4d6f57fb6c1..37bf4148363bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowTTLReq; import org.apache.iotdb.common.rpc.thrift.TSpaceQuota; import org.apache.iotdb.common.rpc.thrift.TThrottleQuota; import org.apache.iotdb.commons.client.IClientManager; @@ -1251,8 +1252,11 @@ public SettableFuture showTTL(ShowTTLStatement showTTLStatemen Map databaseToTTL = new TreeMap<>(); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TShowTTLResp resp = client.showAllTTL(); - databaseToTTL.putAll(resp.getPathTTLMap()); + for (PartialPath pathPattern : showTTLStatement.getPaths()) { + TShowTTLReq req = new TShowTTLReq(Arrays.asList(pathPattern.getNodes())); + TShowTTLResp resp = client.showTTL(req); + databaseToTTL.putAll(resp.getPathTTLMap()); + } } catch (ClientManagerException | TException e) { future.setException(e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 12ddcf2636c74..f9de1c987c6b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -2564,10 +2564,20 @@ public Statement visitUnsetTTL(IoTDBSqlParser.UnsetTTLContext ctx) { return unSetTTLStatement; } + @Override + public Statement visitShowTTL(IoTDBSqlParser.ShowTTLContext ctx) { + ShowTTLStatement showTTLStatement = new ShowTTLStatement(); + for (IoTDBSqlParser.PrefixPathContext prefixPathContext : ctx.prefixPath()) { + PartialPath partialPath = parsePrefixPath(prefixPathContext); + showTTLStatement.addPathPatterns(partialPath); + } + return showTTLStatement; + } + @Override public Statement visitShowAllTTL(IoTDBSqlParser.ShowAllTTLContext ctx) { ShowTTLStatement showTTLStatement = new ShowTTLStatement(); - showTTLStatement.setAll(true); + showTTLStatement.addPathPatterns(new PartialPath(SqlConstant.getSingleRootArray())); return showTTLStatement; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTTLStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTTLStatement.java index 399e861923281..022e6b2173819 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTTLStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTTLStatement.java @@ -28,12 +28,7 @@ import java.util.List; public class ShowTTLStatement extends ShowStatement implements IConfigStatement { - private List pathPatterns = new ArrayList<>(); - private boolean isAll = false; - - public boolean isAll() { - return isAll; - } + private final List pathPatterns = new ArrayList<>(); @Override public R accept(StatementVisitor visitor, C context) { @@ -53,8 +48,4 @@ public List getPaths() { public void addPathPatterns(PartialPath pathPattern) { pathPatterns.add(pathPattern); } - - public void setAll(boolean all) { - isAll = all; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java index 20de56d10fd44..59c7b1a44c5be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java @@ -27,8 +27,6 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.PlainDeviceID; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATER_NO_REGEX; - public class CompactionPathUtils { private CompactionPathUtils() {} @@ -40,7 +38,7 @@ public static PartialPath getPath(IDeviceID device, String measurement) if (plainDeviceId.contains(TsFileConstant.BACK_QUOTE_STRING)) { path = DataNodeDevicePathCache.getInstance().getPartialPath(plainDeviceId); } else { - path = new PartialPath(((PlainDeviceID) device).toStringID().split(PATH_SEPARATER_NO_REGEX)); + path = new PartialPath(plainDeviceId); } return path.concatNode(measurement); } @@ -51,7 +49,7 @@ public static PartialPath getPath(IDeviceID device) throws IllegalPathException if (plainDeviceId.contains(TsFileConstant.BACK_QUOTE_STRING)) { path = DataNodeDevicePathCache.getInstance().getPartialPath(plainDeviceId); } else { - path = new PartialPath(((PlainDeviceID) device).toStringID().split(PATH_SEPARATER_NO_REGEX)); + path = new PartialPath(plainDeviceId); } return path; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java index 081f009b5bbac..8e1f99e69a7f4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java @@ -20,9 +20,10 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.utils.ReadWriteIOUtils; import javax.annotation.concurrent.NotThreadSafe; @@ -30,9 +31,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Objects; /** TTL Cache Tree, which is a prefix B+ tree with each node storing TTL. */ @NotThreadSafe @@ -152,24 +154,6 @@ public long getClosestTTL(String[] nodes) { return ttl; } - public Map getAllTTLUnderOneNode(String[] nodes) { - Map pathTTLMap = new HashMap<>(); - CacheNode current = ttlCacheTree; - for (int i = 1; i < nodes.length; i++) { - current = current.getChild(nodes[i]); - if (current == null) { - return pathTTLMap; - } - } - - // get all ttl under current node - dfsCacheTree( - pathTTLMap, - new StringBuilder(String.join(String.valueOf(IoTDBConstant.PATH_SEPARATOR), nodes)), - current); - return pathTTLMap; - } - /** * Return the ttl of path. If the path does not exist, it means that the TTL is not set, and * return NULL_TTL. @@ -185,23 +169,36 @@ public long getLastNodeTTL(String[] nodes) { return node.ttl; } + /** + * @return key is path contains wildcard between each node + */ public Map getAllPathTTL() { Map result = new HashMap<>(); - dfsCacheTree(result, new StringBuilder(IoTDBConstant.PATH_ROOT), ttlCacheTree); + for (Map.Entry entry : getAllTTLs().entrySet()) { + result.put( + String.join(String.valueOf(IoTDBConstant.PATH_SEPARATOR), entry.getKey()), + entry.getValue()); + } return result; } - private void dfsCacheTree(Map pathTTLMap, StringBuilder path, CacheNode node) { + public Map getAllTTLs() { + Map result = new HashMap<>(); + List pathNodes = new ArrayList<>(); + pathNodes.add(IoTDBConstant.PATH_ROOT); + dfsCacheTree(result, pathNodes, ttlCacheTree); + return result; + } + + private void dfsCacheTree( + Map pathTTLMap, List pathNodes, CacheNode node) { if (node.ttl != NULL_TTL) { - pathTTLMap.put(path.toString(), node.ttl); + pathTTLMap.put(pathNodes.toArray(new String[0]), node.ttl); } - int idx = path.length(); for (Map.Entry entry : node.getChildren().entrySet()) { - dfsCacheTree( - pathTTLMap, - path.append(IoTDBConstant.PATH_SEPARATOR).append(entry.getValue().name), - entry.getValue()); - path.delete(idx, path.length()); + pathNodes.add(entry.getKey()); + dfsCacheTree(pathTTLMap, pathNodes, entry.getValue()); + pathNodes.remove(pathNodes.size() - 1); } } @@ -219,12 +216,13 @@ public void serialize(OutputStream outputStream) throws IOException { outputStream.flush(); } - public void deserialize(InputStream bufferedInputStream) throws IOException { + public void deserialize(InputStream bufferedInputStream) + throws IOException, IllegalPathException { int size = ReadWriteIOUtils.readInt(bufferedInputStream); while (size > 0) { String path = ReadWriteIOUtils.readString(bufferedInputStream); long ttl = ReadWriteIOUtils.readLong(bufferedInputStream); - setTTL(Objects.requireNonNull(path).split(TsFileConstant.PATH_SEPARATER_NO_REGEX), ttl); + setTTL(PathUtils.splitPathToDetachedNodes(path), ttl); size--; } } diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index d81730337c627..66be1e44cfbfd 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -121,12 +121,17 @@ struct TSchemaNode { 2: required byte nodeType } +// for TTL struct TSetTTLReq { 1: required list pathPattern 2: required i64 TTL 3: required bool isDataBase } +struct TShowTTLReq { + 1: required list pathPattern +} + // for File struct TFile { 1: required string fileName diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 25ed9d3c64cf2..c34ea70885459 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1308,8 +1308,8 @@ service IConfigNodeRPCService { // ====================================================== // TTL // ====================================================== - /** Show all ttl */ - TShowTTLResp showAllTTL() + /** Show ttl */ + TShowTTLResp showTTL(common.TShowTTLReq req) /** Update the specific device's TTL */ common.TSStatus setTTL(common.TSetTTLReq req) From 3e22765db28f439ca0ed979ebd47522190f130c6 Mon Sep 17 00:00:00 2001 From: choubenson <2027178562@qq.com> Date: Tue, 11 Jun 2024 20:34:02 +0800 Subject: [PATCH 2/3] fix UT --- .../org/apache/iotdb/db/storageengine/dataregion/TTLTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java index 51334eb92b202..3710a490688aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java @@ -398,7 +398,8 @@ public void testParseShowTTL() { ShowTTLStatement statement1 = (ShowTTLStatement) StatementGenerator.createStatement("SHOW ALL TTL", ZoneId.systemDefault()); - assertTrue(statement1.getPaths().isEmpty()); + assertEquals(1, statement1.getPaths().size()); + assertEquals("root.**", statement1.getPaths().get(0).getFullPath()); } @Test From 01e58123827a28b7cdde811eed123d5c473228c5 Mon Sep 17 00:00:00 2001 From: choubenson <2027178562@qq.com> Date: Mon, 17 Jun 2024 22:05:29 +0800 Subject: [PATCH 3/3] modify details --- .../iotdb/confignode/persistence/TTLInfo.java | 2 +- .../config/executor/ClusterConfigTaskExecutor.java | 1 + .../org/apache/iotdb/commons/path/PartialPath.java | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java index c9b85953f15fb..4cb75bcec4f41 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java @@ -127,7 +127,7 @@ public ShowTTLResp showTTL(ShowTTLPlan plan) { try { PartialPath pathPattern = new PartialPath(plan.getPathPattern()); for (Map.Entry entry : ttlCache.getAllTTLs().entrySet()) { - if (pathPattern.matchFullPath(new PartialPath(entry.getKey()))) { + if (pathPattern.matchFullPath(entry.getKey())) { pathTTLMap.put( String.join(String.valueOf(IoTDBConstant.PATH_SEPARATOR), entry.getKey()), entry.getValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 02cd361c04a55..947742c8625f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -1315,6 +1315,7 @@ public SettableFuture showTTL(ShowTTLStatement showTTLStatemen Map databaseToTTL = new TreeMap<>(); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // TODO: send all paths in one RPC for (PartialPath pathPattern : showTTLStatement.getPaths()) { TShowTTLReq req = new TShowTTLReq(Arrays.asList(pathPattern.getNodes())); TShowTTLResp resp = client.showTTL(req); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 3327f4c9bbea0..59a64352af1bd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -323,6 +323,19 @@ public boolean matchFullPath(PartialPath rPath) { return matchPath(rPath.getNodes(), 0, 0, false, false); } + /** + * Test if current PartialPath matches a full path. Current partialPath acts as a full path + * pattern. rPath is supposed to be a full timeseries path without wildcards. e.g. + * "root.sg.device.*" matches path "root.sg.device.s1" whereas it does not match "root.sg.device" + * and "root.sg.vehicle.s1" + * + * @param rPath a plain full path of a timeseries + * @return true if a successful match, otherwise return false + */ + public boolean matchFullPath(String[] rPath) { + return matchPath(rPath, 0, 0, false, false); + } + /** * Check if current pattern PartialPath can match 1 prefix path. *