diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index 028af358eac42..9a236d5725a10 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -57,6 +57,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin diff --git a/distribution/src/assembly/confignode.xml b/distribution/src/assembly/confignode.xml index 502b90758cae2..3944c62325ef8 100644 --- a/distribution/src/assembly/confignode.xml +++ b/distribution/src/assembly/confignode.xml @@ -54,6 +54,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf diff --git a/distribution/src/assembly/datanode.xml b/distribution/src/assembly/datanode.xml index 84697fda6d742..9b0ae34a2bbb5 100644 --- a/distribution/src/assembly/datanode.xml +++ b/distribution/src/assembly/datanode.xml @@ -45,6 +45,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java index 550b0d7744b0c..cac76888f3362 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java @@ -60,7 +60,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java new file mode 100644 index 0000000000000..3ab24dd94f70a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.Statement; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSetConfigurationIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testSetConfiguration() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"=\"false\""); + statement.execute("set configuration \"enable_unseq_space_compaction\"=\"false\" on 0"); + statement.execute("set configuration \"enable_cross_space_compaction\"=\"false\" on 1"); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + for (ConfigNodeWrapper configNodeWrapper : EnvFactory.getEnv().getConfigNodeWrapperList()) { + String systemPropertiesPath = + configNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_unseq_space_compaction=false")); + } + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String systemPropertiesPath = + dataNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_cross_space_compaction=false")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java index f173e2ff9d600..d800bb8a054a7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java @@ -56,7 +56,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { close(); - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index c2086e8de8919..6d3ffe0b837d8 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -82,7 +82,7 @@ dclStatement ; utilityStatement - : flush | clearCache | settle | startRepairData | stopRepairData | explain + : flush | clearCache | setConfiguration | settle | startRepairData | stopRepairData | explain | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile @@ -973,6 +973,15 @@ clearCache : CLEAR CACHE (ON (LOCAL | CLUSTER))? ; +// Set Configuration +setConfiguration + : SET CONFIGURATION setConfigurationEntry+ (ON INTEGER_LITERAL)? + ; + +setConfigurationEntry + : STRING_LITERAL OPERATOR_SEQ STRING_LITERAL + ; + // Settle settle : SETTLE (prefixPath|tsFilePath=STRING_LITERAL) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index e697642668876..8c0e392f71bd0 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -22,10 +22,15 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-confignode.properties" + IF EXIST "%superior_dir%\conf\iotdb-confignode.properties" ( + set config_file="%superior_dir%\conf\iotdb-confignode.properties" + ) ELSE ( + echo "No configuration file found. Exiting." + exit /b 1 + ) ) for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" @@ -33,6 +38,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" set cn_internal_port=%%i ) +if not defined cn_internal_port ( + echo "cn_internal_port not found in the configuration file. Exiting." + exit /b 1 +) + echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" @@ -40,6 +50,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" set cn_internal_address=%%i ) +if not defined cn_internal_address ( + echo "cn_internal_address not found in the configuration file. Exiting." + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %cn_internal_address%:%cn_internal_port% ^| findstr LISTENING ') do ( taskkill /f /pid %%a echo "close ConfigNode, PID:" %%a diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java index e9aa1e93f9d72..5f3064b4eed73 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java @@ -27,5 +27,7 @@ public enum ConfigNodeRequestType { REMOVE_CONFIG_NODE, DELETE_CONFIG_NODE_PEER, REPORT_CONFIG_NODE_SHUTDOWN, - STOP_CONFIG_NODE + STOP_CONFIG_NODE, + SET_CONFIGURATION, + SHOW_CONFIGURATION, } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java index 1eb2500eb83d1..6f22341e3958b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java @@ -32,6 +32,8 @@ public enum DataNodeRequestType { STOP_REPAIR_DATA, LOAD_CONFIGURATION, SET_SYSTEM_STATUS, + SET_CONFIGURATION, + SHOW_CONFIGURATION, // Region Maintenance CREATE_DATA_REGION, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java index dd201a66b5745..155597ceb43d1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -341,6 +342,12 @@ private void sendAsyncRequestToDataNode( (AsyncTSStatusRPCHandler) clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); break; + case SET_CONFIGURATION: + client.setConfiguration( + (TSetConfigurationReq) clientHandler.getRequest(requestId), + (AsyncTSStatusRPCHandler) + clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); + break; case START_REPAIR_DATA: client.startRepairData( (AsyncTSStatusRPCHandler) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java index 04d2c0917698d..ab85a9badcb9e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -88,6 +89,8 @@ public Object sendSyncRequestToConfigNodeWithRetry( case STOP_CONFIG_NODE: // Only use stopConfigNode when the ConfigNode is removed. return client.stopConfigNode((TConfigNodeLocation) req); + case SET_CONFIGURATION: + return client.setConfiguration((TSetConfigurationReq) req); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index 1d0e564e43cd1..9628bc805094e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -64,7 +64,7 @@ private SyncDataNodeClientPool() { new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); } - public TSStatus sendSyncRequestToDataNodeWithRetry( + public Object sendSyncRequestToDataNodeWithRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType) { Throwable lastException = new TException(); for (int retry = 0; retry < DEFAULT_RETRY_NUM; retry++) { @@ -83,7 +83,7 @@ public TSStatus sendSyncRequestToDataNodeWithRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - public TSStatus sendSyncRequestToDataNodeWithGivenRetry( + public Object sendSyncRequestToDataNodeWithGivenRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType, int retryNum) { Throwable lastException = new TException(); for (int retry = 0; retry < retryNum; retry++) { @@ -102,7 +102,7 @@ public TSStatus sendSyncRequestToDataNodeWithGivenRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - private TSStatus executeSyncRequest( + private Object executeSyncRequest( DataNodeRequestType requestType, SyncDataNodeInternalServiceClient client, Object req) throws TException { switch (requestType) { @@ -138,6 +138,8 @@ private TSStatus executeSyncRequest( return client.deleteOldRegionPeer((TMaintainPeerReq) req); case RESET_PEER_LIST: return client.resetPeerList((TResetPeerListReq) req); + case SHOW_CONFIGURATION: + return client.showConfiguration(); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index 94ed04849ecc3..e73ab25debe42 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -55,13 +55,13 @@ public class ConfigNodeDescriptor { private final ConfigNodeConfig conf = new ConfigNodeConfig(); static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 8238598e651b7..c3e97c960908a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -28,14 +28,17 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; @@ -208,7 +211,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -219,6 +224,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -1496,6 +1502,33 @@ public TSStatus clearCache() { : status; } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + int currentNodeId = CONF.getConfigNodeId(); + if (req.getNodeId() < 0 || currentNodeId == req.getNodeId()) { + URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (url == null || !new File(url.getFile()).exists()) { + return tsStatus; + } + File file = new File(url.getFile()); + Properties properties = new Properties(); + properties.putAll(req.getConfigs()); + try { + ConfigurationFileUtils.updateConfigurationFile(file, properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + if (CONF.getConfigNodeId() == req.getNodeId()) { + return tsStatus; + } + } + tsStatus = confirmLeader(); + return tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? RpcUtils.squashResponseStatusList(nodeManager.setConfiguration(req)) + : tsStatus; + } + @Override public TSStatus startRepairData() { TSStatus status = confirmLeader(); @@ -1520,6 +1553,25 @@ public TSStatus loadConfiguration() { : status; } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) { + if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == nodeId) { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? nodeManager.showConfiguration(nodeId) + : new TShowConfigurationResp(status, ""); + } + @Override public TSStatus setSystemStatus(String systemStatus) { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 3ac0ee6ade41f..919e876bac213 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -23,7 +23,9 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; @@ -479,6 +481,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Clear cache on all DataNodes. */ TSStatus clearCache(); + /** Set Configuration. */ + TSStatus setConfiguration(TSetConfigurationReq req); + /** Check and repair unsorted tsfile by compaction. */ TSStatus startRepairData(); @@ -488,6 +493,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Load configuration on all DataNodes. */ TSStatus loadConfiguration(); + /** Show content of configuration file on specified node */ + TShowConfigurationResp showConfiguration(int nodeId); + /** Set system status on all DataNodes. */ TSStatus setSystemStatus(String status); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 3c3c9358f81de..c85f0fbaba5a2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -26,6 +26,8 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionRoleType; @@ -33,9 +35,11 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.confignode.client.ConfigNodeRequestType; import org.apache.iotdb.confignode.client.DataNodeRequestType; import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool; import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; @@ -725,6 +729,54 @@ public List clearCache() { return clientHandler.getResponseList(); } + public List setConfiguration(TSetConfigurationReq req) { + List responseList = new ArrayList<>(); + + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + Map targetDataNodes = new HashMap<>(); + int nodeId = req.getNodeId(); + // send to datanode + if (dataNodeLocationMap.containsKey(nodeId)) { + targetDataNodes.put(nodeId, dataNodeLocationMap.get(nodeId)); + } else if (nodeId < 0) { + targetDataNodes.putAll(dataNodeLocationMap); + } + if (!targetDataNodes.isEmpty()) { + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + responseList.addAll(clientHandler.getResponseList()); + } + + // send to config node + List configNodes = getRegisteredConfigNodes(); + for (TConfigNodeLocation configNode : configNodes) { + if (configNode.getConfigNodeId() == CONF.getConfigNodeId()) { + continue; + } + if (nodeId >= 0 && nodeId != configNode.getConfigNodeId()) { + continue; + } + TSStatus status = null; + try { + status = + (TSStatus) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + configNode.getInternalEndPoint(), + new TSetConfigurationReq(req.getConfigs(), configNode.getConfigNodeId()), + ConfigNodeRequestType.SET_CONFIGURATION); + } catch (Exception e) { + status = + RpcUtils.getStatus( + TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), e.getMessage()); + } + responseList.add(status); + } + return responseList; + } + public List startRpairData() { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); @@ -752,6 +804,39 @@ public List loadConfiguration() { return clientHandler.getResponseList(); } + public TShowConfigurationResp showConfiguration(int nodeId) { + TShowConfigurationResp resp = new TShowConfigurationResp(); + + // data node + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + if (dataNodeLocationMap.containsKey(nodeId)) { + TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); + return (TShowConfigurationResp) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + null, + DataNodeRequestType.SHOW_CONFIGURATION); + } + + // other config node + for (TConfigNodeLocation registeredConfigNode : getRegisteredConfigNodes()) { + if (registeredConfigNode.getConfigNodeId() != nodeId) { + continue; + } + resp = + (TShowConfigurationResp) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + registeredConfigNode.getInternalEndPoint(), + nodeId, + ConfigNodeRequestType.SHOW_CONFIGURATION); + return resp; + } + return resp; + } + public List setSystemStatus(String status) { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); @@ -763,11 +848,12 @@ public List setSystemStatus(String status) { } public TSStatus setDataNodeStatus(TSetDataNodeStatusReq setDataNodeStatusReq) { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), - setDataNodeStatusReq.getStatus(), - DataNodeRequestType.SET_SYSTEM_STATUS); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), + setDataNodeStatusReq.getStatus(), + DataNodeRequestType.SET_SYSTEM_STATUS); } /** @@ -800,11 +886,12 @@ private TSStatus killSpecificQuery(String queryId, TDataNodeLocation dataNodeLoc .setMessage( "The target DataNode is not existed, please ensure your input is correct"); } else { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeLocation.getInternalEndPoint(), - queryId, - DataNodeRequestType.KILL_QUERY_INSTANCE); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + queryId, + DataNodeRequestType.KILL_QUERY_INSTANCE); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 1e2a7d7f15019..00d9569c6593c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -181,18 +181,20 @@ public boolean invalidateCache(String storageGroupName) throws IOException, TExc if (nodeStatus == NodeStatus.Running) { // Always invalidate PartitionCache first final TSStatus invalidatePartitionStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_PARTITION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_PARTITION_CACHE); final TSStatus invalidateSchemaStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) { LOG.error( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java index ee3f8855d3367..4f82d5c502fbf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java @@ -140,11 +140,12 @@ public void broadcastDisableDataNode(TDataNodeLocation disabledDataNode) { for (TDataNodeConfiguration node : otherOnlineDataNodes) { TDisableDataNodeReq disableReq = new TDisableDataNodeReq(disabledDataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - node.getLocation().getInternalEndPoint(), - disableReq, - DataNodeRequestType.DISABLE_DATA_NODE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + node.getLocation().getInternalEndPoint(), + disableReq, + DataNodeRequestType.DISABLE_DATA_NODE); if (!isSucceed(status)) { LOGGER.error( "{}, BroadcastDisableDataNode meets error, disabledDataNode: {}, error: {}", @@ -224,11 +225,12 @@ public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocatio TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, storageGroup); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - destDataNode.getInternalEndPoint(), - req, - DataNodeRequestType.CREATE_NEW_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + destDataNode.getInternalEndPoint(), + req, + DataNodeRequestType.CREATE_NEW_REGION_PEER); if (isSucceed(status)) { LOGGER.info( @@ -269,11 +271,12 @@ public TSStatus submitAddRegionPeerTask( // destDataNode is where the new RegionReplica is created TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.ADD_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.ADD_REGION_PEER); LOGGER.info( "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}", REGION_MIGRATE_PROCESS, @@ -305,11 +308,12 @@ public TSStatus submitRemoveRegionPeerTask( TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, originalDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.REMOVE_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.REMOVE_REGION_PEER); LOGGER.info( "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}", REGION_MIGRATE_PROCESS, @@ -338,17 +342,19 @@ public TSStatus submitDeleteOldRegionPeerTask( status = configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId()) == NodeStatus.Unknown - ? SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER, - 1) - : SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER); + ? (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER, + 1) + : (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER); LOGGER.info( "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}", REGION_MIGRATE_PROCESS, @@ -499,9 +505,13 @@ public void stopDataNode(TDataNodeLocation dataNode) { REMOVE_DATANODE_PROCESS, dataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - dataNode.getInternalEndPoint(), dataNode, DataNodeRequestType.STOP_DATA_NODE, 2); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getInternalEndPoint(), + dataNode, + DataNodeRequestType.STOP_DATA_NODE, + 2); configManager.getLoadManager().removeNodeCache(dataNode.getDataNodeId()); LOGGER.info( "{}, Stop Data Node result: {}, stoppedDataNode: {}", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java index b3f10181c2016..badc5d2aa96b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java @@ -107,11 +107,12 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, AuthOperationProcedu continue; } status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - pair.getLeft().getLocation().getInternalEndPoint(), - req, - DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + pair.getLeft().getLocation().getInternalEndPoint(), + req, + DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { it.remove(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 82cb2672752c2..2a465f34eaf15 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -23,9 +23,11 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; @@ -808,6 +810,11 @@ public TSStatus clearCache() { return configManager.clearCache(); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return configManager.setConfiguration(req); + } + @Override public TSStatus startRepairData() { return configManager.startRepairData(); @@ -823,6 +830,11 @@ public TSStatus loadConfiguration() { return configManager.loadConfiguration(); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return configManager.showConfiguration(nodeId); + } + @Override public TSStatus setSystemStatus(String status) { return configManager.setSystemStatus(status); diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index d249d42756748..fec6924123462 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -22,10 +22,20 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-datanode.properties" + IF EXIST "%superior_dir%\conf\iotdb-datanode.properties" ( + set config_file=%superior_dir%\conf\iotdb-datanode.properties + ) ELSE ( + echo No configuration file found. Exiting. + exit /b 1 + ) +) + +if not defined config_file ( + echo No configuration file found. Exiting. + exit /b 1 ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" @@ -33,6 +43,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" set dn_rpc_port=%%i ) +if not defined dn_rpc_port ( + echo dn_rpc_port not found in the configuration file. Exiting. + exit /b 1 +) + echo Check whether the rpc_port is used..., port is %dn_rpc_port% for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" @@ -40,6 +55,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" set dn_rpc_address=%%i ) +if not defined dn_rpc_address ( + echo dn_rpc_address not found in the configuration file. Exiting. + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %dn_rpc_address%:%dn_rpc_port%') do ( taskkill /f /pid %%a echo Close DataNode, PID: %%a diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java index 43ab611e54b8a..bb5f557ec0ef5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java @@ -228,6 +228,7 @@ private static AuditLogOperation judgeLogOperation(StatementType type) { case INTERNAL_CREATE_TIMESERIES: case START_REPAIR_DATA: case STOP_REPAIR_DATA: + case SET_CONFIGURATION: return AuditLogOperation.DML; case LIST_USER: case LIST_ROLE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index fd1068b125773..c54b7d5ef4f95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -101,13 +101,13 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } @@ -1849,8 +1849,11 @@ public void loadHotModifiedProps() throws QueryProcessException { throw new QueryProcessException( String.format("Fail to reload config file %s because %s", url, e.getMessage())); } - ReloadLevel reloadLevel = - MetricConfigDescriptor.getInstance().loadHotProps(commonProperties, false); + reloadMetricProperties(commonProperties); + } + + public void reloadMetricProperties(Properties properties) { + ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(properties, false); LOGGER.info("Reload metric service in level {}", reloadLevel); if (reloadLevel == ReloadLevel.RESTART_INTERNAL_REPORTER) { IoTDBInternalReporter internalReporter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index c49b7e0b4451e..cda4f597284cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -24,9 +24,11 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.client.ClientManager; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.factory.ThriftClientFactory; @@ -647,6 +649,12 @@ public TSStatus clearCache() throws TException { () -> client.clearCache(), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.setConfiguration(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus startRepairData() throws TException { return executeRemoteCallWithRetry( @@ -665,6 +673,12 @@ public TSStatus loadConfiguration() throws TException { () -> client.loadConfiguration(), status -> !updateConfigNodeLeader(status)); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return executeRemoteCallWithRetry( + () -> client.showConfiguration(nodeId), resp -> !updateConfigNodeLeader(resp.getStatus())); + } + @Override public TSStatus setSystemStatus(String systemStatus) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 98c862f27b23f..f7a2d3ffcd5fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -23,8 +23,13 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationTemplateResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; @@ -45,6 +50,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.protocol.thrift.OperationType; @@ -2301,6 +2309,67 @@ public TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req) { } } + @Override + public TShowConfigurationTemplateResp showConfigurationTemplate() throws TException { + TShowConfigurationTemplateResp resp = new TShowConfigurationTemplateResp(); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + resp.setContent(ConfigurationFileUtils.readConfigurationTemplateFile()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + TShowConfigurationResp resp = new TShowConfigurationResp(); + resp.setContent(""); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + if (!clientSession.getUsername().equals(AuthorityChecker.SUPER_USER)) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.NO_PERMISSION)); + return resp; + } + + if (IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == nodeId) { + resp.setContent( + ConfigurationFileUtils.readConfigFileContent( + IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME))); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } + + try (ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + return client.showConfiguration(nodeId); + } catch (ClientManagerException e) { + throw new TException(e); + } + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + private TSQueryTemplateResp executeTemplateQueryStatement( Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) { long startTime = System.nanoTime(); @@ -2674,7 +2743,7 @@ private TSExecuteStatementResp createResponse(DatasetHeader header, long queryId return resp; } - private TSStatus getNotLoggedInStatus() { + protected TSStatus getNotLoggedInStatus() { return RpcUtils.getStatus( TSStatusCode.NOT_LOGIN, "Log in failed. Either you are not authorized or the session has timed out."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 6fc2a98e6a521..10155e26c4782 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -24,13 +24,16 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSettleReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -244,6 +247,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.ArrayList; @@ -1709,6 +1713,11 @@ public TSStatus clearCache() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + return StorageEngine.getInstance().setConfiguration(req); + } + @Override public TSStatus settle(TSettleReq req) throws TException { return SettleRequestHandler.getInstance().handleSettleRequest(req); @@ -1724,6 +1733,19 @@ public TSStatus loadConfiguration() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TShowConfigurationResp showConfiguration() { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + @Override public TSStatus setSystemStatus(String status) throws TException { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java index b024ea5a62a38..0786d2fb89386 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.MergeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; @@ -149,6 +150,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -259,6 +261,12 @@ public IConfigTask visitClearCache( return new ClearCacheTask(clearCacheStatement); } + @Override + public IConfigTask visitSetConfiguration( + SetConfigurationStatement setConfigurationStatement, MPPQueryContext context) { + return new SetConfigurationTask(setConfigurationStatement); + } + @Override public IConfigTask visitStartRepairData( StartRepairDataStatement startRepairDataStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index d0ebc9333823b..c2e8b404e6f59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -30,6 +31,7 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; @@ -1023,6 +1025,46 @@ public SettableFuture clearCache(boolean onCluster) { return future; } + @Override + public SettableFuture setConfiguration(TSetConfigurationReq req) { + SettableFuture future = SettableFuture.create(); + + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List ignoredConfigItems = + ConfigurationFileUtils.filterImmutableConfigItems(req.getConfigs()); + TSStatus warningTsStatus = null; + if (!ignoredConfigItems.isEmpty()) { + warningTsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + warningTsStatus.setMessage("ignored config items: " + ignoredConfigItems); + if (req.getConfigs().isEmpty()) { + future.setException(new IoTDBException(warningTsStatus.message, warningTsStatus.code)); + return future; + } + } + + boolean onLocal = IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == req.getNodeId(); + if (onLocal) { + tsStatus = StorageEngine.getInstance().setConfiguration(req); + } else { + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // Send request to some API server + tsStatus = client.setConfiguration(req); + } catch (ClientManagerException | TException e) { + future.setException(e); + } + } + if (warningTsStatus != null) { + tsStatus = warningTsStatus; + } + if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); + } + return future; + } + @Override public SettableFuture startRepairData(boolean onCluster) { SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index df24fec47da68..5adb94a56ce96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; @@ -120,6 +121,8 @@ public interface IConfigTaskExecutor { SettableFuture clearCache(boolean onCluster); + SettableFuture setConfiguration(TSetConfigurationReq tSetConfigurationReq); + SettableFuture loadConfiguration(boolean onCluster); SettableFuture setSystemStatus(boolean onCluster, NodeStatus status); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java new file mode 100644 index 0000000000000..f03c0aed7bd54 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; + +import com.google.common.util.concurrent.ListenableFuture; + +public class SetConfigurationTask implements IConfigTask { + + private final SetConfigurationStatement setConfigurationStatement; + + public SetConfigurationTask(SetConfigurationStatement setConfigurationStatement) { + this.setConfigurationStatement = setConfigurationStatement; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.setConfiguration( + new TSetConfigurationReq( + setConfigurationStatement.getConfigItems(), setConfigurationStatement.getNodeId())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index c0141de41f63d..473dffb08b50e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -196,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -3243,6 +3244,25 @@ public Statement visitClearCache(IoTDBSqlParser.ClearCacheContext ctx) { return clearCacheStatement; } + // Set Configuration + + @Override + public Statement visitSetConfiguration(IoTDBSqlParser.SetConfigurationContext ctx) { + SetConfigurationStatement setConfigurationStatement = + new SetConfigurationStatement(StatementType.SET_CONFIGURATION); + int nodeId = + Integer.parseInt(ctx.INTEGER_LITERAL() == null ? "-1" : ctx.INTEGER_LITERAL().getText()); + Map configItems = new HashMap<>(); + for (IoTDBSqlParser.SetConfigurationEntryContext entry : ctx.setConfigurationEntry()) { + String key = entry.STRING_LITERAL(0).getText().replace("\"", ""); + String value = entry.STRING_LITERAL(1).getText().replace("\"", ""); + configItems.put(key, value); + } + setConfigurationStatement.setNodeId(nodeId); + setConfigurationStatement.setConfigItems(configItems); + return setConfigurationStatement; + } + // Start Repair Data @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 88906ba83f4c9..4f49a5909e750 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -184,4 +184,5 @@ public enum StatementType { SHOW_TOPICS, SHOW_SUBSCRIPTIONS, + SET_CONFIGURATION } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8dba82f490b0..248cf881de12b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -391,6 +392,10 @@ public R visitClearCache(ClearCacheStatement clearCacheStatement, C context) { return visitStatement(clearCacheStatement, context); } + public R visitSetConfiguration(SetConfigurationStatement setConfigurationStatement, C context) { + return visitStatement(setConfigurationStatement, context); + } + public R visitStartRepairData(StartRepairDataStatement startRepairDataStatement, C context) { return visitStatement(startRepairDataStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java new file mode 100644 index 0000000000000..8f519de02807b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class SetConfigurationStatement extends Statement implements IConfigStatement { + + private int nodeId; + private Map configItems; + + public SetConfigurationStatement(StatementType setConfigurationType) { + this.statementType = setConfigurationType; + } + + public void setNodeId(int nodeId) { + this.nodeId = nodeId; + } + + public void setConfigItems(Map configItems) { + this.configItems = configItems; + } + + public int getNodeId() { + return nodeId; + } + + public Map getConfigItems() { + return configItems; + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitSetConfiguration(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index e0455b3329ff8..d26304e113cce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -20,12 +20,15 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.commons.concurrent.ExceptionalCountDownLatch; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; @@ -83,12 +86,14 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -602,6 +607,43 @@ public void clearCache() { BloomFilterCache.getInstance().clear(); } + public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + Map newConfigItems = req.getConfigs(); + if (newConfigItems.isEmpty()) { + return tsStatus; + } + Properties properties = new Properties(); + properties.putAll(newConfigItems); + + URL configFileUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (configFileUrl == null || !(new File(configFileUrl.getFile()).exists())) { + // configuration file not exist, update in mem + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + IoTDBDescriptor.getInstance().reloadMetricProperties(properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return tsStatus; + } + + // 1. append new configuration properties to configuration file + try { + ConfigurationFileUtils.updateConfigurationFile(new File(configFileUrl.getFile()), properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + + // 2. load hot modified properties + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return tsStatus; + } + /** * Add a listener to listen flush start/end events. Notice that this addition only applies to * TsFileProcessors created afterwards. diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 37371509d0cf4..ed0e4b10ea655 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -172,6 +172,15 @@ + + + ${project.basedir}/src/assembly/resources/conf + + iotdb-system.properties + + false + + + + generate-config-unix + compile + + exec + + + sh + + ${project.basedir}/src/assembly/resources/conf/generate_properties.sh + + + + + + + + + + generate-properties-windows + + + windows + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + generate-properties-windows + compile + + exec + + + cmd + + /c + ${project.basedir}\src\assembly\resources\conf\generate_properties.bat + + + + + + + + diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat new file mode 100644 index 0000000000000..f158d7fe6d2f6 --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat @@ -0,0 +1,64 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM +@echo off +setlocal enabledelayedexpansion + +set "source_file=src\assembly\resources\conf\iotdb-system.properties" +set "target_template_file=target\conf\iotdb-system.properties.template" +set "target_properties_file=target\conf\iotdb-system.properties" + +if exist "%target_template_file%" ( + del "%target_template_file%" +) + +if exist "%target_properties_file%" ( + del "%target_properties_file%" +) + +mkdir "%target_template_file%\.." + +copy "%source_file%" "%target_template_file%" + +echo # >> "%target_properties_file%" +echo # Licensed to the Apache Software Foundation (ASF) under one >> "%target_properties_file%" +echo # or more contributor license agreements. See the NOTICE file >> "%target_properties_file%" +echo # distributed with this work for additional information >> "%target_properties_file%" +echo # regarding copyright ownership. The ASF licenses this file >> "%target_properties_file%" +echo # to you under the Apache License, Version 2.0 (the >> "%target_properties_file%" +echo # "License"); you may not use this file except in compliance >> "%target_properties_file%" +echo # with the License. You may obtain a copy of the License at >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # http://www.apache.org/licenses/LICENSE-2.0 >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # Unless required by applicable law or agreed to in writing, >> "%target_properties_file%" +echo # software distributed under the License is distributed on an >> "%target_properties_file%" +echo # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY >> "%target_properties_file%" +echo # KIND, either express or implied. See the License for the >> "%target_properties_file%" +echo # specific language governing permissions and limitations >> "%target_properties_file%" +echo # under the License. >> "%target_properties_file%" +echo # >> "%target_properties_file%" + +for /f "usebackq tokens=*" %%i in ("%target_template_file%") do ( + set "line=%%i" + setlocal enabledelayedexpansion + if not "!line!"=="" if "!line:~0,1!" NEQ "#" ( + echo !line!>>"%target_properties_file%" + ) + endlocal +) \ No newline at end of file diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh new file mode 100644 index 0000000000000..f14329b76a6a0 --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +source_file="src/assembly/resources/conf/iotdb-system.properties" +target_template_file="target/conf/iotdb-system.properties.template" +target_properties_file="target/conf/iotdb-system.properties" + +if [ -f "$target_template_file" ]; then + rm "$target_template_file" +fi + +if [ -f "$target_properties_file" ]; then + rm "$target_properties_file" +fi + +mkdir -p "$(dirname "$target_template_file")" + +cp "$source_file" "$target_template_file" + +# Add license header to the target properties file +cat < "$target_properties_file" +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +EOL + +grep -v '^\s*#' "$target_template_file" | grep -v '^\s*$' >> "$target_properties_file" diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java deleted file mode 100644 index 49b44f0bc02f6..0000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.conf; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.net.URL; -import java.nio.file.Files; -import java.util.concurrent.TimeUnit; - -public class ConfigFileAutoUpdateTool { - - private final String lockFileSuffix = ".lock"; - private final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); - private final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); - private Logger logger = LoggerFactory.getLogger(ConfigFileAutoUpdateTool.class); - private String license = - "#\n" - + "# Licensed to the Apache Software Foundation (ASF) under one\n" - + "# or more contributor license agreements. See the NOTICE file\n" - + "# distributed with this work for additional information\n" - + "# regarding copyright ownership. The ASF licenses this file\n" - + "# to you under the Apache License, Version 2.0 (the\n" - + "# \"License\"); you may not use this file except in compliance\n" - + "# with the License. You may obtain a copy of the License at\n" - + "#\n" - + "# http://www.apache.org/licenses/LICENSE-2.0\n" - + "#\n" - + "# Unless required by applicable law or agreed to in writing,\n" - + "# software distributed under the License is distributed on an\n" - + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" - + "# KIND, either express or implied. See the License for the\n" - + "# specific language governing permissions and limitations\n" - + "# under the License."; - - public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) - throws IOException, InterruptedException { - if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { - return; - } - File systemFile = new File(systemUrl.getFile()); - File configNodeFile = new File(configNodeUrl.getFile()); - File dataNodeFile = new File(dataNodeUrl.getFile()); - File commonFile = new File(commonUrl.getFile()); - - if (systemFile.exists()) { - return; - } - boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); - if (!canUpdate) { - return; - } - - File lockFile = new File(systemFile.getPath() + lockFileSuffix); - acquireTargetFileLock(lockFile); - try { - // other progress updated this file - if (systemFile.exists()) { - return; - } - try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { - raf.write(license.getBytes()); - String configNodeContent = readConfigLines(configNodeFile); - raf.write(configNodeContent.getBytes()); - String dataNodeContent = readConfigLines(dataNodeFile); - raf.write(dataNodeContent.getBytes()); - String commonContent = readConfigLines(commonFile); - raf.write(commonContent.getBytes()); - } - Files.move(lockFile.toPath(), systemFile.toPath()); - } finally { - releaseFileLock(lockFile); - } - } - - private String readConfigLines(File file) throws IOException { - if (!file.exists()) { - return ""; - } - byte[] bytes = Files.readAllBytes(file.toPath()); - String content = new String(bytes); - return content.replace(license, ""); - } - - private void acquireTargetFileLock(File file) throws IOException, InterruptedException { - long totalWaitTime = 0; - while (totalWaitTime < maxTimeMillsToAcquireLock) { - if (file.createNewFile()) { - return; - } - totalWaitTime += waitTimeMillsPerCheck; - Thread.sleep(waitTimeMillsPerCheck); - } - logger.warn( - "Waiting for {} seconds to acquire configuration file update lock." - + " There may have been an unexpected interruption in the last" - + " configuration file update. Ignore temporary file {}", - totalWaitTime / 1000, - file.getName()); - } - - private void releaseFileLock(File file) throws IOException { - Files.deleteIfExists(file.toPath()); - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java new file mode 100644 index 0000000000000..0bd422d462021 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.conf; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.RandomAccessFile; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class ConfigurationFileUtils { + + private static final String lockFileSuffix = ".lock"; + private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); + private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); + private static Logger logger = LoggerFactory.getLogger(ConfigurationFileUtils.class); + private static String license = + "#\n" + + "# Licensed to the Apache Software Foundation (ASF) under one\n" + + "# or more contributor license agreements. See the NOTICE file\n" + + "# distributed with this work for additional information\n" + + "# regarding copyright ownership. The ASF licenses this file\n" + + "# to you under the Apache License, Version 2.0 (the\n" + + "# \"License\"); you may not use this file except in compliance\n" + + "# with the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing,\n" + + "# software distributed under the License is distributed on an\n" + + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "# KIND, either express or implied. See the License for the\n" + + "# specific language governing permissions and limitations\n" + + "# under the License."; + + // This is a temporary implementations + private static final Set ignoreConfigKeys = + new HashSet<>( + Arrays.asList( + "cn_internal_address", + "cn_internal_port", + "cn_consensus_port", + "cn_seed_config_node", + "dn_internal_address", + "dn_internal_port", + "dn_mpp_data_exchange_port", + "dn_schema_region_consensus_port", + "dn_data_region_consensus_port", + "dn_seed_config_node", + "dn_session_timeout_threshold", + "config_node_consensus_protocol_class", + "schema_replication_factor", + "data_replication_factor", + "data_region_consensus_protocol_class", + "series_slot_num", + "series_partition_executor_class", + "time_partition_interval", + "schema_engine_mode", + "tag_attribute_flush_interval", + "tag_attribute_total_size", + "timestamp_precision", + "iotdb_server_encrypt_decrypt_provider", + "iotdb_server_encrypt_decrypt_provider_parameter")); + + public static void checkAndMayUpdate( + URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) + throws IOException, InterruptedException { + if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { + return; + } + File systemFile = new File(systemUrl.getFile()); + File configNodeFile = new File(configNodeUrl.getFile()); + File dataNodeFile = new File(dataNodeUrl.getFile()); + File commonFile = new File(commonUrl.getFile()); + + if (systemFile.exists()) { + return; + } + boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); + if (!canUpdate) { + return; + } + + File lockFile = new File(systemFile.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + try { + // other progress updated this file + if (systemFile.exists()) { + return; + } + try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { + raf.write(license.getBytes()); + String configNodeContent = readConfigLinesWithoutLicense(configNodeFile); + raf.write(configNodeContent.getBytes()); + String dataNodeContent = readConfigLinesWithoutLicense(dataNodeFile); + raf.write(dataNodeContent.getBytes()); + String commonContent = readConfigLinesWithoutLicense(commonFile); + raf.write(commonContent.getBytes()); + } + Files.move(lockFile.toPath(), systemFile.toPath()); + } finally { + releaseFileLock(lockFile); + } + } + + public static String readConfigFileContent(URL url) throws IOException { + if (url == null) { + return ""; + } + File f = new File(url.getFile()); + if (!f.exists()) { + return ""; + } + return readConfigLines(f); + } + + public static String readConfigurationTemplateFile() throws IOException { + StringBuilder content = new StringBuilder(); + try (InputStream inputStream = + ConfigurationFileUtils.class + .getClassLoader() + .getResourceAsStream(CommonConfig.SYSTEM_CONFIG_NAME); + InputStreamReader isr = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(isr)) { + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append(System.lineSeparator()); + } + } catch (IOException e) { + logger.warn("Failed to read configuration template", e); + throw e; + } + return content.toString(); + } + + public static List filterImmutableConfigItems(Map configItems) { + List ignoredConfigItems = new ArrayList<>(); + for (String ignoredKey : ignoreConfigKeys) { + if (configItems.containsKey(ignoredKey)) { + configItems.remove(ignoredKey); + ignoredConfigItems.add(ignoredKey); + } + } + return ignoredConfigItems; + } + + public static void updateConfigurationFile(File file, Properties newConfigItems) + throws IOException, InterruptedException { + // read configuration file + List lines = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } + // generate new configuration file content in memory + StringBuilder contentsOfNewConfigurationFile = new StringBuilder(); + for (String currentLine : lines) { + if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + int equalsIndex = currentLine.indexOf('='); + // replace old config + if (equalsIndex != -1) { + String key = currentLine.substring(0, equalsIndex).trim(); + String value = currentLine.substring(equalsIndex + 1).trim(); + if (!newConfigItems.containsKey(key)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + if (newConfigItems.getProperty(key).equals(value)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + newConfigItems.remove(key); + } else { + contentsOfNewConfigurationFile + .append("#") + .append(currentLine) + .append(System.lineSeparator()); + } + } + } + if (newConfigItems.isEmpty()) { + // No configuration needs to be modified + return; + } + File lockFile = new File(file.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + logger.info("Updating configuration file {}", file.getAbsolutePath()); + try { + try (FileWriter writer = new FileWriter(lockFile)) { + writer.write(contentsOfNewConfigurationFile.toString()); + // add new config items + newConfigItems.store(writer, null); + writer.flush(); + } + Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } finally { + releaseFileLock(lockFile); + } + } + + private static String readConfigLinesWithoutLicense(File file) throws IOException { + if (!file.exists()) { + return ""; + } + byte[] bytes = Files.readAllBytes(file.toPath()); + String content = new String(bytes); + return content.replace(license, ""); + } + + private static String readConfigLines(File file) throws IOException { + if (!file.exists()) { + return ""; + } + byte[] bytes = Files.readAllBytes(file.toPath()); + return new String(bytes); + } + + private static void acquireTargetFileLock(File file) throws IOException, InterruptedException { + long totalWaitTime = 0; + while (totalWaitTime < maxTimeMillsToAcquireLock) { + if (file.createNewFile()) { + return; + } + totalWaitTime += waitTimeMillsPerCheck; + Thread.sleep(waitTimeMillsPerCheck); + } + logger.warn( + "Waiting for {} seconds to acquire configuration file update lock." + + " There may have been an unexpected interruption in the last" + + " configuration file update. Ignore temporary file {}", + totalWaitTime / 1000, + file.getName()); + } + + private static void releaseFileLock(File file) throws IOException { + Files.deleteIfExists(file.toPath()); + } +} diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index d81730337c627..49dc50d94c476 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -127,6 +127,11 @@ struct TSetTTLReq { 3: required bool isDataBase } +struct TSetConfigurationReq { + 1: required map configs + 2: required i32 nodeId +} + // for File struct TFile { 1: required string fileName @@ -210,4 +215,15 @@ enum TAggregationType { MAX_BY, MIN_BY, UDAF -} \ No newline at end of file +} + +struct TShowConfigurationTemplateResp { + 1: required TSStatus status + 2: required string content +} + +struct TShowConfigurationResp { + 1: required TSStatus status + 2: required string content +} + diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 25ed9d3c64cf2..ad75ba17c1d31 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1327,6 +1327,12 @@ service IConfigNodeRPCService { /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ common.TSStatus clearCache() + /** Set configuration on specified node */ + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + + /** Show content of configuration file */ + common.TShowConfigurationResp showConfiguration(1: i32 nodeId) + /** Check and repair unsorted tsfile by compaction */ common.TSStatus startRepairData() diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index a99a8e599c937..6069199bd64c9 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -634,6 +634,10 @@ service IClientRPCService { TSQueryTemplateResp querySchemaTemplate(1:TSQueryTemplateReq req); + common.TShowConfigurationTemplateResp showConfigurationTemplate(); + + common.TShowConfigurationResp showConfiguration(1:i32 nodeId); + common.TSStatus setSchemaTemplate(1:TSSetSchemaTemplateReq req); common.TSStatus unsetSchemaTemplate(1:TSUnsetSchemaTemplateReq req); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 2b90240fce0fa..d4ef59eee2b79 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -851,6 +851,10 @@ service IDataNodeRPCService { common.TSStatus clearCache() + common.TShowConfigurationResp showConfiguration() + + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + common.TSStatus loadConfiguration() common.TSStatus setSystemStatus(string status)