From 5a5856de955ed72d42402574d76ffa7bd107b6f4 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 May 2024 16:08:15 +0800 Subject: [PATCH 01/45] merge all config file to iotdb-system.properties --- .../confignode/conf/ConfigNodeDescriptor.java | 48 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 104 +- .../conf/rest/IoTDBRestServiceDescriptor.java | 9 +- .../resources/conf/iotdb-system.properties | 1539 +++++++++++++++++ .../iotdb/commons/conf/CommonConfig.java | 5 +- .../conf/ConfigFileAutoUpdateTool.java | 108 ++ 6 files changed, 1726 insertions(+), 87 deletions(-) create mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index ee9621d2393c3..cea89efb02bb5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -36,7 +37,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -54,6 +54,19 @@ public class ConfigNodeDescriptor { private final ConfigNodeConfig conf = new ConfigNodeConfig(); + static { + ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); + URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); + try { + updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + } catch (Exception e) { + LOGGER.error("Failed to update config file", e); + } + } + private ConfigNodeDescriptor() { loadProps(); } @@ -67,7 +80,7 @@ public ConfigNodeConfig getConf() { * * @return url object if location exit, otherwise null. */ - public URL getPropsUrl(String configFileName) { + public static URL getPropsUrl(String configFileName) { // Check if a config-directory was specified first. String urlString = System.getProperty(ConfigNodeConstant.CONFIGNODE_CONF, null); // If it wasn't, check if a home directory was provided @@ -102,37 +115,12 @@ else if (!urlString.endsWith(".properties")) { } private void loadProps() { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); Properties commonProperties = new Properties(); - if (url != null) { - try (InputStream inputStream = url.openStream()) { - - LOGGER.info("Start to read config file {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - - } catch (FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject ConfigNode startup.", url, e); - System.exit(-1); - } catch (IOException e) { - LOGGER.error("Cannot load config file, reject ConfigNode startup.", e); - System.exit(-1); - } catch (Exception e) { - LOGGER.error("Incorrect format in config file, reject ConfigNode startup.", e); - System.exit(-1); - } - } else { - LOGGER.warn( - "Couldn't load the configuration {} from any of the known sources.", - CommonConfig.CONFIG_NAME); - } - - url = getPropsUrl(ConfigNodeConstant.CONF_FILE_NAME); + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url != null) { try (InputStream inputStream = url.openStream()) { LOGGER.info("start reading ConfigNode conf file: {}", url); - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - commonProperties.putAll(properties); + commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); loadProperties(commonProperties); } catch (IOException | BadNodeUrlException e) { LOGGER.error("Couldn't load ConfigNode conf file, reject ConfigNode startup.", e); @@ -151,7 +139,7 @@ private void loadProps() { } else { LOGGER.warn( "Couldn't load the configuration {} from any of the known sources.", - ConfigNodeConstant.CONF_FILE_NAME); + CommonConfig.SYSTEM_CONFIG_NAME); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 690ad97f6f5e3..856d73cd9ac77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -87,9 +88,9 @@ public class IoTDBDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDescriptor.class); - private final CommonDescriptor commonDescriptor = CommonDescriptor.getInstance(); + private static final CommonDescriptor commonDescriptor = CommonDescriptor.getInstance(); - private final IoTDBConfig conf = new IoTDBConfig(); + private static final IoTDBConfig conf = new IoTDBConfig(); private static final long MAX_THROTTLE_THRESHOLD = 600 * 1024 * 1024 * 1024L; @@ -99,6 +100,19 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; + static { + ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); + URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); + try { + updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + } catch (Exception e) { + LOGGER.error("Failed to update config file", e); + } + } + protected IoTDBDescriptor() { loadProps(); ServiceLoader propertiesLoaderServiceLoader = @@ -136,7 +150,7 @@ public IoTDBConfig getConfig() { * * @return url object if location exit, otherwise null. */ - public URL getPropsUrl(String configFileName) { + public static URL getPropsUrl(String configFileName) { String urlString = commonDescriptor.getConfDir(); if (urlString == null) { // If urlString wasn't provided, try to find a default config in the root of the classpath. @@ -174,28 +188,9 @@ else if (!urlString.endsWith(".properties")) { /** load a property file and set TsfileDBConfig variables. */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); Properties commonProperties = new Properties(); - if (url != null) { - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to read config file {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } catch (FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject DataNode startup.", url, e); - System.exit(-1); - } catch (IOException e) { - LOGGER.error("Cannot load config file, reject DataNode startup.", e); - System.exit(-1); - } catch (Exception e) { - LOGGER.error("Incorrect format in config file, reject DataNode startup.", e); - System.exit(-1); - } - } else { - LOGGER.warn( - "Couldn't load the configuration {} from any of the known sources.", - CommonConfig.CONFIG_NAME); - } - url = getPropsUrl(IoTDBConfig.CONFIG_NAME); + // if new properties file exist, skip old properties files + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url != null) { try (InputStream inputStream = url.openStream()) { LOGGER.info("Start to read config file {}", url); @@ -225,7 +220,7 @@ private void loadProps() { } else { LOGGER.warn( "Couldn't load the configuration {} from any of the known sources.", - IoTDBConfig.CONFIG_NAME); + CommonConfig.SYSTEM_CONFIG_NAME); } } @@ -1725,37 +1720,30 @@ public void loadHotModifiedProps(Properties properties) throws QueryProcessExcep } public void loadHotModifiedProps() throws QueryProcessException { - URL url = getPropsUrl(CommonConfig.CONFIG_NAME); - if (url == null) { - LOGGER.warn("Couldn't load the configuration from any of the known sources."); - return; - } - Properties commonProperties = new Properties(); - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to reload config file {}", url); - commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } catch (Exception e) { - LOGGER.warn("Fail to reload config file {}", url, e); - throw new QueryProcessException( - String.format("Fail to reload config file %s because %s", url, e.getMessage())); - } - url = getPropsUrl(IoTDBConfig.CONFIG_NAME); - if (url == null) { - LOGGER.warn("Couldn't load the configuration from any of the known sources."); - return; + URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (url != null && new File(url.getFile()).exists()) { + loadHotModifiedPropertiesFromUrl(commonProperties, url); + } else { + url = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); + if (url == null) { + LOGGER.warn("Couldn't load the configuration from any of the known sources."); + return; + } + loadHotModifiedPropertiesFromUrl(commonProperties, url); + + url = getPropsUrl(IoTDBConfig.CONFIG_NAME); + if (url == null) { + LOGGER.warn("Couldn't load the configuration from any of the known sources."); + return; + } + loadHotModifiedPropertiesFromUrl(commonProperties, url); } - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to reload config file {}", url); - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - commonProperties.putAll(properties); + try { loadHotModifiedProps(commonProperties); } catch (Exception e) { - LOGGER.warn("Fail to reload config file {}", url, e); - throw new QueryProcessException( - String.format("Fail to reload config file %s because %s", url, e.getMessage())); + LOGGER.error("Failed to reload configs because {}", e.getMessage()); } ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(commonProperties); LOGGER.info("Reload metric service in level {}", reloadLevel); @@ -1773,6 +1761,20 @@ public void loadHotModifiedProps() throws QueryProcessException { } } + private void loadHotModifiedPropertiesFromUrl(Properties commonProperties, URL url) + throws QueryProcessException { + try (InputStream inputStream = url.openStream()) { + LOGGER.info("Start to reload config file {}", url); + Properties properties = new Properties(); + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + commonProperties.putAll(properties); + } catch (Exception e) { + LOGGER.warn("Fail to reload config file {}", url, e); + throw new QueryProcessException( + String.format("Fail to reload config file %s because %s", url, e.getMessage())); + } + } + private void initMemoryAllocate(Properties properties) { String memoryAllocateProportion = properties.getProperty("datanode_memory_proportion", null); if (memoryAllocateProportion == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index e0d0df3dee954..48725ef1839d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.conf.rest; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -40,11 +41,9 @@ public class IoTDBRestServiceDescriptor { private final IoTDBRestServiceConfig conf = new IoTDBRestServiceConfig(); protected IoTDBRestServiceDescriptor() { - Properties properties = loadProps(IoTDBRestServiceConfig.CONFIG_NAME); - if (properties != null) { - if (!properties.containsKey("enable_rest_service")) { - properties = loadProps(IoTDBRestServiceConfig.OLD_CONFIG_NAME); - } + URL systemConfig = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (systemConfig != null) { + Properties properties = loadProps(CommonConfig.SYSTEM_CONFIG_NAME); if (properties != null) { loadProps(properties); } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties new file mode 100644 index 0000000000000..c58bf3026ee0a --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -0,0 +1,1539 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#################### +### Config Node RPC Configuration +#################### + +# Used for RPC communication inside cluster. +# Could set 127.0.0.1(for local test) or ipv4 address. +# Datatype: String +cn_internal_address=127.0.0.1 + +# Used for RPC communication inside cluster. +# Datatype: int +cn_internal_port=10710 + +# Used for consensus communication among ConfigNodes inside cluster. +# Datatype: int +cn_consensus_port=10720 + +#################### +### Seed ConfigNode +#################### + +# For the first ConfigNode to start, cn_seed_config_node points to its own cn_internal_address:cn_internal_port. +# For other ConfigNodes that to join the cluster, cn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. +# Note: After this ConfigNode successfully joins the cluster for the first time, this parameter is no longer used. +# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. +# Format: address:port e.g. 127.0.0.1:10710 +# Datatype: String +cn_seed_config_node=127.0.0.1:10710 + +#################### +### Directory configuration +#################### + +# system dir +# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/system). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# cn_system_dir=data\\confignode\\system +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# cn_system_dir=data/confignode/system + +# consensus dir +# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/consensus). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. +# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# cn_consensus_dir=data\\confignode\\consensus +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# cn_consensus_dir=data/confignode/consensus + +# pipe_receiver_file_dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${cn_system_dir}/pipe/receiver). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If pipe_receiver_file_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dir=data\\confignode\\system\\pipe\\receiver +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dir=data/confignode/system/pipe/receiver + +#################### +### thrift rpc configuration +#################### + +# this feature is under development, set this as false before it is done. +# Datatype: boolean +# cn_rpc_thrift_compression_enable=false + +# if true, a snappy based compression method will be called before sending data by the network +# Datatype: boolean +# this feature is under development, set this as false before it is done. +# cn_rpc_advanced_compression_enable=false + +# Datatype: int +# cn_rpc_max_concurrent_client_num=65535 + +# thrift max frame size, 512MB by default +# Datatype: int +# cn_thrift_max_frame_size=536870912 + +# thrift init buffer size +# Datatype: int +# cn_thrift_init_buffer_size=1024 + +# Thrift socket and connection timeout between raft nodes, in milliseconds. +# Datatype: int +# cn_connection_timeout_ms=60000 + +# selector thread (TAsyncClientManager) nums for async thread in a clientManager +# Datatype: int +# cn_selector_thread_nums_of_client_manager=1 + +# The maximum number of clients that can be allocated for a node in a clientManager. +# when the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked +# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. +# Datatype: int +# cn_max_client_count_for_each_node_in_client_manager=300 + +#################### +### Metric Configuration +#################### + +# The reporters of metric module to report metrics +# If there are more than one reporter, please separate them by commas ",". +# Options: [JMX, PROMETHEUS] +# Datatype: String +# cn_metric_reporter_list= + +# The level of metric module +# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] +# Datatype: String +# cn_metric_level=CORE + +# The period of async collection of some metrics in second +# Datatype: int +# cn_metric_async_collect_period=5 + +# The port of prometheus reporter of metric module +# Datatype: int +# cn_metric_prometheus_reporter_port=9091 + + + +#################### +### Data Node RPC Configuration +#################### + +# Used for connection of IoTDB native clients(Session) +# Could set 127.0.0.1(for local test) or ipv4 address +# Datatype: String +dn_rpc_address=0.0.0.0 + +# Used for connection of IoTDB native clients(Session) +# Bind with dn_rpc_address +# Datatype: int +dn_rpc_port=6667 + +# Used for communication inside cluster. +# could set 127.0.0.1(for local test) or ipv4 address. +# Datatype: String +dn_internal_address=127.0.0.1 + +# Used for communication inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_internal_port=10730 + +# Port for data exchange among DataNodes inside cluster +# Bind with dn_internal_address +# Datatype: int +dn_mpp_data_exchange_port=10740 + +# port for consensus's communication for schema region inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_schema_region_consensus_port=10750 + +# port for consensus's communication for data region inside cluster. +# Bind with dn_internal_address +# Datatype: int +dn_data_region_consensus_port=10760 + +# Datatype: long +# The time of data node waiting for the next retry to join into the cluster. +# dn_join_cluster_retry_interval_ms=5000 + +#################### +### SSL Configuration +#################### + +# Does dn_rpc_port enable SSL +# enable_thrift_ssl=false + +# Rest Service enabled SSL +# enable_https=false + +# SSL key store path +# linux e.g. /home/iotdb/server.keystore (absolute path) or server.keystore (relative path) +# windows e.g. C:\\iotdb\\server.keystore (absolute path) or server.keystore (relative path) +# key_store_path= + +# SSL key store password +# key_store_pwd= + +#################### +### Seed ConfigNode +#################### + +# dn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. +# Note: After this DataNode successfully joins the cluster for the first time, this parameter is no longer used. +# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. +# Format: address:port e.g. 127.0.0.1:10710 +# Datatype: String +dn_seed_config_node=127.0.0.1:10710 + +#################### +### Connection Configuration +#################### + +# The maximum session idle time. unit: ms +# Idle sessions are the ones that performs neither query or non-query operations for a period of time +# Set to 0 to disable session timeout +# Datatype: int +# dn_session_timeout_threshold=0 + +# Datatype: boolean +# dn_rpc_thrift_compression_enable=false + +# if true, a snappy based compression method will be called before sending data by the network +# Datatype: boolean +# this feature is under development, set this as false before it is done. +# dn_rpc_advanced_compression_enable=false + +# Datatype: int +# dn_rpc_selector_thread_count=1 + +# Datatype: int +# dn_rpc_min_concurrent_client_num=1 + +# Datatype: int +# dn_rpc_max_concurrent_client_num=65535 + +# thrift max frame size, 512MB by default +# Datatype: int +# dn_thrift_max_frame_size=536870912 + +# thrift init buffer size +# Datatype: int +# dn_thrift_init_buffer_size=1024 + +# Thrift socket and connection timeout between raft nodes, in milliseconds. +# Datatype: int +# dn_connection_timeout_ms=60000 + +# selector thread (TAsyncClientManager) nums for async thread in a clientManager +# Datatype: int +# dn_selector_thread_count_of_client_manager=1 + +# The maximum number of clients that can be allocated for a node in a clientManager. +# When the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked +# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. +# Datatype: int +# dn_max_client_count_for_each_node_in_client_manager=300 + +#################### +### Directory Configuration +#################### + +# system dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/system). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_system_dir=data\\datanode\\system +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_system_dir=data/datanode/system + + +# data dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/data). +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If data_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_data_dirs=data\\datanode\\data +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_data_dirs=data/datanode/data + + +# multi_dir_strategy +# The strategy is used to choose a directory from data_dirs for the system to store a new tsfile. +# System provides two strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy. +# The info of the two strategies are as follows: +# 1. SequenceStrategy: the system will choose the directory in sequence. +# 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space. +# Set SequenceStrategy or MaxDiskUsableSpaceFirstStrategy to apply the corresponding strategy. +# If this property is unset, system will use SequenceStrategy as default strategy. +# For this property, fully-qualified class name (include package name) and simple class name are both acceptable. +# dn_multi_dir_strategy=SequenceStrategy + +# consensus dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If consensus_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_consensus_dir=data\\datanode\\consensus +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_consensus_dir=data/datanode/consensus + +# wal dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If wal_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_wal_dirs=data\\datanode\\wal +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_wal_dirs=data/datanode/wal + +# tracing dir +# Uncomment following fields to configure the tracing root directory. +# For Windows platform, the index is as follows: +# dn_tracing_dir=datanode\\tracing +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_tracing_dir=datanode/tracing + +# sync dir +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# dn_sync_dir=data\\datanode\\sync +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# dn_sync_dir=data/datanode/sync + +# sort_tmp_dir +# This property is used to configure the temporary directory for sorting operation. +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# Note: If sort_tmp_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# sort_tmp_dir=data\\datanode\\tmp +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# sort_tmp_dir=data/datanode/tmp + +# pipe_receiver_file_dirs +# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${dn_system_dir}/pipe/receiver). +# If it is absolute, system will save the data in the exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. +# If there are more than one directory, please separate them by commas ",". +# Note: If pipe_receiver_file_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. +# For windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dirs=data\\datanode\\system\\pipe\\receiver +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# pipe_receiver_file_dirs=data/datanode/system/pipe/receiver + +#################### +### Metric Configuration +#################### + +# The reporters of metric module to report metrics +# If there are more than one reporter, please separate them by commas ",". +# Options: [JMX, PROMETHEUS] +# Datatype: String +# dn_metric_reporter_list= + +# The level of metric module +# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] +# Datatype: String +# dn_metric_level=CORE + +# The period of async collection of some metrics in second +# Datatype: int +# dn_metric_async_collect_period=5 + +# The port of prometheus reporter of metric module +# Datatype: int +dn_metric_prometheus_reporter_port=9092 + +# The type of internal reporter in metric module, used for checking flushed point number +# Options: [MEMORY, IOTDB] +# Datatype: String +# dn_metric_internal_reporter_type=MEMORY + +#################### +### REST Service Configuration +#################### + +# Is the REST service enabled +# enable_rest_service=false + +# the binding port of the REST service +# rest_service_port=18080 + +# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json +# enable_swagger=false + +# the default row limit to a REST query response when the rowSize parameter is not given in request +# rest_query_default_row_size_limit=10000 + +# the expiration time of the user login information cache (in seconds) +# cache_expire_in_seconds=28800 + +# maximum number of users can be stored in the user login cache. +# cache_max_num=100 + +# init capacity of users can be stored in the user login cache. +# cache_init_num=10 + +# Is client authentication required +# client_auth=false + +# SSL trust store path +# trust_store_path="" + +# SSL trust store password. +# trust_store_pwd="" + +# SSL timeout (in seconds) +# idle_timeout_in_seconds=50000 + + + + +#################### +### Cluster Configuration +#################### + +# Used for indicate cluster name and distinguish different cluster. +# Datatype: string +cluster_name=defaultCluster + +#################### +### Replication configuration +#################### + +# ConfigNode consensus protocol type. +# This parameter is unmodifiable after ConfigNode starts for the first time. +# These consensus protocols are currently supported: +# 1. org.apache.iotdb.consensus.ratis.RatisConsensus +# 2. org.apache.iotdb.consensus.simple.SimpleConsensus (Only 1 ConfigNode can be deployed) +# Datatype: string +# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus + +# Default number of schema replicas +# Can not be changed after the first start +# Datatype: int +schema_replication_factor=1 + +# SchemaRegion consensus protocol type. +# This parameter is unmodifiable after ConfigNode starts for the first time. +# These consensus protocols are currently supported: +# 1. org.apache.iotdb.consensus.ratis.RatisConsensus +# 2. org.apache.iotdb.consensus.simple.SimpleConsensus (The schema_replication_factor can only be set to 1) +# Datatype: string +# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus + +# Default number of data replicas +# Can not be changed after the first start +# Datatype: int +data_replication_factor=1 + +# DataRegion consensus protocol type. +# This parameter is unmodifiable after ConfigNode starts for the first time. +# These consensus protocols are currently supported: +# 1. org.apache.iotdb.consensus.simple.SimpleConsensus (The data_replication_factor can only be set to 1) +# 2. org.apache.iotdb.consensus.iot.IoTConsensus +# 3. org.apache.iotdb.consensus.ratis.RatisConsensus +# Datatype: string +# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus + +#################### +### Load balancing configuration +#################### + +# All parameters in Partition configuration is unmodifiable after ConfigNode starts for the first time. +# And these parameters should be consistent within the ConfigNodeGroup. +# Number of SeriesPartitionSlots per Database +# Datatype: Integer +# series_slot_num=1000 + +# SeriesPartitionSlot executor class +# These hashing algorithms are currently supported: +# 1. BKDRHashExecutor(Default) +# 2. APHashExecutor +# 3. JSHashExecutor +# 4. SDBMHashExecutor +# Also, if you want to implement your own SeriesPartition executor, you can inherit the SeriesPartitionExecutor class and +# modify this parameter to correspond to your Java class +# Datatype: String +# series_partition_executor_class=org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor + +# The policy of extension SchemaRegionGroup for each Database. +# These policies are currently supported: +# 1. CUSTOM(Each Database will allocate schema_region_group_per_database RegionGroups as soon as created) +# 2. AUTO(Each Database will automatically extend SchemaRegionGroups based on the data it has) +# Datatype: String +# schema_region_group_extension_policy=AUTO + +# When set schema_region_group_extension_policy=CUSTOM, +# this parameter is the default number of SchemaRegionGroups for each Database. +# When set schema_region_group_extension_policy=AUTO, +# this parameter is the default minimal number of SchemaRegionGroups for each Database. +# Datatype: Integer +# default_schema_region_group_num_per_database=1 + +# Only take effect when set schema_region_group_extension_policy=AUTO. +# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode. +# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema, +# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases. +# Datatype: Double +# schema_region_per_data_node=1.0 + +# The policy of extension DataRegionGroup for each Database. +# These policies are currently supported: +# 1. CUSTOM(Each Database will allocate data_region_group_per_database DataRegionGroups as soon as created) +# 2. AUTO(Each Database will automatically extend DataRegionGroups based on the data it has) +# Datatype: String +# data_region_group_extension_policy=AUTO + +# When set data_region_group_extension_policy=CUSTOM, +# this parameter is the default number of DataRegionGroups for each Database. +# When set data_region_group_extension_policy=AUTO, +# this parameter is the default minimal number of DataRegionGroups for each Database. +# Datatype: Integer +# default_data_region_group_num_per_database=2 + +# Only take effect when set data_region_group_extension_policy=AUTO. +# This parameter is the maximum number of DataRegions expected to be managed by each DataNode. +# Notice: Since each Database requires at least two DataRegionGroups to manage its data, +# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases. +# Datatype: Double +# data_region_per_data_node=5.0 + +# Whether to enable auto leader balance for Ratis consensus protocol. +# The ConfigNode-leader will balance the leader of Ratis-RegionGroups by leader_distribution_policy if set true. +# Notice: Default is false because the Ratis is unstable for this function. +# Datatype: Boolean +# enable_auto_leader_balance_for_ratis_consensus=true + +# Whether to enable auto leader balance for IoTConsensus protocol. +# The ConfigNode-leader will balance the leader of IoTConsensus-RegionGroups by leader_distribution_policy if set true. +# Notice: Default is true because the IoTConsensus depends on this function to distribute leader. +# Datatype: Boolean +# enable_auto_leader_balance_for_iot_consensus=true + +#################### +### Cluster management +#################### + +# Time partition interval in milliseconds, and partitioning data inside each data region, default is equal to one week +# Can not be changed after the first start +# Datatype: long +# time_partition_interval=604800000 + +# The heartbeat interval in milliseconds, default is 1000ms +# Datatype: long +# heartbeat_interval_in_ms=1000 + +# Disk remaining threshold at which DataNode is set to ReadOnly status +# Datatype: double(percentage) +# disk_space_warning_threshold=0.05 + +#################### +### Memory Control Configuration +#################### + +# Memory Allocation Ratio: StorageEngine, QueryEngine, SchemaEngine, Consensus, StreamingEngine and Free Memory. +# The parameter form is a:b:c:d:e:f, where a, b, c, d, e and f are integers. for example: 1:1:1:1:1:1 , 6:2:1:1:1:1 +# If you have high level of writing pressure and low level of reading pressure, please adjust it to for example 6:1:1:1:1:1 +# datanode_memory_proportion=3:3:1:1:1:1 + +# Schema Memory Allocation Ratio: SchemaRegion, SchemaCache, and PartitionCache. +# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:2:1 +# schema_memory_proportion=5:4:1 + +# Memory allocation ratio in StorageEngine: Write, Compaction +# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 +# storage_engine_memory_proportion=8:2 + +# Memory allocation ratio in writing: Memtable, TimePartitionInfo +# Memtable is the total memory size of all memtables +# TimePartitionInfo is the total memory size of last flush time of all data regions +# write_memory_proportion=19:1 + +# primitive array size (length of each array) in array pool +# Datatype: int +# primitive_array_size=64 + +# Ratio of compaction memory for chunk metadata maintains in memory when doing compaction +# Datatype: double +# chunk_metadata_size_proportion=0.1 + +# Ratio of write memory for invoking flush disk, 0.4 by default +# If you have extremely high write load (like batch=1000), it can be set lower than the default value like 0.2 +# Datatype: double +# flush_proportion=0.4 + +# Ratio of write memory allocated for buffered arrays, 0.6 by default +# Datatype: double +# buffered_arrays_memory_proportion=0.6 + +# Ratio of write memory for rejecting insertion, 0.8 by default +# If you have extremely high write load (like batch=1000) and the physical memory size is large enough, +# it can be set higher than the default value like 0.9 +# Datatype: double +# reject_proportion=0.8 + +# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory +# Datatype: double +# device_path_cache_proportion=0.05 + +# If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001 +# Datatype: double +# write_memory_variation_report_proportion=0.001 + +# When an inserting is rejected, waiting period (in ms) to check system again, 50 by default. +# If the insertion has been rejected and the read load is low, it can be set larger. +# Datatype: int +# check_period_when_insert_blocked=50 + +# size of ioTaskQueue. The default value is 10 +# Datatype: int +# io_task_queue_size_for_flushing=10 + +# If true, we will estimate each query's possible memory footprint before executing it and deny it if its estimated memory exceeds current free memory +# Datatype: bool +# enable_query_memory_estimation=true + +#################### +### Schema Engine Configuration +#################### + +# The schema management mode of schema engine. Currently support Memory and PBTree. +# This config of all DataNodes in one cluster must keep same. +# Datatype: string +# schema_engine_mode=Memory + +# cache size for partition. +# This cache is used to improve partition fetch from config node. +# Datatype: int +# partition_cache_size=1000 + +# Size of log buffer in each metadata operation plan(in byte). +# If the size of a metadata operation plan is larger than this parameter, then it will be rejected by SchemaRegion +# If it sets a value smaller than 0, use the default value 1024*1024 +# Datatype: int +# mlog_buffer_size=1048576 + +# The cycle when metadata log is periodically forced to be written to disk(in milliseconds) +# If sync_mlog_period_in_ms=0 it means force metadata log to be written to disk after each refreshment +# Set this parameter to 0 may slow down the operation on slow disk. +# sync_mlog_period_in_ms=100 + +# interval num for tag and attribute records when force flushing to disk +# When a certain amount of tag and attribute records is reached, they will be force flushed to disk +# It is possible to lose at most tag_attribute_flush_interval records +# tag_attribute_flush_interval=1000 + +# max size for a storage block for tags and attributes of one time series. If the combined size of tags and +# attributes exceeds the tag_attribute_total_size, a new storage block will be allocated to continue storing +# the excess data. +# the unit is byte +# Datatype: int +# tag_attribute_total_size=700 + +# max measurement num of internal request +# When creating timeseries with Session.createMultiTimeseries, the user input plan, the timeseries num of +# which exceeds this num, will be split to several plans with timeseries no more than this num. +# max_measurement_num_of_internal_request=10000 + +# Policy of DataNodeSchemaCache eviction. +# Support FIFO and LRU policy. FIFO takes low cache update overhead. LRU takes high cache hit rate. +# datanode_schema_cache_eviction_policy=FIFO + +# This configuration parameter sets the maximum number of time series allowed in the cluster. +# The value should be a positive integer representing the desired threshold. +# When the threshold is reached, users will be prohibited from creating new time series. +# -1 means unlimited +# cluster_timeseries_limit_threshold=-1 + +# This configuration parameter sets the maximum number of device allowed in the cluster. +# The value should be a positive integer representing the desired threshold. +# When the threshold is reached, users will be prohibited from creating new time series. +# -1 means unlimited +# cluster_device_limit_threshold=-1 + +# This configuration parameter sets the maximum number of Cluster Databases allowed. +# The value should be a positive integer representing the desired threshold. +# When the threshold is reached, users will be prohibited from creating new databases. +# -1 means unlimited. +# database_limit_threshold = -1 + + + +#################### +### Configurations for creating schema automatically +#################### + +# Whether creating schema automatically is enabled +# If true, then create database and timeseries automatically when not exists in insertion +# Or else, user need to create database and timeseries before insertion. +# Datatype: boolean +# enable_auto_create_schema=true + +# Database level when creating schema automatically is enabled +# e.g. root.sg0.d1.s2 +# we will set root.sg0 as the database if database level is 1 +# Datatype: int +# default_storage_group_level=1 + +# ALL data types: BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT + +# register time series as which type when receiving boolean string "true" or "false" +# Datatype: TSDataType +# Options: BOOLEAN, TEXT +# boolean_string_infer_type=BOOLEAN + +# register time series as which type when receiving an integer string and using float or double may lose precision +# Datatype: TSDataType +# Options: DOUBLE, FLOAT, INT32, INT64, TEXT +# integer_string_infer_type=DOUBLE + +# register time series as which type when receiving a floating number string "6.7" +# Datatype: TSDataType +# Options: DOUBLE, FLOAT, TEXT +# floating_string_infer_type=DOUBLE + +# register time series as which type when receiving the Literal NaN. +# Datatype: TSDataType +# Options: DOUBLE, FLOAT, TEXT +# nan_string_infer_type=DOUBLE + +# BOOLEAN encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_boolean_encoding=RLE + +# INT32 encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_int32_encoding=TS_2DIFF + +# INT64 encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_int64_encoding=TS_2DIFF + +# FLOAT encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_float_encoding=GORILLA + +# DOUBLE encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_double_encoding=GORILLA + +# TEXT encoding when creating schema automatically is enabled +# Datatype: TSEncoding +# default_text_encoding=PLAIN + +#################### +### Query Configurations +#################### + +# The read consistency level +# These consistency levels are currently supported: +# 1. strong(Default, read from the leader replica) +# 2. weak(Read from a random replica) +# Datatype: string +# read_consistency_level=strong + +# Whether to cache meta data(BloomFilter, ChunkMetadata and TimeSeriesMetadata) or not. +# Datatype: boolean +# meta_data_cache_enable=true + +# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others. +# The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50 +# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50 + +# Whether to enable LAST cache +# Datatype: boolean +# enable_last_cache=true + +# Core size of ThreadPool of MPP data exchange +# Datatype: int +# mpp_data_exchange_core_pool_size=10 + +# Max size of ThreadPool of MPP data exchange +# Datatype: int +# mpp_data_exchange_max_pool_size=10 + +# Max waiting time for MPP data exchange +# Datatype: int +# mpp_data_exchange_keep_alive_time_in_ms=1000 + +# The max execution time of a DriverTask +# Datatype: int, Unit: ms +# driver_task_execution_time_slice_in_ms=200 + +# The max capacity of a TsBlock +# Datatype: int, Unit: byte +# max_tsblock_size_in_bytes=131072 + +# The max number of lines in a single TsBlock +# Datatype: int +# max_tsblock_line_number=1000 + +# Time cost(ms) threshold for slow query +# Datatype: long +# slow_query_threshold=10000 + +# The max executing time of query. unit: ms +# Datatype: int +# query_timeout_threshold=60000 + +# The maximum allowed concurrently executing queries +# Datatype: int +# max_allowed_concurrent_queries=1000 + +# How many threads can concurrently execute query statement. When <= 0, use CPU core number. +# Datatype: int +# query_thread_count=0 + +# How many pipeline drivers will be created for one fragment instance. When <= 0, use CPU core number / 2. +# Datatype: int +# degree_of_query_parallelism=0 + +# The threshold of count map size when calculating the MODE aggregation function +# Datatype: int +# mode_map_size_threshold=10000 + +# The amount of data iterate each time in server (the number of data strips, that is, the number of different timestamps.) +# Datatype: int +# batch_size=100000 + +# The memory for external sort in sort operator, when the data size is smaller than sort_buffer_size_in_bytes, the sort operator will use in-memory sort. +# Datatype: long +# sort_buffer_size_in_bytes=1048576 + +# The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged. +# Datatype: int +# merge_threshold_of_explain_analyze=10 + +#################### +### Storage Engine Configuration +#################### + +# Use this value to set timestamp precision as "ms", "us" or "ns". +# Once the precision has been set, it can not be changed. +# Datatype: String +# timestamp_precision=ms + +# When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted. +# Datatype: Boolean +# timestamp_precision_check_enabled=true + +# Default TTL for databases that are not set TTL by statements, If not set (default), the TTL will be unlimited. +# Negative value means the TTL is unlimited. +# Notice: if this property is changed, previous created database which are not set TTL will also be affected. +# Datatype: long +# Unit: ms +# default_ttl_in_ms=-1 + +# When the waiting time (in ms) of an inserting exceeds this, throw an exception. 10000 by default. +# If the insertion has been rejected and the read load is low, it can be set larger +# Datatype: int +# max_waiting_time_when_insert_blocked=10000 + +# Add a switch to enable separate sequence and unsequence data. +# If it is true, then data will be separated into seq and unseq data dir. If it is false, then all data will be written into unseq data dir. +# Datatype: boolean +# enable_separate_data=true + +# What will the system do when unrecoverable error occurs. +# Datatype: String +# Optional strategies are as follows: +# 1. CHANGE_TO_READ_ONLY: set system status to read-only and the system only accepts query operations. +# 2. SHUTDOWN: the system will be shutdown. +# handle_system_error=CHANGE_TO_READ_ONLY + +# Whether to timed flush sequence tsfiles' memtables. +# Datatype: boolean +# enable_timed_flush_seq_memtable=true + +# If a memTable's last update time is older than current time minus this, the memtable will be flushed to disk. +# Only check sequence tsfiles' memtables. +# The default flush interval is 10 * 60 * 1000. (unit: ms) +# Datatype: long +# seq_memtable_flush_interval_in_ms=600000 + +# The interval to check whether sequence memtables need flushing. +# The default flush check interval is 30 * 1000. (unit: ms) +# Datatype: long +# seq_memtable_flush_check_interval_in_ms=30000 + +# Whether to timed flush unsequence tsfiles' memtables. +# Datatype: boolean +# enable_timed_flush_unseq_memtable=true + +# If a memTable's last update time is older than current time minus this, the memtable will be flushed to disk. +# Only check unsequence tsfiles' memtables. +# The default flush interval is 10 * 60 * 1000. (unit: ms) +# Datatype: long +# unseq_memtable_flush_interval_in_ms=600000 + +# The interval to check whether unsequence memtables need flushing. +# The default flush check interval is 30 * 1000. (unit: ms) +# Datatype: long +# unseq_memtable_flush_check_interval_in_ms=30000 + +# The sort algorithms used in the memtable's TVList +# TIM: default tim sort, +# QUICK: quick sort, +# BACKWARD: backward sort +# tvlist_sort_algorithm=TIM + +# When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 100000. +# Datatype: int +# avg_series_point_number_threshold=100000 + +# How many threads can concurrently flush. When <= 0, use CPU core number. +# Datatype: int +# flush_thread_count=0 + +# In one insert (one device, one timestamp, multiple measurements), +# if enable partial insert, one measurement failure will not impact other measurements +# Datatype: boolean +# enable_partial_insert=true + +# the interval to log recover progress of each vsg when starting iotdb +# Datatype: int +# recovery_log_interval_in_ms=5000 + +# If using v0.13 client to insert data, please set this configuration to true. +# Notice: if using v0.13/v1.0 client or setting Client Version to V_0_13 manually, enable this config will disable insert redirection. +# Datatype: boolean +# 0.13_data_insert_adapt=false + +# Verify that TSfiles generated by Flush, Load, and Compaction are correct. The following is verified: +# 1. Check whether the file contains a header and a tail +# 2. Check whether files can be deserialized successfully +# 3. Check whether the file contains data +# 4. Whether there is time range overlap between data, whether it is increased, and whether the metadata index offset of the sequence is correct +# Datatype: boolean +# enable_tsfile_validation=false + +#################### +### Compaction Configurations +#################### +# sequence space compaction: only compact the sequence files +# Datatype: boolean +# enable_seq_space_compaction=true + +# unsequence space compaction: only compact the unsequence files +# Datatype: boolean +# enable_unseq_space_compaction=true + +# cross space compaction: compact the unsequence files into the overlapped sequence files +# Datatype: boolean +# enable_cross_space_compaction=true + +# the selector of cross space compaction task +# Options: rewrite +# cross_selector=rewrite + +# the compaction performer of cross space compaction task +# Options: read_point, fast +# cross_performer=fast + +# the selector of inner sequence space compaction task +# Options: size_tiered +# inner_seq_selector=size_tiered + +# the performer of inner sequence space compaction task +# Options: read_chunk, fast +# inner_seq_performer=read_chunk + +# the selector of inner unsequence space compaction task +# Options: size_tiered +# inner_unseq_selector=size_tiered + +# the performer of inner unsequence space compaction task +# Options: read_point, fast +# inner_unseq_performer=fast + +# The priority of compaction execution +# INNER_CROSS: prioritize inner space compaction, reduce the number of files first +# CROSS_INNER: prioritize cross space compaction, eliminate the unsequence files first +# BALANCE: alternate two compaction types +# compaction_priority=BALANCE + +# The size of candidate compaction task queue. +# Datatype: int +# candidate_compaction_task_queue_size = 200 + +# This parameter is used in two places: +# 1. The target tsfile size of inner space compaction. +# 2. The candidate size of seq tsfile in cross space compaction will be smaller than target_compaction_file_size * 1.5. +# In most cases, the target file size of cross compaction won't exceed this threshold, and if it does, it will not be much larger than it. +# default is 2GB +# Datatype: long, Unit: byte +# target_compaction_file_size=2147483648 + +# The target chunk size in compaction and when memtable reaches this threshold, flush the memtable to disk. +# default is 1MB +# Datatype: long, Unit: byte +# target_chunk_size=1048576 + +# The target point nums in one chunk in compaction +# Datatype: long +# target_chunk_point_num=100000 + +# If the chunk size is lower than this threshold, it will be deserialize into points, default is 128 byte +# Datatype: long, Unit:byte +# chunk_size_lower_bound_in_compaction=128 + +# If the chunk point num is lower than this threshold, it will be deserialize into points +# Datatype: long +# chunk_point_num_lower_bound_in_compaction=100 + +# The max file when selecting inner space compaction candidate files +# Datatype: int +# max_inner_compaction_candidate_file_num=30 + +# The max file when selecting cross space compaction candidate files +# At least one unseq file with it's overlapped seq files will be selected even exceeded this number +# Datatype: int +# max_cross_compaction_candidate_file_num=500 + +# The max total size when selecting cross space compaction candidate files +# At least one unseq file with it's overlapped seq files will be selected even exceeded this number +# Datatype: long, Unit: byte +# max_cross_compaction_candidate_file_size=5368709120 + +# The min inner compaction level of unsequence file which can be selected as candidate +# Datatype: int +# min_cross_compaction_unseq_file_level=1 + +# If one merge file selection runs for more than this time, it will be ended and its current +# selection will be used as final selection. +# When < 0, it means time is unbounded. +# Datatype: long, Unit: ms +# cross_compaction_file_selection_time_budget=30000 + +# How many threads will be set up to perform compaction, 10 by default. +# Set to 1 when less than or equal to 0. +# Datatype: int +# compaction_thread_count=10 + +# The interval of compaction task schedule +# Datatype: long, Unit: ms +# compaction_schedule_interval_in_ms=60000 + +# The interval of compaction task submission +# Datatype: long, Unit: ms +# compaction_submission_interval_in_ms=60000 + +# The limit of write throughput merge can reach per second +# values less than or equal to 0 means no limit +# Datatype: int, Unit: megabyte +# compaction_write_throughput_mb_per_sec=16 + +# The limit of read throughput merge can reach per second +# values less than or equal to 0 means no limit +# Datatype: int, Unit: megabyte +# compaction_read_throughput_mb_per_sec=0 + +# The limit of read operation merge can reach per second +# values less than or equal to 0 means no limit +# Datatype: int +# compaction_read_operation_per_sec=0 + +# The number of sub compaction threads to be set up to perform compaction. +# Currently only works for nonAligned data in cross space compaction and unseq inner space compaction. +# Set to 1 when less than or equal to 0. +# Datatype: int +# sub_compaction_thread_count=4 + +# Redundancy value of disk availability, only use for inner compaction. +# When disk availability is lower than the sum of (disk_space_warning_threshold + inner_compaction_task_selection_disk_redundancy), inner compaction tasks containing mods files are selected first. +# DataType: double +# inner_compaction_task_selection_disk_redundancy=0.05 + +# Mods file size threshold, only use for inner compaction. +# When the size of the mods file corresponding to TsFile exceeds this value, inner compaction tasks containing mods files are selected first. +# DataType: long +# inner_compaction_task_selection_mods_file_threshold=10485760 + +# The number of threads to be set up to select compaction task. +# Datatype: int +# compaction_schedule_thread_num=4 + +#################### +### Write Ahead Log Configuration +#################### + +# Write mode of wal +# The details of these three modes are as follows: +# 1. DISABLE: the system will disable wal. +# 2. SYNC: the system will submit wal synchronously, write request will not return until its wal is fsynced to the disk successfully. +# 3. ASYNC: the system will submit wal asynchronously, write request will return immediately no matter its wal is fsynced to the disk successfully. +# The write performance order is DISABLE > ASYNC > SYNC, but only SYNC mode can ensure data durability. +# wal_mode=ASYNC + +# Max number of wal nodes, each node corresponds to one wal directory +# This parameter is only valid in the standalone mode. IoTConsensus uses one wal per data region and RatisConsensus doesn't use wal. +# The default value 0 means the number is determined by the system, the number is in the range of [data region num / 2, data region num]. +# Notice: this value affects write performance significantly. +# For non-SSD disks, values between one third and half of databases number are recommended. +# Datatype: int +# max_wal_nodes_num=0 + +# Duration a wal flush operation will wait before calling fsync in the async mode +# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. +# Notice: this value affects write performance significantly, values in the range of 10ms-2000ms are recommended. +# Datatype: long +# wal_async_mode_fsync_delay_in_ms=1000 + +# Duration a wal flush operation will wait before calling fsync in the sync mode +# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. +# Notice: this value affects write performance significantly, values in the range of 0ms-10ms are recommended. +# Datatype: long +# wal_sync_mode_fsync_delay_in_ms=3 + +# Buffer size of each wal node +# If it's a value smaller than 0, use the default value 32 * 1024 * 1024 bytes (32MB). +# Datatype: int +# wal_buffer_size_in_byte=33554432 + +# Blocking queue capacity of each wal buffer, restricts maximum number of WALEdits cached in the blocking queue. +# Datatype: int +# wal_buffer_queue_capacity=500 + +# Size threshold of each wal file +# When a wal file's size exceeds this, the wal file will be closed and a new wal file will be created. +# If it's a value smaller than 0, use the default value 30 * 1024 * 1024 (30MB). +# Datatype: long +# wal_file_size_threshold_in_byte=31457280 + +# Minimum ratio of effective information in wal files +# This value should be between 0.0 and 1.0 +# If effective information ratio is below this value, MemTable snapshot or flush will be triggered. +# Increase this value when wal occupies too much disk space. But, if this parameter is too large, the write performance may decline. +# Datatype: double +# wal_min_effective_info_ratio=0.1 + +# MemTable size threshold for triggering MemTable snapshot in wal +# When a memTable's size (in byte) exceeds this, wal can flush this memtable to disk, otherwise wal will snapshot this memtable in wal. +# If it's a value smaller than 0, use the default value 8 * 1024 * 1024 bytes (8MB). +# Datatype: long +# wal_memtable_snapshot_threshold_in_byte=8388608 + +# MemTable's max snapshot number in wal +# If one memTable's snapshot number in wal exceeds this value, it will be flushed to disk. +# Datatype: int +# max_wal_memtable_snapshot_num=1 + +# The period when outdated wal files are periodically deleted +# If this value is too large, outdated wal files may not able to be deleted in time. +# If it's a value smaller than 0, use the default value 20 * 1000 ms (20 seconds). +# Datatype: long +# delete_wal_files_period_in_ms=20000 + +# The minimum size of wal files when throttle down in IoTConsensus +# If this value is not set, it will be carefully chosen according to the available disk space. +# If this value is set smaller than 0, it will default to 50 * 1024 * 1024 * 1024 bytes (50GB). +# Datatype: long +# iot_consensus_throttle_threshold_in_byte=53687091200 + +# Maximum wait time of write cache in IoTConsensus +# If this value is less than or equal to 0, use the default value 10 * 1000 ms (10s) +# Datatype: long +# iot_consensus_cache_window_time_in_ms=-1 + +#################### +### IoTConsensus Configuration +#################### + +# The maximum log entries num in IoTConsensus Batch +# Datatype: int +# data_region_iot_max_log_entries_num_per_batch = 1024 + +# The maximum size in IoTConsensus Batch +# Datatype: int +# data_region_iot_max_size_per_batch = 16777216 + +# The maximum pending batches num in IoTConsensus +# Datatype: int +# data_region_iot_max_pending_batches_num = 5 + +# The maximum memory ratio for queue in IoTConsensus +# Datatype: double +# data_region_iot_max_memory_ratio_for_queue = 0.6 + +# The maximum transit size in byte per second for region migration +# values less than or equal to 0 means no limit +# Datatype: long +# region_migration_speed_limit_bytes_per_second = 33554432 + +#################### +### TsFile Configurations +#################### + +# Datatype: int +# group_size_in_byte=134217728 + +# The memory size for each series writer to pack page, default value is 64KB +# Datatype: int +# page_size_in_byte=65536 + +# The maximum number of data points in a page, default 10000 +# Datatype: int +# max_number_of_points_in_page=10000 + +# The threshold for pattern matching in regex +# Datatype: int +# pattern_matching_threshold=1000000 + +# Floating-point precision +# Datatype: int +# float_precision=2 + +# Encoder of value series. default value is PLAIN. +# For int, long data type, also supports TS_2DIFF and RLE(run-length encoding), GORILLA and ZIGZAG. +# value_encoder=PLAIN + +# Compression configuration +# Data compression method, supports UNCOMPRESSED, SNAPPY, ZSTD, LZMA2 or LZ4. Default value is LZ4 +# And it is also used as the default compressor of time column in aligned timeseries. +# compressor=LZ4 + +#################### +### Authorization Configuration +#################### + +# which class to serve for authorization. By default, it is LocalFileAuthorizer. +# Another choice is org.apache.iotdb.commons.auth.authorizer.OpenIdAuthorizer +# authorizer_provider_class=org.apache.iotdb.commons.auth.authorizer.LocalFileAuthorizer + +# If OpenIdAuthorizer is enabled, then openID_url must be set. +# openID_url= + +# encryption provider class +# iotdb_server_encrypt_decrypt_provider=org.apache.iotdb.commons.security.encrypt.MessageDigestEncrypt + +# encryption provided class parameter +# iotdb_server_encrypt_decrypt_provider_parameter= + +# Cache size of user and role +# Datatype: int +# author_cache_size=1000 + +# Cache expire time of user and role +# Datatype: int +# author_cache_expire_time=30 + +#################### +### UDF Configuration +#################### + +# Used to estimate the memory usage of text fields in a UDF query. +# It is recommended to set this value to be slightly larger than the average length of all text +# records. +# Datatype: int +# udf_initial_byte_array_length_for_memory_control=48 + +# How much memory may be used in ONE UDF query (in MB). +# The upper limit is 20% of allocated memory for read. +# Datatype: float +# udf_memory_budget_in_mb=30.0 + +# UDF memory allocation ratio. +# The parameter form is a:b:c, where a, b, and c are integers. +# udf_reader_transformer_collector_memory_proportion=1:1:1 + +# UDF lib dir +# If this property is unset, system will save the data in the default relative path directory under +# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/udf). +# +# If it is absolute, system will save the data in exact location it points to. +# If it is relative, system will save the data in the relative path directory it indicates under the +# UDF folder. +# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative +# path. +# +# For Windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is +# absolute. Otherwise, it is relative. +# udf_lib_dir=ext\\udf +# +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# udf_lib_dir=ext/udf + +#################### +### Trigger Configuration +#################### + +# Uncomment the following field to configure the trigger lib directory. +# For Windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is +# absolute. Otherwise, it is relative. +# trigger_lib_dir=ext\\trigger +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# trigger_lib_dir=ext/trigger + +# How many times will we retry to found an instance of stateful trigger on DataNodes +# 3 by default. +# Datatype: int +# stateful_trigger_retry_num_when_not_found=3 + + +#################### +### Select-Into Configuration +#################### + +# The maximum memory occupied by the data to be written when executing select-into statements. +# Datatype: long +# into_operation_buffer_size_in_byte=104857600 + +# The maximum number of rows can be processed in insert-tablet-plan when executing select-into statements. +# When <= 0, use 10000. +# Datatype: int +# select_into_insert_tablet_plan_row_limit=10000 + +# The number of threads in the thread pool that execute insert-tablet tasks +# Datatype: int +# into_operation_execution_thread_count=2 + +#################### +### Continuous Query Configuration +#################### + +# The number of threads in the scheduled thread pool that submit continuous query tasks periodically +# Datatype: int +# continuous_query_submit_thread_count=2 + +# The minimum value of the continuous query execution time interval +# Datatype: long(duration) +# continuous_query_min_every_interval_in_ms=1000 + +#################### +### Pipe Configuration +#################### + +# Uncomment the following field to configure the pipe lib directory. +# For Windows platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is +# absolute. Otherwise, it is relative. +# pipe_lib_dir=ext\\pipe +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# pipe_lib_dir=ext/pipe + +# The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. +# The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). +# pipe_subtask_executor_max_thread_num=5 + +# The connection timeout (in milliseconds) for the thrift client. +# pipe_sink_timeout_ms=900000 + +# The maximum number of selectors that can be used in the sink. +# Recommend to set this value to less than or equal to pipe_sink_max_client_number. +# pipe_sink_selector_number=4 + +# The maximum number of clients that can be used in the sink. +# pipe_sink_max_client_number=16 + +# Whether to enable receiving pipe data through air gap. +# The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. +# pipe_air_gap_receiver_enabled=false + +# The port for the server to receive pipe data through air gap. +# pipe_air_gap_receiver_port=9780 + +#################### +### RatisConsensus Configuration +#################### + +# max payload size for a single log-sync-RPC from leader to follower(in byte, by default 16MB) +# config_node_ratis_log_appender_buffer_size_max=16777216 +# schema_region_ratis_log_appender_buffer_size_max=16777216 +# data_region_ratis_log_appender_buffer_size_max=16777216 + +# trigger a snapshot when snapshot_trigger_threshold logs are written +# config_node_ratis_snapshot_trigger_threshold=400000 +# schema_region_ratis_snapshot_trigger_threshold=400000 +# data_region_ratis_snapshot_trigger_threshold=400000 + +# allow flushing Raft Log asynchronously +# config_node_ratis_log_unsafe_flush_enable=false +# schema_region_ratis_log_unsafe_flush_enable=false +# data_region_ratis_log_unsafe_flush_enable=false + +# max capacity of a single Log segment file (in byte, by default 24MB) +# config_node_ratis_log_segment_size_max_in_byte=25165824 +# schema_region_ratis_log_segment_size_max_in_byte=25165824 +# data_region_ratis_log_segment_size_max_in_byte=25165824 +# config_node_simple_consensus_log_segment_size_max_in_byte=25165824 + +# flow control window for ratis grpc log appender +# config_node_ratis_grpc_flow_control_window=4194304 +# schema_region_ratis_grpc_flow_control_window=4194304 +# data_region_ratis_grpc_flow_control_window=4194304 +# config_node_ratis_grpc_leader_outstanding_appends_max=128 +# schema_region_ratis_grpc_leader_outstanding_appends_max=128 +# data_region_ratis_grpc_leader_outstanding_appends_max=128 +# config_node_ratis_log_force_sync_num=128 +# schema_region_ratis_log_force_sync_num=128 +# data_region_ratis_log_force_sync_num=128 + +# min election timeout for leader election +# config_node_ratis_rpc_leader_election_timeout_min_ms=2000 +# schema_region_ratis_rpc_leader_election_timeout_min_ms=2000 +# data_region_ratis_rpc_leader_election_timeout_min_ms=2000 + +# max election timeout for leader election +# config_node_ratis_rpc_leader_election_timeout_max_ms=4000 +# schema_region_ratis_rpc_leader_election_timeout_max_ms=4000 +# data_region_ratis_rpc_leader_election_timeout_max_ms=4000 + +# ratis client retry threshold +# config_node_ratis_request_timeout_ms=10000 +# schema_region_ratis_request_timeout_ms=10000 +# data_region_ratis_request_timeout_ms=10000 + +# currently we use exponential back-off retry policy for ratis +# config_node_ratis_max_retry_attempts=10 +# config_node_ratis_initial_sleep_time_ms=100 +# config_node_ratis_max_sleep_time_ms=10000 +# schema_region_ratis_max_retry_attempts=10 +# schema_region_ratis_initial_sleep_time_ms=100 +# schema_region_ratis_max_sleep_time_ms=10000 +# data_region_ratis_max_retry_attempts=10 +# data_region_ratis_initial_sleep_time_ms=100 +# data_region_ratis_max_sleep_time_ms=10000 + +# first election timeout +# ratis_first_election_timeout_min_ms=50 +# ratis_first_election_timeout_max_ms=150 + +# preserve certain logs when take snapshot and purge +# config_node_ratis_preserve_logs_num_when_purge=1000 +# schema_region_ratis_preserve_logs_num_when_purge=1000 +# data_region_ratis_preserve_logs_num_when_purge=1000 + +# Raft Log disk size control +# config_node_ratis_log_max_size = 2147483648 +# schema_region_ratis_log_max_size = 2147483648 +# data_region_ratis_log_max_size = 21474836480 + +# Raft periodic snapshot interval, time unit is second +# config_node_ratis_periodic_snapshot_interval=86400 +# schema_region_ratis_periodic_snapshot_interval=86400 +# data_region_ratis_periodic_snapshot_interval=86400 + +#################### +### Procedure Configuration +#################### + +# Default number of worker thread count +# Datatype: int +# procedure_core_worker_thread_count=4 + +# Default time interval of completed procedure cleaner work in, time unit is second +# Datatype: int +# procedure_completed_clean_interval=30 + +# Default ttl of completed procedure, time unit is second +# Datatype: int +# procedure_completed_evict_ttl=800 + +#################### +### MQTT Broker Configuration +#################### + +# whether to enable the mqtt service. +# Datatype: boolean +# enable_mqtt_service=false + +# the mqtt service binding host. +# Datatype: String +# mqtt_host=127.0.0.1 + +# the mqtt service binding port. +# Datatype: int +# mqtt_port=1883 + +# the handler pool size for handing the mqtt messages. +# Datatype: int +# mqtt_handler_pool_size=1 + +# the mqtt message payload formatter. +# Datatype: String +# mqtt_payload_formatter=json + +# max length of mqtt message in byte +# Datatype: int +# mqtt_max_message_size=1048576 + +#################### +### IoTDB-ML Configuration +#################### + +# The thread count which can be used for model inference operation. +# model_inference_execution_thread_count=5 + +#################### +### Load TsFile Configuration +#################### + +# Load clean up task is used to clean up the unsuccessful loaded tsfile after a certain period of time. +# The parameter is the delay time after an unsuccessful load operation (in seconds). +# load_clean_up_task_execution_delay_time_seconds=1800 diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 86014cd7b6c60..ee42c0ca32ffc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -36,7 +36,10 @@ public class CommonConfig { - public static final String CONFIG_NAME = "iotdb-common.properties"; + public static final String CONFIG_NODE_CONFIG_NAME = "iotdb-confignode.properties"; + public static final String DATA_NODE_CONFIG_NAME = "iotdb-datanode.properties"; + public static final String COMMON_CONFIG_NAME = "iotdb-common.properties"; + public static final String SYSTEM_CONFIG_NAME = "iotdb-system.properties"; private static final Logger logger = LoggerFactory.getLogger(CommonConfig.class); // Open ID Secret diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java new file mode 100644 index 0000000000000..ffd1038385e6c --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.conf; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URL; +import java.nio.file.Files; +import java.util.concurrent.TimeUnit; + +public class ConfigFileAutoUpdateTool { + + private final String lockFileSuffix = ".lock"; + private String license = + "#\n" + + "# Licensed to the Apache Software Foundation (ASF) under one\n" + + "# or more contributor license agreements. See the NOTICE file\n" + + "# distributed with this work for additional information\n" + + "# regarding copyright ownership. The ASF licenses this file\n" + + "# to you under the Apache License, Version 2.0 (the\n" + + "# \"License\"); you may not use this file except in compliance\n" + + "# with the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing,\n" + + "# software distributed under the License is distributed on an\n" + + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "# KIND, either express or implied. See the License for the\n" + + "# specific language governing permissions and limitations\n" + + "# under the License."; + + public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) + throws IOException, InterruptedException { + if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { + return; + } + File systemFile = new File(systemUrl.getFile()); + File configNodeFile = new File(configNodeUrl.getFile()); + File dataNodeFile = new File(dataNodeUrl.getFile()); + File commonFile = new File(commonUrl.getFile()); + + if (!systemFile.exists()) { + if (!configNodeFile.exists() || !dataNodeFile.exists() || !commonFile.exists()) { + return; + } + systemFile.createNewFile(); + } + + acquireTargetFileLock(systemFile); + try { + // other progress updated this file + if (systemFile.length() != 0) { + return; + } + try (RandomAccessFile raf = new RandomAccessFile(systemFile, "rw")) { + raf.write(license.getBytes()); + String configNodeContent = readConfigLines(configNodeFile); + raf.write(configNodeContent.getBytes()); + String dataNodeContent = readConfigLines(dataNodeFile); + raf.write(dataNodeContent.getBytes()); + String commonContent = readConfigLines(commonFile); + raf.write(commonContent.getBytes()); + } + } finally { + releaseFileLock(systemFile); + } + } + + private String readConfigLines(File file) throws IOException { + byte[] bytes = Files.readAllBytes(file.toPath()); + String content = new String(bytes); + return content.replace(license, ""); + } + + private void acquireTargetFileLock(File file) throws IOException, InterruptedException { + long waitTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(20); + File lockFile = new File(file.getPath() + lockFileSuffix); + while (System.currentTimeMillis() < waitTime) { + if (lockFile.createNewFile()) { + return; + } + Thread.sleep(TimeUnit.MICROSECONDS.toMillis(100)); + } + } + + private void releaseFileLock(File file) throws IOException { + Files.deleteIfExists(new File(file.getPath() + lockFileSuffix).toPath()); + } +} From 2dab0a2ff4a5338aed2bdb8b824768808aa4f354 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 May 2024 16:21:46 +0800 Subject: [PATCH 02/45] modify loadHotModifiedProps --- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 856d73cd9ac77..309825006a914 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1720,30 +1720,21 @@ public void loadHotModifiedProps(Properties properties) throws QueryProcessExcep } public void loadHotModifiedProps() throws QueryProcessException { - Properties commonProperties = new Properties(); - URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); - if (url != null && new File(url.getFile()).exists()) { - loadHotModifiedPropertiesFromUrl(commonProperties, url); - } else { - url = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); - if (url == null) { - LOGGER.warn("Couldn't load the configuration from any of the known sources."); - return; - } - loadHotModifiedPropertiesFromUrl(commonProperties, url); - - url = getPropsUrl(IoTDBConfig.CONFIG_NAME); - if (url == null) { - LOGGER.warn("Couldn't load the configuration from any of the known sources."); - return; - } - loadHotModifiedPropertiesFromUrl(commonProperties, url); + if (url == null) { + LOGGER.warn("Couldn't load the configuration from any of the known sources."); + return; } - try { + + Properties commonProperties = new Properties(); + try (InputStream inputStream = url.openStream()) { + LOGGER.info("Start to reload config file {}", url); + commonProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); loadHotModifiedProps(commonProperties); } catch (Exception e) { - LOGGER.error("Failed to reload configs because {}", e.getMessage()); + LOGGER.warn("Fail to reload config file {}", url, e); + throw new QueryProcessException( + String.format("Fail to reload config file %s because %s", url, e.getMessage())); } ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(commonProperties); LOGGER.info("Reload metric service in level {}", reloadLevel); From da41d83dc15eb058e248ea7ef0b83ad22dc819c4 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 May 2024 17:06:55 +0800 Subject: [PATCH 03/45] check old version config files exist --- .../apache/iotdb/tool/IoTDBDataBackTool.java | 20 +++++++++---------- .../conf/ConfigFileAutoUpdateTool.java | 8 +++++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index 7d3315d1f67d6..e5f76e1b7f9f2 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tool; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -162,10 +163,10 @@ public static void main(String[] args) { System.setProperty("IOTDB_HOME", System.getenv("IOTDB_HOME")); argsParse(args); File sourceDir = new File(sourcePath); - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); - initDataNodeProperties(dataProperties); - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); - initConfigNodeProperties(configProperties); + + Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); + initDataNodeProperties(properties); + initConfigNodeProperties(properties); StringBuilder targetDirString = new StringBuilder(); Map copyMap = new HashMap<>(); @@ -429,10 +430,9 @@ private static void ioTDBDataBack( private static void countNodeBack(String targetDirString, Map copyMap) { File sourceDir = new File(sourcePath); - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); - initDataNodeProperties(dataProperties); - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); - initConfigNodeProperties(configProperties); + Properties properties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); + initDataNodeProperties(properties); + initConfigNodeProperties(properties); copyMap.put( sourceDir.getAbsolutePath() + File.separatorChar + ".env", @@ -480,7 +480,7 @@ private static void countDataNodeFile( Map copyMap, Map dnDataDirsMap, Map dnMapProperties) { - Properties dataProperties = getProperties(IoTDBConfig.CONFIG_NAME); + Properties dataProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); initDataNodeProperties(dataProperties); String dnSystemDir = dataProperties.getProperty("dn_system_dir"); @@ -553,7 +553,7 @@ private static void countDataNodeFile( private static void countConfigNodeFile( String targetDirString, Map copyMap, Map cnMapProperties) { - Properties configProperties = getProperties(CONFIG_NODE_CONF_NAME); + Properties configProperties = getProperties(CommonConfig.SYSTEM_CONFIG_NAME); initConfigNodeProperties(configProperties); String bakCnSystemDir = targetDirString + File.separatorChar + DEFAULT_CN_SYSTEM_DIR; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java index ffd1038385e6c..ee93362c457b2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java @@ -58,10 +58,12 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, File dataNodeFile = new File(dataNodeUrl.getFile()); File commonFile = new File(commonUrl.getFile()); + boolean canUpdate = configNodeFile.exists() && dataNodeFile.exists() && commonFile.exists(); + if (!canUpdate) { + return; + } + if (!systemFile.exists()) { - if (!configNodeFile.exists() || !dataNodeFile.exists() || !commonFile.exists()) { - return; - } systemFile.createNewFile(); } From 6b1e85dd3f2ea7bdfa98559291a2cb867c563d53 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 May 2024 18:20:22 +0800 Subject: [PATCH 04/45] modify docker --- docker/src/main/DockerCompose/entrypoint.sh | 2 +- .../DockerCompose/replace-conf-from-env.sh | 13 +---------- docker/src/main/Dockerfile-1c1d | 14 +++++------ .../manager/node/ClusterNodeStartUtils.java | 23 +++++-------------- 4 files changed, 15 insertions(+), 37 deletions(-) diff --git a/docker/src/main/DockerCompose/entrypoint.sh b/docker/src/main/DockerCompose/entrypoint.sh index 55be9899b003c..72beea010eeb2 100755 --- a/docker/src/main/DockerCompose/entrypoint.sh +++ b/docker/src/main/DockerCompose/entrypoint.sh @@ -36,7 +36,7 @@ function on_stop(){ trap 'on_stop' SIGTERM SIGKILL SIGQUIT -replace-conf-from-env.sh ${start_what} +replace-conf-from-env.sh case "$1" in datanode) diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh b/docker/src/main/DockerCompose/replace-conf-from-env.sh index 93be28bc24250..887faf1bdcd63 100755 --- a/docker/src/main/DockerCompose/replace-conf-from-env.sh +++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh @@ -19,6 +19,7 @@ # conf_path=${IOTDB_HOME}/conf +target_files="iotdb-system.properties" function process_single(){ local key_value="$1" @@ -48,17 +49,5 @@ function replace_configs(){ done } -case "$1" in - confignode) - target_files="iotdb-common.properties iotdb-confignode.properties" - ;; - datanode) - target_files="iotdb-common.properties iotdb-datanode.properties" - ;; - all) - target_files="iotdb-common.properties iotdb-confignode.properties iotdb-datanode.properties" - ;; -esac - replace_configs diff --git a/docker/src/main/Dockerfile-1c1d b/docker/src/main/Dockerfile-1c1d index 9aacd0aa0ca96..71548de08b787 100644 --- a/docker/src/main/Dockerfile-1c1d +++ b/docker/src/main/Dockerfile-1c1d @@ -30,13 +30,13 @@ RUN apt update \ && rm /apache-iotdb-*-bin.zip \ && mv /apache-iotdb-* /iotdb \ && mv /start-1c1d.sh /iotdb/sbin \ - && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-datanode.properties \ - && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-confignode.properties \ - && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-confignode.properties \ - && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-datanode.properties \ - && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ - && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ - && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-common.properties \ + && sed -i 's/dn_internal_address=127.0.0.1/dn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/cn_internal_address=127.0.0.1/cn_internal_address=0.0.0.0/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/cn_seed_config_node=127.0.0.1:10710/cn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/dn_seed_config_node=127.0.0.1:10710/dn_seed_config_node=0.0.0.0:10710/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/config_node_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus/schema_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ + && sed -i 's/# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus/data_region_consensus_protocol_class=org.apache.iotdb.consensus.simple.SimpleConsensus/g' /iotdb/conf/iotdb-system.properties \ && apt remove unzip -y \ && apt autoremove -y \ && apt purge --auto-remove -y \ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java index 697dd65525d9a..13cd42393fab2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java @@ -25,8 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeType; -import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.confignode.conf.ConfigNodeConstant; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.rpc.TSStatusCode; @@ -56,11 +55,6 @@ private ClusterNodeStartUtils() { public static TSStatus confirmNodeRegistration( NodeType nodeType, String clusterName, Object nodeLocation, ConfigManager configManager) { - - final String CONF_FILE_NAME = - NodeType.ConfigNode.equals(nodeType) - ? ConfigNodeConstant.CONF_FILE_NAME - : IoTDBConstant.DATA_NODE_CONF_FILE_NAME; TSStatus status = new TSStatus(); /* Reject start if the cluster name is error */ @@ -77,8 +71,8 @@ public static TSStatus confirmNodeRegistration( nodeType.getNodeType(), clusterName, CLUSTER_NAME, - CONF_FILE_NAME, - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME, + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } @@ -116,7 +110,7 @@ public static TSStatus confirmNodeRegistration( nodeType.getNodeType(), conflictEndPoints, nodeType.getNodeType(), - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } else { /* Accept registration if all TEndPoints aren't conflict */ @@ -130,11 +124,6 @@ public static TSStatus confirmNodeRestart( int nodeId, Object nodeLocation, ConfigManager configManager) { - - final String CONF_FILE_NAME = - NodeType.ConfigNode.equals(nodeType) - ? ConfigNodeConstant.CONF_FILE_NAME - : IoTDBConstant.DATA_NODE_CONF_FILE_NAME; TSStatus status = new TSStatus(); /* Reject restart if the cluster name is error */ @@ -151,8 +140,8 @@ public static TSStatus confirmNodeRestart( nodeType.getNodeType(), clusterName, CLUSTER_NAME, - CONF_FILE_NAME, - CONF_FILE_NAME)); + CommonConfig.SYSTEM_CONFIG_NAME, + CommonConfig.SYSTEM_CONFIG_NAME)); return status; } From 0a2b4b9892a952f9a14b3331cbf07fd5f46942bc Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 20 May 2024 15:32:20 +0800 Subject: [PATCH 05/45] modify scripts --- .../apache/iotdb/tool/IoTDBDataBackTool.java | 19 ++--- .../resources/sbin/start-confignode.bat | 30 ++++---- .../resources/sbin/stop-confignode.bat | 10 ++- .../resources/sbin/stop-confignode.sh | 8 +- .../confignode/conf/ConfigNodeConstant.java | 1 - .../confignode/conf/ConfigNodeDescriptor.java | 6 +- .../iotdb/confignode/service/ConfigNode.java | 3 +- .../resources/sbin/start-datanode.bat | 48 +++++------- .../assembly/resources/sbin/stop-datanode.bat | 10 ++- .../assembly/resources/sbin/stop-datanode.sh | 7 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +- .../db/conf/rest/IoTDBRestServiceConfig.java | 2 - .../schemaregion/tag/TagLogFile.java | 2 +- .../resources/sbin/destroy-confignode.bat | 6 +- .../resources/sbin/destroy-confignode.sh | 7 +- .../resources/sbin/destroy-datanode.bat | 6 +- .../resources/sbin/destroy-datanode.sh | 8 +- .../assembly/resources/sbin/health_check.bat | 76 +++++++++---------- .../assembly/resources/sbin/health_check.sh | 7 +- .../assembly/resources/sbin/iotdb-common.sh | 26 ++++++- .../iotdb/commons/conf/CommonConfig.java | 6 +- 21 files changed, 172 insertions(+), 122 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java index e5f76e1b7f9f2..44530aa9cc97c 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; @@ -62,8 +61,6 @@ public class IoTDBDataBackTool { static String targetDataDirParam = ""; static String targetWalDirParam = ""; - static String DATA_NODE_CONF_NAME = IoTDBConfig.CONFIG_NAME; - static String CONFIG_NODE_CONF_NAME = "iotdb-confignode.properties"; static AtomicInteger fileCount = new AtomicInteger(0); static AtomicInteger targetFileCount = new AtomicInteger(0); static AtomicInteger processFileCount = new AtomicInteger(0); @@ -201,7 +198,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { countDataNodeFile(targetDirString.toString(), copyMap, dnDataDirsMap, dnMapProperties); @@ -223,7 +220,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) { countConfigNodeFile(targetDirString.toString(), copyMap, cnMapProperties); @@ -246,14 +243,14 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); propertiesFileUpdate( targetDirString.toString() + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } @@ -279,7 +276,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("datanode")) { countNodeBack(targetDirString.toString(), copyMap); @@ -299,7 +296,7 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } else if (nodeTypeParam.equalsIgnoreCase("all") || nodeTypeParam.isEmpty()) { countNodeBack(targetDirString.toString(), copyMap); @@ -321,14 +318,14 @@ public static void main(String[] args) { + File.separatorChar + "conf" + File.separatorChar - + CONFIG_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, cnMapProperties); propertiesFileUpdate( targetDirString.toString() + File.separatorChar + "conf" + File.separatorChar - + DATA_NODE_CONF_NAME, + + CommonConfig.SYSTEM_CONFIG_NAME, dnMapProperties); } } diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat index 532c799330511..f73f13d21c003 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat @@ -83,26 +83,30 @@ IF EXIST "%CONFIGNODE_CONF%\confignode-env.bat" ( ) @REM CHECK THE PORT USAGES -IF EXIST "%CONFIGNODE_CONF%\iotdb-confignode.properties" ( +@REM SET CONFIG FILE +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-confignode.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-confignode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-confignode.properties" +) ELSE ( + set CONFIG_FILE= +) + +IF DEFINED CONFIG_FILE ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - "%CONFIGNODE_CONF%\iotdb-confignode.properties"') do ( + "%CONFIG_FILE%"') do ( set cn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - "%CONFIGNODE_CONF%\iotdb-confignode.properties"') do ( + "%CONFIG_FILE%"') do ( set cn_consensus_port=%%i ) -) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties"') do ( - set cn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties"') do ( - set cn_consensus_port=%%i - ) ) ELSE ( - echo "Can't find iotdb-confignode.properties, check the default ports" + echo "Can't find iotdb-system.properties or iotdb-confignode.properties, check the default ports" set cn_internal_port=10710 set cn_consensus_port=10720 ) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index c82a095ec1fab..f7db1e156d366 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -22,15 +22,21 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ +IF EXIST "%superior%\conf\iotdb-system.properties" ( + set config_file="%superior_dir%\iotdb-system.properties" +) ELSE ( + set config_file="%superior_dir%\iotdb-confignode.properties" +) + for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" -"%superior_dir%\conf\iotdb-confignode.properties"') do ( +"%config_file%"') do ( set cn_internal_port=%%i ) echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" -"%superior_dir%\conf\iotdb-confignode.properties"') do ( +"%config_file%"') do ( set cn_internal_address=%%i ) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh index c9467faec55f9..b44d35fb03f8a 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh @@ -19,7 +19,13 @@ # CONFIGNODE_CONF="$(dirname "$0")/../conf" -cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) + +if [ -f "${CONFIG_DIR}/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) +else + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) +fi + echo Check whether the internal_port is used..., port is "$cn_internal_port" diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java index ca521e3a73c91..5724eb1862fe9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java @@ -25,7 +25,6 @@ public class ConfigNodeConstant { public static final String CONFIGNODE_CONF = "CONFIGNODE_CONF"; public static final String CONFIGNODE_HOME = "CONFIGNODE_HOME"; - public static final String CONF_FILE_NAME = "iotdb-confignode.properties"; public static final String SYSTEM_FILE_NAME = "confignode-system.properties"; public static final String CONFIGNODE_PACKAGE = "org.apache.iotdb.confignode.service"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index cea89efb02bb5..db777cfa26d73 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -57,9 +57,9 @@ public class ConfigNodeDescriptor { static { ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); - URL configNodeUrl = getPropsUrl(CommonConfig.CONFIG_NODE_CONFIG_NAME); - URL dataNodeUrl = getPropsUrl(CommonConfig.DATA_NODE_CONFIG_NAME); - URL commonConfigUrl = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index b92918f58a5be..3ee8af7ec77cb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -323,8 +323,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { - LOGGER.error( - "Please set the cn_seed_config_node parameter in iotdb-confignode.properties file."); + LOGGER.error("Please set the cn_seed_config_node parameter in iotdb-system.properties file."); throw new StartupException("The seedConfigNode setting in conf is empty"); } diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat index 569374a89d26d..c6fd64e8fb34b 100755 --- a/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/start-datanode.bat @@ -102,51 +102,43 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" ( echo "Can't find datanode-env.bat" ) +@REM SET CONFIG FILE +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( + set CONFIG_FILE="%IOTDB_CONF%\iotdb-datanode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( + set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) ELSE ( + set CONFIG_FILE= +) + @REM CHECK THE PORT USAGES -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF DEFINED CONFIG_FILE ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_rpc_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_mpp_data_exchange_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_schema_region_consensus_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - "%IOTDB_CONF%\iotdb-datanode.properties"') do ( - set dn_data_region_consensus_port=%%i - ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( - set dn_rpc_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( - set dn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_mpp_data_exchange_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_schema_region_consensus_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - "%IOTDB_HOME%\conf\iotdb-datanode.properties"') do ( + "%CONFIG_FILE%"') do ( set dn_data_region_consensus_port=%%i ) ) ELSE ( - echo "Can't find iotdb-datanode.properties, check the default ports" + echo "Can't find iotdb-system.properties or iotdb-datanode.properties, check the default ports" set dn_rpc_port=6667 set dn_internal_port=10730 set dn_mpp_data_exchange_port=10740 diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index 2197251570e54..27d09f156b1d6 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -22,15 +22,21 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ +IF EXIST "%superior%\conf\iotdb-system.properties" ( + set config_file="%superior_dir%\iotdb-system.properties" +) ELSE ( + set config_file="%superior_dir%\iotdb-datanode.properties" +) + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" -"%superior_dir%\conf\iotdb-datanode.properties"') do ( +"%config_file%"') do ( set dn_rpc_port=%%i ) echo Check whether the rpc_port is used..., port is %dn_rpc_port% for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" -"%superior_dir%\conf\iotdb-datanode.properties"') do ( +"%config_file%"') do ( set dn_rpc_address=%%i ) diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh index cb253f1a1edf3..92d18ebd668b1 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh @@ -19,7 +19,12 @@ # DATANODE_CONF="`dirname "$0"`/../conf" -dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` + +if [ -f "${CONFIG_DIR}/iotdb-system.properties" ]; then + dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-system.properties` +else + dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` +fi force="" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 309825006a914..1ad67a99da5e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -103,9 +103,9 @@ public class IoTDBDescriptor { static { ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); - URL configNodeUrl = getPropsUrl(CommonConfig.CONFIG_NODE_CONFIG_NAME); - URL dataNodeUrl = getPropsUrl(CommonConfig.DATA_NODE_CONFIG_NAME); - URL commonConfigUrl = getPropsUrl(CommonConfig.COMMON_CONFIG_NAME); + URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); + URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); + URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 50c698079ff1e..64c0f65fe3034 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -20,8 +20,6 @@ package org.apache.iotdb.db.conf.rest; public class IoTDBRestServiceConfig { - static final String CONFIG_NAME = "iotdb-datanode.properties"; - static final String OLD_CONFIG_NAME = "iotdb-common.properties"; /** If the enableRestService is true, we will start REST Service. */ private boolean enableRestService = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java index 7a9a16aa20db9..9cf9d51cf2119 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java @@ -49,7 +49,7 @@ public class TagLogFile implements AutoCloseable { private FileChannel fileChannel; private static final String LENGTH_EXCEED_MSG = "Tag/Attribute exceeds the max length limit. " - + "Please enlarge tag_attribute_total_size in iotdb-common.properties"; + + "Please enlarge tag_attribute_total_size in iotdb-system.properties"; private static final int MAX_LENGTH = CommonDescriptor.getInstance().getConfig().getTagAttributeTotalSize(); diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat index 36691cff4095f..0fd3bd5d99471 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.bat @@ -35,7 +35,11 @@ if not "%CLEAN_SERVICE%"=="y" if not "%CLEAN_SERVICE%"=="Y" ( start cmd /c "%IOTDB_HOME%\\sbin\\stop-confignode.bat -f" timeout /t 3 > nul rmdir /s /q "%IOTDB_HOME%\data\confignode\" 2>nul -set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-confignode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set IOTDB_CONFIGNODE_CONFIG="%IOTDB_HOME%\conf\iotdb-confignode.properties" +) set "delimiter=,;" for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_system_dir" %IOTDB_CONFIGNODE_CONFIG%') do ( diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh index 88a5e3488308b..fee8969936206 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-confignode.sh @@ -33,7 +33,12 @@ fi nohup bash ${IOTDB_HOME}/sbin/stop-confignode.sh -f >/dev/null 2>&1 & rm -rf ${IOTDB_HOME}/data/confignode/ >/dev/null 2>&1 & -IOTDB_CONFIGNODE_CONFIG=${IOTDB_HOME}/conf/iotdb-confignode.properties +if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + IOTDB_CONFIGNODE_CONFIG="${IOTDB_HOME}/conf/iotdb-system.properties" +else + IOTDB_CONFIGNODE_CONFIG="${IOTDB_HOME}/conf/iotdb-confignode.properties" +fi + cn_system_dir=$(echo $(grep '^cn_system_dir=' ${IOTDB_CONFIGNODE_CONFIG} || echo "data/confignode/system") | sed 's/.*=//') cn_consensus_dir=$(echo $(grep '^cn_consensus_dir=' ${IOTDB_CONFIGNODE_CONFIG} || echo "data/confignode/consensus") | sed 's/.*=//') diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat index 1ccf0346f504a..26aab580e851f 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.bat @@ -34,7 +34,11 @@ if not "%CLEAN_SERVICE%"=="y" if not "%CLEAN_SERVICE%"=="Y" ( start cmd /c "%IOTDB_HOME%\\sbin\\stop-datanode.bat -f" timeout /t 3 > nul rmdir /s /q "%IOTDB_HOME%\data\datanode\" 2>nul -set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set IOTDB_DATANODE_CONFIG="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) set "delimiter=,;" for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_system_dir" %IOTDB_DATANODE_CONFIG%') do ( diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh index 98e287ef4e6f8..9a1580a2a9e63 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/destroy-datanode.sh @@ -32,7 +32,13 @@ fi nohup bash ${IOTDB_HOME}/sbin/stop-datanode.sh -f >/dev/null 2>&1 & rm -rf ${IOTDB_HOME}/data/datanode/ >/dev/null 2>&1 & -IOTDB_DATANODE_CONFIG=${IOTDB_HOME}/conf/iotdb-datanode.properties + +if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + IOTDB_DATANODE_CONFIG="${IOTDB_HOME}/conf/iotdb-system.properties" +else + IOTDB_DATANODE_CONFIG="${IOTDB_HOME}/conf/iotdb-datanode.properties" +fi + dn_system_dir=$(echo $(grep '^dn_system_dir=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/system") | sed 's/.*=//') dn_data_dirs=$(echo $(grep '^dn_data_dirs=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/data") | sed 's/.*=//') dn_consensus_dir=$(echo $(grep '^dn_consensus_dir=' ${IOTDB_DATANODE_CONFIG} || echo "data/datanode/consensus") | sed 's/.*=//') diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat index f3df96a54b19c..b9c4a099ad2c6 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat @@ -234,7 +234,11 @@ exit /b :local_dirs_check setlocal enabledelayedexpansion -set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +) for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do ( if "%%a"=="dn_data_dirs" ( set "dn_data_dirs=%%b" @@ -352,46 +356,36 @@ if defined operation_dirs ( exit /b :local_ports_check -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-datanode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( + set DATANODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-datanode.properties" +) ELSE ( + set DATANODE_CONFIG_FILE_PATH= +) +IF DEFINED DATANODE_CONFIG_FILE_PATH ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_rpc_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_mpp_data_exchange_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_schema_region_consensus_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - %IOTDB_CONF%\iotdb-datanode.properties') do ( - set dn_data_region_consensus_port=%%i - ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-datanode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_rpc_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_internal_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_mpp_data_exchange_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_mpp_data_exchange_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_schema_region_consensus_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( - set dn_schema_region_consensus_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_data_region_consensus_port" - %IOTDB_HOME%\conf\iotdb-datanode.properties') do ( + %DATANODE_CONFIG_FILE_PATH%') do ( set dn_data_region_consensus_port=%%i ) ) ELSE ( @@ -402,24 +396,26 @@ IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( set dn_data_region_consensus_port=10760 ) -IF EXIST "%IOTDB_CONF%\iotdb-confignode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_CONF%\iotdb-confignode.properties" +) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( + set CONFIGNODE_CONFIG_FILE_PATH="%IOTDB_HOME%\conf\iotdb-confignode.properties" +) ELSE ( + set CONFIGNODE_CONFIG_FILE_PATH= +) +IF DEFINED CONFIGNODE_CONFIG_FILE_PATH ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - %IOTDB_CONF%\iotdb-confignode.properties') do ( + %CONFIGNODE_CONFIG_FILE_PATH%') do ( set cn_internal_port=%%i ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - %IOTDB_CONF%\iotdb-confignode.properties') do ( + %CONFIGNODE_CONFIG_FILE_PATH%') do ( set cn_consensus_port=%%i ) -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_internal_port" - %IOTDB_HOME%\conf\iotdb-confignode.properties') do ( - set cn_internal_port=%%i - ) - for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^cn_consensus_port" - %IOTDB_HOME%\conf\iotdb-confignode.properties') do ( - set cn_consensus_port=%%i - ) ) ELSE ( set cn_internal_port=10710 set cn_consensus_port=10720 diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh index 19739024ab1b6..5ec862a61fc8d 100644 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh @@ -170,7 +170,12 @@ get_properties_value() { local file_name=$1 local property_name=$2 local default_value=$3 - local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties") + if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + local file_path="${IOTDB_HOME}/conf/iotdb-system.properties" + else + local file_path="${IOTDB_HOME}/conf/${file_name}.properties" + fi + local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}") if [ -z "$property_value" ]; then property_value="$default_value" fi diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh index 7539ec8834891..753d1bccb0517 100755 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh @@ -156,7 +156,19 @@ checkDataNodePortUsages () { echo "Warning: If you do not use sudo, the checking may not detect all the occupied ports." fi occupied=false - if [ -f "$IOTDB_CONF/iotdb-datanode.properties" ]; then + if [ -f "$IOTDB_CONF/iotdb-system.properties" ]; then + dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-system.properties) + elif [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then + dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_HOME}"/conf/iotdb-system.properties) + elif [ -f "$IOTDB_CONF/iotdb-datanode.properties" ]; then dn_rpc_port=$(sed '/^dn_rpc_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_internal_port=$(sed '/^dn_internal_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_mpp_data_exchange_port=$(sed '/^dn_mpp_data_exchange_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) @@ -169,7 +181,7 @@ checkDataNodePortUsages () { dn_schema_region_consensus_port=$(sed '/^dn_schema_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) dn_data_region_consensus_port=$(sed '/^dn_data_region_consensus_port=/!d;s/.*=//' "${IOTDB_CONF}"/iotdb-datanode.properties) else - echo "Warning: cannot find iotdb-datanode.properties, check the default configuration" + echo "Warning: cannot find iotdb-system.properties or iotdb-datanode.properties, check the default configuration" fi dn_rpc_port=${dn_rpc_port:-6667} dn_internal_port=${dn_internal_port:-10730} @@ -244,14 +256,20 @@ checkConfigNodePortUsages () { echo "Warning: If you do not use sudo, the checking may not detect all the occupied ports." fi occupied=false - if [ -f "$CONFIGNODE_CONF/iotdb-confignode.properties" ]; then + if [ -f "$CONFIGNODE_CONF/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) + cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) + elif [ -f "$CONFIGNODE_HOME/conf/iotdb-system.properties" ]; then + cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-system.properties) + cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-system.properties) + elif [ -f "$CONFIGNODE_CONF/iotdb-confignode.properties" ]; then cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) elif [ -f "$CONFIGNODE_HOME/conf/iotdb-confignode.properties" ]; then cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-confignode.properties) cn_consensus_port=$(sed '/^cn_consensus_port=/!d;s/.*=//' "${CONFIGNODE_HOME}"/conf/iotdb-confignode.properties) else - echo "Cannot find iotdb-confignode.properties, check the default configuration" + echo "Cannot find iotdb-system.properties or iotdb-confignode.properties, check the default configuration" fi cn_internal_port=${cn_internal_port:-10710} cn_consensus_port=${cn_consensus_port:-10720} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index ee42c0ca32ffc..c39c60e069fcf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -36,9 +36,9 @@ public class CommonConfig { - public static final String CONFIG_NODE_CONFIG_NAME = "iotdb-confignode.properties"; - public static final String DATA_NODE_CONFIG_NAME = "iotdb-datanode.properties"; - public static final String COMMON_CONFIG_NAME = "iotdb-common.properties"; + public static final String OLD_CONFIG_NODE_CONFIG_NAME = "iotdb-confignode.properties"; + public static final String OLD_DATA_NODE_CONFIG_NAME = "iotdb-datanode.properties"; + public static final String OLD_COMMON_CONFIG_NAME = "iotdb-common.properties"; public static final String SYSTEM_CONFIG_NAME = "iotdb-system.properties"; private static final Logger logger = LoggerFactory.getLogger(CommonConfig.class); From 930994bc752dfcbf91c00623c0b56dac0fe7c554 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 20 May 2024 16:30:42 +0800 Subject: [PATCH 06/45] modify auto update tool --- .../confignode/src/assembly/confignode.xml | 4 ++-- iotdb-core/datanode/src/assembly/server.xml | 4 ++-- .../conf/ConfigFileAutoUpdateTool.java | 22 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/iotdb-core/confignode/src/assembly/confignode.xml b/iotdb-core/confignode/src/assembly/confignode.xml index 7e858de63a043..242e3fb810d02 100644 --- a/iotdb-core/confignode/src/assembly/confignode.xml +++ b/iotdb-core/confignode/src/assembly/confignode.xml @@ -44,8 +44,8 @@ - ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties - conf/iotdb-common.properties + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties + conf/iotdb-system.properties ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh diff --git a/iotdb-core/datanode/src/assembly/server.xml b/iotdb-core/datanode/src/assembly/server.xml index b1a3025979451..96bfdd92748a6 100644 --- a/iotdb-core/datanode/src/assembly/server.xml +++ b/iotdb-core/datanode/src/assembly/server.xml @@ -39,8 +39,8 @@ - ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties - conf/iotdb-common.properties + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties + conf/iotdb-system.properties diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java index ee93362c457b2..e80a3b33089f1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java @@ -58,22 +58,22 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, File dataNodeFile = new File(dataNodeUrl.getFile()); File commonFile = new File(commonUrl.getFile()); + if (systemFile.exists()) { + return; + } boolean canUpdate = configNodeFile.exists() && dataNodeFile.exists() && commonFile.exists(); if (!canUpdate) { return; } - if (!systemFile.exists()) { - systemFile.createNewFile(); - } - - acquireTargetFileLock(systemFile); + File lockFile = new File(systemFile.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); try { // other progress updated this file - if (systemFile.length() != 0) { + if (systemFile.exists()) { return; } - try (RandomAccessFile raf = new RandomAccessFile(systemFile, "rw")) { + try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { raf.write(license.getBytes()); String configNodeContent = readConfigLines(configNodeFile); raf.write(configNodeContent.getBytes()); @@ -82,8 +82,9 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, String commonContent = readConfigLines(commonFile); raf.write(commonContent.getBytes()); } + Files.move(lockFile.toPath(), systemFile.toPath()); } finally { - releaseFileLock(systemFile); + releaseFileLock(lockFile); } } @@ -95,9 +96,8 @@ private String readConfigLines(File file) throws IOException { private void acquireTargetFileLock(File file) throws IOException, InterruptedException { long waitTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(20); - File lockFile = new File(file.getPath() + lockFileSuffix); while (System.currentTimeMillis() < waitTime) { - if (lockFile.createNewFile()) { + if (file.createNewFile()) { return; } Thread.sleep(TimeUnit.MICROSECONDS.toMillis(100)); @@ -105,6 +105,6 @@ private void acquireTargetFileLock(File file) throws IOException, InterruptedExc } private void releaseFileLock(File file) throws IOException { - Files.deleteIfExists(new File(file.getPath() + lockFileSuffix).toPath()); + Files.deleteIfExists(file.toPath()); } } From ae88211bd8b5b05ce92432e2b259e86344dec977 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 20 May 2024 17:06:14 +0800 Subject: [PATCH 07/45] fix config update tool --- .../apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java index e80a3b33089f1..e28d6b23c50a3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java @@ -61,7 +61,7 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, if (systemFile.exists()) { return; } - boolean canUpdate = configNodeFile.exists() && dataNodeFile.exists() && commonFile.exists(); + boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); if (!canUpdate) { return; } @@ -89,6 +89,9 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, } private String readConfigLines(File file) throws IOException { + if (!file.exists()) { + return ""; + } byte[] bytes = Files.readAllBytes(file.toPath()); String content = new String(bytes); return content.replace(license, ""); From de2128d5d32822ed196d14d26257536fbefdfd11 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 22 May 2024 10:59:57 +0800 Subject: [PATCH 08/45] fix ut --- .../iotdb/it/env/cluster/ClusterConstant.java | 1 + .../env/cluster/node/ConfigNodeWrapper.java | 7 +- .../it/env/cluster/node/DataNodeWrapper.java | 7 +- .../conf/iotdb-confignode.properties | 147 --- .../resources/sbin/stop-confignode.sh | 2 +- .../iotdb-confignode.properties | 31 - ...mon.properties => iotdb-system.properties} | 13 + .../iotdb-confignode.properties | 31 - ...mon.properties => iotdb-system.properties} | 13 + .../iotdb-confignode.properties | 31 - ...mon.properties => iotdb-system.properties} | 13 + .../resources/conf/iotdb-datanode.properties | 316 ----- .../assembly/resources/sbin/stop-datanode.sh | 2 +- .../datanode1conf/iotdb-common.properties | 22 - ...ode.properties => iotdb-system.properties} | 7 +- .../datanode2conf/iotdb-common.properties | 22 - ...ode.properties => iotdb-system.properties} | 5 + .../datanode3conf/iotdb-common.properties | 22 - ...ode.properties => iotdb-system.properties} | 7 +- .../test/resources/iotdb-system.properties | 76 ++ .../resources/conf/iotdb-common.properties | 1106 ----------------- 21 files changed, 141 insertions(+), 1740 deletions(-) delete mode 100644 iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties delete mode 100644 iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties rename iotdb-core/confignode/src/test/resources/confignode1conf/{iotdb-common.properties => iotdb-system.properties} (76%) delete mode 100644 iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties rename iotdb-core/confignode/src/test/resources/confignode2conf/{iotdb-common.properties => iotdb-system.properties} (76%) delete mode 100644 iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties rename iotdb-core/confignode/src/test/resources/confignode3conf/{iotdb-common.properties => iotdb-system.properties} (76%) delete mode 100644 iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties delete mode 100644 iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties rename iotdb-core/datanode/src/test/resources/datanode1conf/{iotdb-datanode.properties => iotdb-system.properties} (88%) delete mode 100644 iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties rename iotdb-core/datanode/src/test/resources/datanode2conf/{iotdb-datanode.properties => iotdb-system.properties} (90%) delete mode 100644 iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties rename iotdb-core/datanode/src/test/resources/datanode3conf/{iotdb-datanode.properties => iotdb-system.properties} (88%) create mode 100644 iotdb-core/datanode/src/test/resources/iotdb-system.properties delete mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index 6cf9fb71c1cb0..e113fb662a558 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -118,6 +118,7 @@ public class ClusterConstant { public static final String CONFIG_NODE_PROPERTIES_FILE = "iotdb-confignode.properties"; public static final String DATA_NODE_PROPERTIES_FILE = "iotdb-datanode.properties"; public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties"; + public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties"; public static final String SYSTEM_PROPERTIES_FILE = "system.properties"; public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties"; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index 91a53b04e0e17..6bd90c9d36cdc 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -33,17 +33,16 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_CONSENSUS_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_METRIC_IOTDB_REPORTER_HOST; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CN_SYSTEM_DIR; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_PROPERTIES; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; @@ -91,12 +90,12 @@ public ConfigNodeWrapper( @Override protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", CONFIG_NODE_PROPERTIES_FILE); + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override protected String getTargetCommonConfigPath() { - return workDirFilePath("conf", COMMON_PROPERTIES_FILE); + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index e6d16f9860b3b..0a12cfae43c38 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -29,13 +29,11 @@ import java.util.List; import static org.apache.iotdb.consensus.ConsensusFactory.SIMPLE_CONSENSUS; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.COMMON_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES; @@ -53,6 +51,7 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_SYSTEM_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_TRACING_DIR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DN_WAL_DIRS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.IOTDB_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAIN_CLASS_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MAX_TSBLOCK_SIZE_IN_BYTES; import static org.apache.iotdb.it.env.cluster.ClusterConstant.MQTT_HOST; @@ -125,12 +124,12 @@ public DataNodeWrapper( @Override protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", DATA_NODE_PROPERTIES_FILE); + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override protected String getTargetCommonConfigPath() { - return workDirFilePath("conf", COMMON_PROPERTIES_FILE); + return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } @Override diff --git a/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties b/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties deleted file mode 100644 index 8257cc040295f..0000000000000 --- a/iotdb-core/confignode/src/assembly/resources/conf/iotdb-confignode.properties +++ /dev/null @@ -1,147 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#################### -### Config Node RPC Configuration -#################### - -# Used for RPC communication inside cluster. -# Could set 127.0.0.1(for local test) or ipv4 address. -# Datatype: String -cn_internal_address=127.0.0.1 - -# Used for RPC communication inside cluster. -# Datatype: int -cn_internal_port=10710 - -# Used for consensus communication among ConfigNodes inside cluster. -# Datatype: int -cn_consensus_port=10720 - -#################### -### Seed ConfigNode -#################### - -# For the first ConfigNode to start, cn_seed_config_node points to its own cn_internal_address:cn_internal_port. -# For other ConfigNodes that to join the cluster, cn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. -# Note: After this ConfigNode successfully joins the cluster for the first time, this parameter is no longer used. -# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. -# Format: address:port e.g. 127.0.0.1:10710 -# Datatype: String -cn_seed_config_node=127.0.0.1:10710 - -#################### -### Directory configuration -#################### - -# system dir -# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/system). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# cn_system_dir=data\\confignode\\system -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# cn_system_dir=data/confignode/system - -# consensus dir -# If this property is unset, system will save the data in the default relative path directory under the confignode folder(i.e., %CONFIGNODE_HOME%/data/confignode/consensus). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the confignode folder. -# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# cn_consensus_dir=data\\confignode\\consensus -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# cn_consensus_dir=data/confignode/consensus - -# pipe_receiver_file_dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${cn_system_dir}/pipe/receiver). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If pipe_receiver_file_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dir=data\\confignode\\system\\pipe\\receiver -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dir=data/confignode/system/pipe/receiver - -#################### -### thrift rpc configuration -#################### - -# this feature is under development, set this as false before it is done. -# Datatype: boolean -# cn_rpc_thrift_compression_enable=false - -# if true, a snappy based compression method will be called before sending data by the network -# Datatype: boolean -# this feature is under development, set this as false before it is done. -# cn_rpc_advanced_compression_enable=false - -# Datatype: int -# cn_rpc_max_concurrent_client_num=65535 - -# thrift max frame size, 512MB by default -# Datatype: int -# cn_thrift_max_frame_size=536870912 - -# thrift init buffer size -# Datatype: int -# cn_thrift_init_buffer_size=1024 - -# Thrift socket and connection timeout between raft nodes, in milliseconds. -# Datatype: int -# cn_connection_timeout_ms=60000 - -# selector thread (TAsyncClientManager) nums for async thread in a clientManager -# Datatype: int -# cn_selector_thread_nums_of_client_manager=1 - -# The maximum number of clients that can be allocated for a node in a clientManager. -# when the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked -# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. -# Datatype: int -# cn_max_client_count_for_each_node_in_client_manager=300 - -#################### -### Metric Configuration -#################### - -# The reporters of metric module to report metrics -# If there are more than one reporter, please separate them by commas ",". -# Options: [JMX, PROMETHEUS] -# Datatype: String -# cn_metric_reporter_list= - -# The level of metric module -# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] -# Datatype: String -# cn_metric_level=CORE - -# The period of async collection of some metrics in second -# Datatype: int -# cn_metric_async_collect_period=5 - -# The port of prometheus reporter of metric module -# Datatype: int -# cn_metric_prometheus_reporter_port=9091 \ No newline at end of file diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh index b44d35fb03f8a..5604afd3d32fa 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh @@ -20,7 +20,7 @@ CONFIGNODE_CONF="$(dirname "$0")/../conf" -if [ -f "${CONFIG_DIR}/iotdb-system.properties" ]; then +if [ -f "${CONFIGNODE_CONF}/iotdb-system.properties" ]; then cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-system.properties) else cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties deleted file mode 100644 index ade8959df9936..0000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10710 -cn_consensus_port=10720 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode1/system -cn_data_dirs=target/confignode1/data -cn_consensus_dir=target/confignode1/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9091 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties index 8981b21285bbd..b396e373f8699 100644 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10710 +cn_consensus_port=10720 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode1/system +cn_data_dirs=target/confignode1/data +cn_consensus_dir=target/confignode1/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9091 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties deleted file mode 100644 index eabd45e89e91d..0000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10711 -cn_consensus_port=10721 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode2/system -cn_data_dirs=target/confignode2/data -cn_consensus_dir=target/confignode2/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9093 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties index a9789fedabbe4..ef3ba921f96c2 100644 --- a/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode2conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10711 +cn_consensus_port=10721 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode2/system +cn_data_dirs=target/confignode2/data +cn_consensus_dir=target/confignode2/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9093 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties deleted file mode 100644 index ad706b698ec5e..0000000000000 --- a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-confignode.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cn_internal_address=127.0.0.1 -cn_internal_port=10712 -cn_consensus_port=10722 -cn_seed_config_node=127.0.0.1:10710 -cn_system_dir=target/confignode3/system -cn_data_dirs=target/confignode3/data -cn_consensus_dir=target/confignode3/consensus - -cn_metric_reporter_list=PROMETHEUS -cn_metric_level=IMPORTANT -cn_metric_async_collect_period=5 -cn_metric_prometheus_reporter_port=9095 \ No newline at end of file diff --git a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties similarity index 76% rename from iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties rename to iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties index 6a95388bd72c9..48f99317a2fdf 100644 --- a/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-common.properties +++ b/iotdb-core/confignode/src/test/resources/confignode3conf/iotdb-system.properties @@ -17,6 +17,19 @@ # under the License. # +cn_internal_address=127.0.0.1 +cn_internal_port=10712 +cn_consensus_port=10722 +cn_seed_config_node=127.0.0.1:10710 +cn_system_dir=target/confignode3/system +cn_data_dirs=target/confignode3/data +cn_consensus_dir=target/confignode3/consensus + +cn_metric_reporter_list=PROMETHEUS +cn_metric_level=IMPORTANT +cn_metric_async_collect_period=5 +cn_metric_prometheus_reporter_port=9095 + timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus diff --git a/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties b/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties deleted file mode 100644 index 949e340075001..0000000000000 --- a/iotdb-core/datanode/src/assembly/resources/conf/iotdb-datanode.properties +++ /dev/null @@ -1,316 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -#################### -### Data Node RPC Configuration -#################### - -# Used for connection of IoTDB native clients(Session) -# Could set 127.0.0.1(for local test) or ipv4 address -# Datatype: String -dn_rpc_address=0.0.0.0 - -# Used for connection of IoTDB native clients(Session) -# Bind with dn_rpc_address -# Datatype: int -dn_rpc_port=6667 - -# Used for communication inside cluster. -# could set 127.0.0.1(for local test) or ipv4 address. -# Datatype: String -dn_internal_address=127.0.0.1 - -# Used for communication inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_internal_port=10730 - -# Port for data exchange among DataNodes inside cluster -# Bind with dn_internal_address -# Datatype: int -dn_mpp_data_exchange_port=10740 - -# port for consensus's communication for schema region inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_schema_region_consensus_port=10750 - -# port for consensus's communication for data region inside cluster. -# Bind with dn_internal_address -# Datatype: int -dn_data_region_consensus_port=10760 - -# Datatype: long -# The time of data node waiting for the next retry to join into the cluster. -# dn_join_cluster_retry_interval_ms=5000 - -#################### -### SSL Configuration -#################### - -# Does dn_rpc_port enable SSL -# enable_thrift_ssl=false - -# Rest Service enabled SSL -# enable_https=false - -# SSL key store path -# linux e.g. /home/iotdb/server.keystore (absolute path) or server.keystore (relative path) -# windows e.g. C:\\iotdb\\server.keystore (absolute path) or server.keystore (relative path) -# key_store_path= - -# SSL key store password -# key_store_pwd= - -#################### -### Seed ConfigNode -#################### - -# dn_seed_config_node points to any running ConfigNode's cn_internal_address:cn_internal_port. -# Note: After this DataNode successfully joins the cluster for the first time, this parameter is no longer used. -# Each node automatically maintains the list of ConfigNodes and traverses connections when restarting. -# Format: address:port e.g. 127.0.0.1:10710 -# Datatype: String -dn_seed_config_node=127.0.0.1:10710 - -#################### -### Connection Configuration -#################### - -# The maximum session idle time. unit: ms -# Idle sessions are the ones that performs neither query or non-query operations for a period of time -# Set to 0 to disable session timeout -# Datatype: int -# dn_session_timeout_threshold=0 - -# Datatype: boolean -# dn_rpc_thrift_compression_enable=false - -# if true, a snappy based compression method will be called before sending data by the network -# Datatype: boolean -# this feature is under development, set this as false before it is done. -# dn_rpc_advanced_compression_enable=false - -# Datatype: int -# dn_rpc_selector_thread_count=1 - -# Datatype: int -# dn_rpc_min_concurrent_client_num=1 - -# Datatype: int -# dn_rpc_max_concurrent_client_num=65535 - -# thrift max frame size, 512MB by default -# Datatype: int -# dn_thrift_max_frame_size=536870912 - -# thrift init buffer size -# Datatype: int -# dn_thrift_init_buffer_size=1024 - -# Thrift socket and connection timeout between raft nodes, in milliseconds. -# Datatype: int -# dn_connection_timeout_ms=60000 - -# selector thread (TAsyncClientManager) nums for async thread in a clientManager -# Datatype: int -# dn_selector_thread_count_of_client_manager=1 - -# The maximum number of clients that can be allocated for a node in a clientManager. -# When the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked -# for a while, then ClientManager will throw ClientManagerException if there are no clients after the block time. -# Datatype: int -# dn_max_client_count_for_each_node_in_client_manager=300 - -#################### -### Directory Configuration -#################### - -# system dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/system). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_system_dir=data\\datanode\\system -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_system_dir=data/datanode/system - - -# data dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode/data). -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If data_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_data_dirs=data\\datanode\\data -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_data_dirs=data/datanode/data - - -# multi_dir_strategy -# The strategy is used to choose a directory from data_dirs for the system to store a new tsfile. -# System provides two strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy. -# The info of the two strategies are as follows: -# 1. SequenceStrategy: the system will choose the directory in sequence. -# 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space. -# Set SequenceStrategy or MaxDiskUsableSpaceFirstStrategy to apply the corresponding strategy. -# If this property is unset, system will use SequenceStrategy as default strategy. -# For this property, fully-qualified class name (include package name) and simple class name are both acceptable. -# dn_multi_dir_strategy=SequenceStrategy - -# consensus dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If consensus_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_consensus_dir=data\\datanode\\consensus -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_consensus_dir=data/datanode/consensus - -# wal dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If wal_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_wal_dirs=data\\datanode\\wal -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_wal_dirs=data/datanode/wal - -# tracing dir -# Uncomment following fields to configure the tracing root directory. -# For Windows platform, the index is as follows: -# dn_tracing_dir=datanode\\tracing -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_tracing_dir=datanode/tracing - -# sync dir -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# dn_sync_dir=data\\datanode\\sync -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# dn_sync_dir=data/datanode/sync - -# sort_tmp_dir -# This property is used to configure the temporary directory for sorting operation. -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# Note: If sort_tmp_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# sort_tmp_dir=data\\datanode\\tmp -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# sort_tmp_dir=data/datanode/tmp - -# pipe_receiver_file_dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/${dn_system_dir}/pipe/receiver). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If pipe_receiver_file_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dirs=data\\datanode\\system\\pipe\\receiver -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# pipe_receiver_file_dirs=data/datanode/system/pipe/receiver - -#################### -### Metric Configuration -#################### - -# The reporters of metric module to report metrics -# If there are more than one reporter, please separate them by commas ",". -# Options: [JMX, PROMETHEUS] -# Datatype: String -# dn_metric_reporter_list= - -# The level of metric module -# Options: [OFF, CORE, IMPORTANT, NORMAL, ALL] -# Datatype: String -# dn_metric_level=CORE - -# The period of async collection of some metrics in second -# Datatype: int -# dn_metric_async_collect_period=5 - -# The port of prometheus reporter of metric module -# Datatype: int -dn_metric_prometheus_reporter_port=9092 - -# The type of internal reporter in metric module, used for checking flushed point number -# Options: [MEMORY, IOTDB] -# Datatype: String -# dn_metric_internal_reporter_type=MEMORY - -#################### -### REST Service Configuration -#################### - -# Is the REST service enabled -# enable_rest_service=false - -# the binding port of the REST service -# rest_service_port=18080 - -# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json -# enable_swagger=false - -# the default row limit to a REST query response when the rowSize parameter is not given in request -# rest_query_default_row_size_limit=10000 - -# the expiration time of the user login information cache (in seconds) -# cache_expire_in_seconds=28800 - -# maximum number of users can be stored in the user login cache. -# cache_max_num=100 - -# init capacity of users can be stored in the user login cache. -# cache_init_num=10 - -# Is client authentication required -# client_auth=false - -# SSL trust store path -# trust_store_path="" - -# SSL trust store password. -# trust_store_pwd="" - -# SSL timeout (in seconds) -# idle_timeout_in_seconds=50000 diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh index 92d18ebd668b1..71bdcef0a82b5 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh @@ -20,7 +20,7 @@ DATANODE_CONF="`dirname "$0"`/../conf" -if [ -f "${CONFIG_DIR}/iotdb-system.properties" ]; then +if [ -f "${DATANODE_CONF}/iotdb-system.properties" ]; then dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-system.properties` else dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` diff --git a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties deleted file mode 100644 index 143c0e0b7f007..0000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode1/ext/udf -trigger_lib_dir=target/datanode1/ext/trigger -pipe_lib_dir=target/datanode1/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties similarity index 88% rename from iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties index a7ed41f26152e..5bccd18242b0d 100644 --- a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-system.properties @@ -39,4 +39,9 @@ sort_tmp_dir=target/datanode1/tmp dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 -dn_metric_prometheus_reporter_port=9097 \ No newline at end of file +dn_metric_prometheus_reporter_port=9097 + +timestamp_precision=ms +udf_lib_dir=target/datanode1/ext/udf +trigger_lib_dir=target/datanode1/ext/trigger +pipe_lib_dir=target/datanode1/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties deleted file mode 100644 index 9cf060d61fdd2..0000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode2/ext/udf -trigger_lib_dir=target/datanode2/ext/trigger -pipe_lib_dir=target/datanode2/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties similarity index 90% rename from iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties index a8508a79678df..26f56c3c03cee 100644 --- a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-system.properties @@ -40,3 +40,8 @@ dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 dn_metric_prometheus_reporter_port=9099 + +timestamp_precision=ms +udf_lib_dir=target/datanode2/ext/udf +trigger_lib_dir=target/datanode2/ext/trigger +pipe_lib_dir=target/datanode2/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties deleted file mode 100644 index 1b48e21f3037e..0000000000000 --- a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-common.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -timestamp_precision=ms -udf_lib_dir=target/datanode3/ext/udf -trigger_lib_dir=target/datanode3/ext/trigger -pipe_lib_dir=target/datanode3/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties similarity index 88% rename from iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties rename to iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties index d4faf82e09b35..9bde591037785 100644 --- a/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-datanode.properties +++ b/iotdb-core/datanode/src/test/resources/datanode3conf/iotdb-system.properties @@ -39,4 +39,9 @@ sort_tmp_dir=target/datanode3/tmp dn_metric_reporter_list=PROMETHEUS dn_metric_level=IMPORTANT dn_metric_async_collect_period=5 -dn_metric_prometheus_reporter_port=9101 \ No newline at end of file +dn_metric_prometheus_reporter_port=9101 + +timestamp_precision=ms +udf_lib_dir=target/datanode3/ext/udf +trigger_lib_dir=target/datanode3/ext/trigger +pipe_lib_dir=target/datanode3/ext/pipe \ No newline at end of file diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties new file mode 100644 index 0000000000000..86943f5ed3479 --- /dev/null +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +base_dir=target/tmp +dn_data_dirs=target/data +dn_wal_dirs=target/wal +index_root_dir=target/index +dn_tracing_dir=target/data/tracing +minimum_pbtree_segment_in_bytes=0 +page_cache_in_pbtree=10 +dn_internal_address=0.0.0.0 +dn_sync_dir=target/sync +dn_seed_config_node=127.0.0.1:10710 + + +#################### +### REST Service Configuration +#################### + +# Is the REST service enabled +enable_rest_service=true + +# the binding port of the REST service +# rest_service_port=18080 + +# Whether to display rest service interface information through swagger. eg: http://ip:port/swagger.json +# enable_swagger=false + +# the default row limit to a REST query response when the rowSize parameter is not given in request +# rest_query_default_row_size_limit=10000 + +# the expiration time of the user login information cache (in seconds) +# cache_expire_in_seconds=28800 + +# maximum number of users can be stored in the user login cache. +# cache_max_num=100 + +# init capacity of users can be stored in the user login cache. +# cache_init_num=10 + +# is SSL enabled +# enable_https=false + +# SSL key store path +# key_store_path= + +# SSL key store password +# key_store_pwd= + +# Is client authentication required +# client_auth=false + +# SSL trust store path +# trust_store_path= + +# SSL trust store password. +# trust_store_pwd= + +# SSL timeout (in seconds) +# idle_timeout_in_seconds=50000 diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties deleted file mode 100644 index 827eeca2ad7c2..0000000000000 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ /dev/null @@ -1,1106 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#################### -### Cluster Configuration -#################### - -# Used for indicate cluster name and distinguish different cluster. -# Datatype: string -cluster_name=defaultCluster - -#################### -### Replication configuration -#################### - -# ConfigNode consensus protocol type. -# This parameter is unmodifiable after ConfigNode starts for the first time. -# These consensus protocols are currently supported: -# 1. org.apache.iotdb.consensus.ratis.RatisConsensus -# 2. org.apache.iotdb.consensus.simple.SimpleConsensus (Only 1 ConfigNode can be deployed) -# Datatype: string -# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus - -# Default number of schema replicas -# Can not be changed after the first start -# Datatype: int -schema_replication_factor=1 - -# SchemaRegion consensus protocol type. -# This parameter is unmodifiable after ConfigNode starts for the first time. -# These consensus protocols are currently supported: -# 1. org.apache.iotdb.consensus.ratis.RatisConsensus -# 2. org.apache.iotdb.consensus.simple.SimpleConsensus (The schema_replication_factor can only be set to 1) -# Datatype: string -# schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus - -# Default number of data replicas -# Can not be changed after the first start -# Datatype: int -data_replication_factor=1 - -# DataRegion consensus protocol type. -# This parameter is unmodifiable after ConfigNode starts for the first time. -# These consensus protocols are currently supported: -# 1. org.apache.iotdb.consensus.simple.SimpleConsensus (The data_replication_factor can only be set to 1) -# 2. org.apache.iotdb.consensus.iot.IoTConsensus -# 3. org.apache.iotdb.consensus.ratis.RatisConsensus -# Datatype: string -# data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus - -#################### -### Load balancing configuration -#################### - -# All parameters in Partition configuration is unmodifiable after ConfigNode starts for the first time. -# And these parameters should be consistent within the ConfigNodeGroup. -# Number of SeriesPartitionSlots per Database -# Datatype: Integer -# series_slot_num=1000 - -# SeriesPartitionSlot executor class -# These hashing algorithms are currently supported: -# 1. BKDRHashExecutor(Default) -# 2. APHashExecutor -# 3. JSHashExecutor -# 4. SDBMHashExecutor -# Also, if you want to implement your own SeriesPartition executor, you can inherit the SeriesPartitionExecutor class and -# modify this parameter to correspond to your Java class -# Datatype: String -# series_partition_executor_class=org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor - -# The policy of extension SchemaRegionGroup for each Database. -# These policies are currently supported: -# 1. CUSTOM(Each Database will allocate schema_region_group_per_database RegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend SchemaRegionGroups based on the data it has) -# Datatype: String -# schema_region_group_extension_policy=AUTO - -# When set schema_region_group_extension_policy=CUSTOM, -# this parameter is the default number of SchemaRegionGroups for each Database. -# When set schema_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of SchemaRegionGroups for each Database. -# Datatype: Integer -# default_schema_region_group_num_per_database=1 - -# Only take effect when set schema_region_group_extension_policy=AUTO. -# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode. -# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema, -# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases. -# Datatype: Double -# schema_region_per_data_node=1.0 - -# The policy of extension DataRegionGroup for each Database. -# These policies are currently supported: -# 1. CUSTOM(Each Database will allocate data_region_group_per_database DataRegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend DataRegionGroups based on the data it has) -# Datatype: String -# data_region_group_extension_policy=AUTO - -# When set data_region_group_extension_policy=CUSTOM, -# this parameter is the default number of DataRegionGroups for each Database. -# When set data_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of DataRegionGroups for each Database. -# Datatype: Integer -# default_data_region_group_num_per_database=2 - -# Only take effect when set data_region_group_extension_policy=AUTO. -# This parameter is the maximum number of DataRegions expected to be managed by each DataNode. -# Notice: Since each Database requires at least two DataRegionGroups to manage its data, -# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases. -# Datatype: Double -# data_region_per_data_node=5.0 - -# Whether to enable auto leader balance for Ratis consensus protocol. -# The ConfigNode-leader will balance the leader of Ratis-RegionGroups by leader_distribution_policy if set true. -# Notice: Default is false because the Ratis is unstable for this function. -# Datatype: Boolean -# enable_auto_leader_balance_for_ratis_consensus=true - -# Whether to enable auto leader balance for IoTConsensus protocol. -# The ConfigNode-leader will balance the leader of IoTConsensus-RegionGroups by leader_distribution_policy if set true. -# Notice: Default is true because the IoTConsensus depends on this function to distribute leader. -# Datatype: Boolean -# enable_auto_leader_balance_for_iot_consensus=true - -#################### -### Cluster management -#################### - -# Time partition interval in milliseconds, and partitioning data inside each data region, default is equal to one week -# Can not be changed after the first start -# Datatype: long -# time_partition_interval=604800000 - -# The heartbeat interval in milliseconds, default is 1000ms -# Datatype: long -# heartbeat_interval_in_ms=1000 - -# Disk remaining threshold at which DataNode is set to ReadOnly status -# Datatype: double(percentage) -# disk_space_warning_threshold=0.05 - -#################### -### Memory Control Configuration -#################### - -# Memory Allocation Ratio: StorageEngine, QueryEngine, SchemaEngine, Consensus, StreamingEngine and Free Memory. -# The parameter form is a:b:c:d:e:f, where a, b, c, d, e and f are integers. for example: 1:1:1:1:1:1 , 6:2:1:1:1:1 -# If you have high level of writing pressure and low level of reading pressure, please adjust it to for example 6:1:1:1:1:1 -# datanode_memory_proportion=3:3:1:1:1:1 - -# Schema Memory Allocation Ratio: SchemaRegion, SchemaCache, and PartitionCache. -# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:2:1 -# schema_memory_proportion=5:4:1 - -# Memory allocation ratio in StorageEngine: Write, Compaction -# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 -# storage_engine_memory_proportion=8:2 - -# Memory allocation ratio in writing: Memtable, TimePartitionInfo -# Memtable is the total memory size of all memtables -# TimePartitionInfo is the total memory size of last flush time of all data regions -# write_memory_proportion=19:1 - -# primitive array size (length of each array) in array pool -# Datatype: int -# primitive_array_size=64 - -# Ratio of compaction memory for chunk metadata maintains in memory when doing compaction -# Datatype: double -# chunk_metadata_size_proportion=0.1 - -# Ratio of write memory for invoking flush disk, 0.4 by default -# If you have extremely high write load (like batch=1000), it can be set lower than the default value like 0.2 -# Datatype: double -# flush_proportion=0.4 - -# Ratio of write memory allocated for buffered arrays, 0.6 by default -# Datatype: double -# buffered_arrays_memory_proportion=0.6 - -# Ratio of write memory for rejecting insertion, 0.8 by default -# If you have extremely high write load (like batch=1000) and the physical memory size is large enough, -# it can be set higher than the default value like 0.9 -# Datatype: double -# reject_proportion=0.8 - -# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory -# Datatype: double -# device_path_cache_proportion=0.05 - -# If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001 -# Datatype: double -# write_memory_variation_report_proportion=0.001 - -# When an inserting is rejected, waiting period (in ms) to check system again, 50 by default. -# If the insertion has been rejected and the read load is low, it can be set larger. -# Datatype: int -# check_period_when_insert_blocked=50 - -# size of ioTaskQueue. The default value is 10 -# Datatype: int -# io_task_queue_size_for_flushing=10 - -# If true, we will estimate each query's possible memory footprint before executing it and deny it if its estimated memory exceeds current free memory -# Datatype: bool -# enable_query_memory_estimation=true - -#################### -### Schema Engine Configuration -#################### - -# The schema management mode of schema engine. Currently support Memory and PBTree. -# This config of all DataNodes in one cluster must keep same. -# Datatype: string -# schema_engine_mode=Memory - -# cache size for partition. -# This cache is used to improve partition fetch from config node. -# Datatype: int -# partition_cache_size=1000 - -# Size of log buffer in each metadata operation plan(in byte). -# If the size of a metadata operation plan is larger than this parameter, then it will be rejected by SchemaRegion -# If it sets a value smaller than 0, use the default value 1024*1024 -# Datatype: int -# mlog_buffer_size=1048576 - -# The cycle when metadata log is periodically forced to be written to disk(in milliseconds) -# If sync_mlog_period_in_ms=0 it means force metadata log to be written to disk after each refreshment -# Set this parameter to 0 may slow down the operation on slow disk. -# sync_mlog_period_in_ms=100 - -# interval num for tag and attribute records when force flushing to disk -# When a certain amount of tag and attribute records is reached, they will be force flushed to disk -# It is possible to lose at most tag_attribute_flush_interval records -# tag_attribute_flush_interval=1000 - -# max size for a storage block for tags and attributes of one time series. If the combined size of tags and -# attributes exceeds the tag_attribute_total_size, a new storage block will be allocated to continue storing -# the excess data. -# the unit is byte -# Datatype: int -# tag_attribute_total_size=700 - -# max measurement num of internal request -# When creating timeseries with Session.createMultiTimeseries, the user input plan, the timeseries num of -# which exceeds this num, will be split to several plans with timeseries no more than this num. -# max_measurement_num_of_internal_request=10000 - -# Policy of DataNodeSchemaCache eviction. -# Support FIFO and LRU policy. FIFO takes low cache update overhead. LRU takes high cache hit rate. -# datanode_schema_cache_eviction_policy=FIFO - -# This configuration parameter sets the maximum number of time series allowed in the cluster. -# The value should be a positive integer representing the desired threshold. -# When the threshold is reached, users will be prohibited from creating new time series. -# -1 means unlimited -# cluster_timeseries_limit_threshold=-1 - -# This configuration parameter sets the maximum number of device allowed in the cluster. -# The value should be a positive integer representing the desired threshold. -# When the threshold is reached, users will be prohibited from creating new time series. -# -1 means unlimited -# cluster_device_limit_threshold=-1 - -# This configuration parameter sets the maximum number of Cluster Databases allowed. -# The value should be a positive integer representing the desired threshold. -# When the threshold is reached, users will be prohibited from creating new databases. -# -1 means unlimited. -# database_limit_threshold = -1 - - - -#################### -### Configurations for creating schema automatically -#################### - -# Whether creating schema automatically is enabled -# If true, then create database and timeseries automatically when not exists in insertion -# Or else, user need to create database and timeseries before insertion. -# Datatype: boolean -# enable_auto_create_schema=true - -# Database level when creating schema automatically is enabled -# e.g. root.sg0.d1.s2 -# we will set root.sg0 as the database if database level is 1 -# Datatype: int -# default_storage_group_level=1 - -# ALL data types: BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT - -# register time series as which type when receiving boolean string "true" or "false" -# Datatype: TSDataType -# Options: BOOLEAN, TEXT -# boolean_string_infer_type=BOOLEAN - -# register time series as which type when receiving an integer string and using float or double may lose precision -# Datatype: TSDataType -# Options: DOUBLE, FLOAT, INT32, INT64, TEXT -# integer_string_infer_type=DOUBLE - -# register time series as which type when receiving a floating number string "6.7" -# Datatype: TSDataType -# Options: DOUBLE, FLOAT, TEXT -# floating_string_infer_type=DOUBLE - -# register time series as which type when receiving the Literal NaN. -# Datatype: TSDataType -# Options: DOUBLE, FLOAT, TEXT -# nan_string_infer_type=DOUBLE - -# BOOLEAN encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_boolean_encoding=RLE - -# INT32 encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_int32_encoding=TS_2DIFF - -# INT64 encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_int64_encoding=TS_2DIFF - -# FLOAT encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_float_encoding=GORILLA - -# DOUBLE encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_double_encoding=GORILLA - -# TEXT encoding when creating schema automatically is enabled -# Datatype: TSEncoding -# default_text_encoding=PLAIN - -#################### -### Query Configurations -#################### - -# The read consistency level -# These consistency levels are currently supported: -# 1. strong(Default, read from the leader replica) -# 2. weak(Read from a random replica) -# Datatype: string -# read_consistency_level=strong - -# Whether to cache meta data(BloomFilter, ChunkMetadata and TimeSeriesMetadata) or not. -# Datatype: boolean -# meta_data_cache_enable=true - -# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others. -# The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50 -# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50 - -# Whether to enable LAST cache -# Datatype: boolean -# enable_last_cache=true - -# Core size of ThreadPool of MPP data exchange -# Datatype: int -# mpp_data_exchange_core_pool_size=10 - -# Max size of ThreadPool of MPP data exchange -# Datatype: int -# mpp_data_exchange_max_pool_size=10 - -# Max waiting time for MPP data exchange -# Datatype: int -# mpp_data_exchange_keep_alive_time_in_ms=1000 - -# The max execution time of a DriverTask -# Datatype: int, Unit: ms -# driver_task_execution_time_slice_in_ms=200 - -# The max capacity of a TsBlock -# Datatype: int, Unit: byte -# max_tsblock_size_in_bytes=131072 - -# The max number of lines in a single TsBlock -# Datatype: int -# max_tsblock_line_number=1000 - -# Time cost(ms) threshold for slow query -# Datatype: long -# slow_query_threshold=10000 - -# The max executing time of query. unit: ms -# Datatype: int -# query_timeout_threshold=60000 - -# The maximum allowed concurrently executing queries -# Datatype: int -# max_allowed_concurrent_queries=1000 - -# How many threads can concurrently execute query statement. When <= 0, use CPU core number. -# Datatype: int -# query_thread_count=0 - -# How many pipeline drivers will be created for one fragment instance. When <= 0, use CPU core number / 2. -# Datatype: int -# degree_of_query_parallelism=0 - -# The threshold of count map size when calculating the MODE aggregation function -# Datatype: int -# mode_map_size_threshold=10000 - -# The amount of data iterate each time in server (the number of data strips, that is, the number of different timestamps.) -# Datatype: int -# batch_size=100000 - -# The memory for external sort in sort operator, when the data size is smaller than sort_buffer_size_in_bytes, the sort operator will use in-memory sort. -# Datatype: long -# sort_buffer_size_in_bytes=1048576 - -# The threshold of operator count in the result set of EXPLAIN ANALYZE, if the number of operator in the result set is larger than this threshold, operator will be merged. -# Datatype: int -# merge_threshold_of_explain_analyze=10 - -#################### -### Storage Engine Configuration -#################### - -# Use this value to set timestamp precision as "ms", "us" or "ns". -# Once the precision has been set, it can not be changed. -# Datatype: String -# timestamp_precision=ms - -# When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted. -# Datatype: Boolean -# timestamp_precision_check_enabled=true - -# Default TTL for databases that are not set TTL by statements, If not set (default), the TTL will be unlimited. -# Negative value means the TTL is unlimited. -# Notice: if this property is changed, previous created database which are not set TTL will also be affected. -# Datatype: long -# Unit: ms -# default_ttl_in_ms=-1 - -# When the waiting time (in ms) of an inserting exceeds this, throw an exception. 10000 by default. -# If the insertion has been rejected and the read load is low, it can be set larger -# Datatype: int -# max_waiting_time_when_insert_blocked=10000 - -# Add a switch to enable separate sequence and unsequence data. -# If it is true, then data will be separated into seq and unseq data dir. If it is false, then all data will be written into unseq data dir. -# Datatype: boolean -# enable_separate_data=true - -# What will the system do when unrecoverable error occurs. -# Datatype: String -# Optional strategies are as follows: -# 1. CHANGE_TO_READ_ONLY: set system status to read-only and the system only accepts query operations. -# 2. SHUTDOWN: the system will be shutdown. -# handle_system_error=CHANGE_TO_READ_ONLY - -# Whether to timed flush sequence tsfiles' memtables. -# Datatype: boolean -# enable_timed_flush_seq_memtable=true - -# If a memTable's last update time is older than current time minus this, the memtable will be flushed to disk. -# Only check sequence tsfiles' memtables. -# The default flush interval is 10 * 60 * 1000. (unit: ms) -# Datatype: long -# seq_memtable_flush_interval_in_ms=600000 - -# The interval to check whether sequence memtables need flushing. -# The default flush check interval is 30 * 1000. (unit: ms) -# Datatype: long -# seq_memtable_flush_check_interval_in_ms=30000 - -# Whether to timed flush unsequence tsfiles' memtables. -# Datatype: boolean -# enable_timed_flush_unseq_memtable=true - -# If a memTable's last update time is older than current time minus this, the memtable will be flushed to disk. -# Only check unsequence tsfiles' memtables. -# The default flush interval is 10 * 60 * 1000. (unit: ms) -# Datatype: long -# unseq_memtable_flush_interval_in_ms=600000 - -# The interval to check whether unsequence memtables need flushing. -# The default flush check interval is 30 * 1000. (unit: ms) -# Datatype: long -# unseq_memtable_flush_check_interval_in_ms=30000 - -# The sort algorithms used in the memtable's TVList -# TIM: default tim sort, -# QUICK: quick sort, -# BACKWARD: backward sort -# tvlist_sort_algorithm=TIM - -# When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 100000. -# Datatype: int -# avg_series_point_number_threshold=100000 - -# How many threads can concurrently flush. When <= 0, use CPU core number. -# Datatype: int -# flush_thread_count=0 - -# In one insert (one device, one timestamp, multiple measurements), -# if enable partial insert, one measurement failure will not impact other measurements -# Datatype: boolean -# enable_partial_insert=true - -# the interval to log recover progress of each vsg when starting iotdb -# Datatype: int -# recovery_log_interval_in_ms=5000 - -# If using v0.13 client to insert data, please set this configuration to true. -# Notice: if using v0.13/v1.0 client or setting Client Version to V_0_13 manually, enable this config will disable insert redirection. -# Datatype: boolean -# 0.13_data_insert_adapt=false - -# Verify that TSfiles generated by Flush, Load, and Compaction are correct. The following is verified: -# 1. Check whether the file contains a header and a tail -# 2. Check whether files can be deserialized successfully -# 3. Check whether the file contains data -# 4. Whether there is time range overlap between data, whether it is increased, and whether the metadata index offset of the sequence is correct -# Datatype: boolean -# enable_tsfile_validation=false - -#################### -### Compaction Configurations -#################### -# sequence space compaction: only compact the sequence files -# Datatype: boolean -# enable_seq_space_compaction=true - -# unsequence space compaction: only compact the unsequence files -# Datatype: boolean -# enable_unseq_space_compaction=true - -# cross space compaction: compact the unsequence files into the overlapped sequence files -# Datatype: boolean -# enable_cross_space_compaction=true - -# the selector of cross space compaction task -# Options: rewrite -# cross_selector=rewrite - -# the compaction performer of cross space compaction task -# Options: read_point, fast -# cross_performer=fast - -# the selector of inner sequence space compaction task -# Options: size_tiered -# inner_seq_selector=size_tiered - -# the performer of inner sequence space compaction task -# Options: read_chunk, fast -# inner_seq_performer=read_chunk - -# the selector of inner unsequence space compaction task -# Options: size_tiered -# inner_unseq_selector=size_tiered - -# the performer of inner unsequence space compaction task -# Options: read_point, fast -# inner_unseq_performer=fast - -# The priority of compaction execution -# INNER_CROSS: prioritize inner space compaction, reduce the number of files first -# CROSS_INNER: prioritize cross space compaction, eliminate the unsequence files first -# BALANCE: alternate two compaction types -# compaction_priority=BALANCE - -# The size of candidate compaction task queue. -# Datatype: int -# candidate_compaction_task_queue_size = 200 - -# This parameter is used in two places: -# 1. The target tsfile size of inner space compaction. -# 2. The candidate size of seq tsfile in cross space compaction will be smaller than target_compaction_file_size * 1.5. -# In most cases, the target file size of cross compaction won't exceed this threshold, and if it does, it will not be much larger than it. -# default is 2GB -# Datatype: long, Unit: byte -# target_compaction_file_size=2147483648 - -# The target chunk size in compaction and when memtable reaches this threshold, flush the memtable to disk. -# default is 1MB -# Datatype: long, Unit: byte -# target_chunk_size=1048576 - -# The target point nums in one chunk in compaction -# Datatype: long -# target_chunk_point_num=100000 - -# If the chunk size is lower than this threshold, it will be deserialize into points, default is 128 byte -# Datatype: long, Unit:byte -# chunk_size_lower_bound_in_compaction=128 - -# If the chunk point num is lower than this threshold, it will be deserialize into points -# Datatype: long -# chunk_point_num_lower_bound_in_compaction=100 - -# The max file when selecting inner space compaction candidate files -# Datatype: int -# max_inner_compaction_candidate_file_num=30 - -# The max file when selecting cross space compaction candidate files -# At least one unseq file with it's overlapped seq files will be selected even exceeded this number -# Datatype: int -# max_cross_compaction_candidate_file_num=500 - -# The max total size when selecting cross space compaction candidate files -# At least one unseq file with it's overlapped seq files will be selected even exceeded this number -# Datatype: long, Unit: byte -# max_cross_compaction_candidate_file_size=5368709120 - -# The min inner compaction level of unsequence file which can be selected as candidate -# Datatype: int -# min_cross_compaction_unseq_file_level=1 - -# If one merge file selection runs for more than this time, it will be ended and its current -# selection will be used as final selection. -# When < 0, it means time is unbounded. -# Datatype: long, Unit: ms -# cross_compaction_file_selection_time_budget=30000 - -# How many threads will be set up to perform compaction, 10 by default. -# Set to 1 when less than or equal to 0. -# Datatype: int -# compaction_thread_count=10 - -# The interval of compaction task schedule -# Datatype: long, Unit: ms -# compaction_schedule_interval_in_ms=60000 - -# The interval of compaction task submission -# Datatype: long, Unit: ms -# compaction_submission_interval_in_ms=60000 - -# The limit of write throughput merge can reach per second -# values less than or equal to 0 means no limit -# Datatype: int, Unit: megabyte -# compaction_write_throughput_mb_per_sec=16 - -# The limit of read throughput merge can reach per second -# values less than or equal to 0 means no limit -# Datatype: int, Unit: megabyte -# compaction_read_throughput_mb_per_sec=0 - -# The limit of read operation merge can reach per second -# values less than or equal to 0 means no limit -# Datatype: int -# compaction_read_operation_per_sec=0 - -# The number of sub compaction threads to be set up to perform compaction. -# Currently only works for nonAligned data in cross space compaction and unseq inner space compaction. -# Set to 1 when less than or equal to 0. -# Datatype: int -# sub_compaction_thread_count=4 - -# Redundancy value of disk availability, only use for inner compaction. -# When disk availability is lower than the sum of (disk_space_warning_threshold + inner_compaction_task_selection_disk_redundancy), inner compaction tasks containing mods files are selected first. -# DataType: double -# inner_compaction_task_selection_disk_redundancy=0.05 - -# Mods file size threshold, only use for inner compaction. -# When the size of the mods file corresponding to TsFile exceeds this value, inner compaction tasks containing mods files are selected first. -# DataType: long -# inner_compaction_task_selection_mods_file_threshold=10485760 - -# The number of threads to be set up to select compaction task. -# Datatype: int -# compaction_schedule_thread_num=4 - -#################### -### Write Ahead Log Configuration -#################### - -# Write mode of wal -# The details of these three modes are as follows: -# 1. DISABLE: the system will disable wal. -# 2. SYNC: the system will submit wal synchronously, write request will not return until its wal is fsynced to the disk successfully. -# 3. ASYNC: the system will submit wal asynchronously, write request will return immediately no matter its wal is fsynced to the disk successfully. -# The write performance order is DISABLE > ASYNC > SYNC, but only SYNC mode can ensure data durability. -# wal_mode=ASYNC - -# Max number of wal nodes, each node corresponds to one wal directory -# This parameter is only valid in the standalone mode. IoTConsensus uses one wal per data region and RatisConsensus doesn't use wal. -# The default value 0 means the number is determined by the system, the number is in the range of [data region num / 2, data region num]. -# Notice: this value affects write performance significantly. -# For non-SSD disks, values between one third and half of databases number are recommended. -# Datatype: int -# max_wal_nodes_num=0 - -# Duration a wal flush operation will wait before calling fsync in the async mode -# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. -# Notice: this value affects write performance significantly, values in the range of 10ms-2000ms are recommended. -# Datatype: long -# wal_async_mode_fsync_delay_in_ms=1000 - -# Duration a wal flush operation will wait before calling fsync in the sync mode -# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. -# Notice: this value affects write performance significantly, values in the range of 0ms-10ms are recommended. -# Datatype: long -# wal_sync_mode_fsync_delay_in_ms=3 - -# Buffer size of each wal node -# If it's a value smaller than 0, use the default value 32 * 1024 * 1024 bytes (32MB). -# Datatype: int -# wal_buffer_size_in_byte=33554432 - -# Blocking queue capacity of each wal buffer, restricts maximum number of WALEdits cached in the blocking queue. -# Datatype: int -# wal_buffer_queue_capacity=500 - -# Size threshold of each wal file -# When a wal file's size exceeds this, the wal file will be closed and a new wal file will be created. -# If it's a value smaller than 0, use the default value 30 * 1024 * 1024 (30MB). -# Datatype: long -# wal_file_size_threshold_in_byte=31457280 - -# Minimum ratio of effective information in wal files -# This value should be between 0.0 and 1.0 -# If effective information ratio is below this value, MemTable snapshot or flush will be triggered. -# Increase this value when wal occupies too much disk space. But, if this parameter is too large, the write performance may decline. -# Datatype: double -# wal_min_effective_info_ratio=0.1 - -# MemTable size threshold for triggering MemTable snapshot in wal -# When a memTable's size (in byte) exceeds this, wal can flush this memtable to disk, otherwise wal will snapshot this memtable in wal. -# If it's a value smaller than 0, use the default value 8 * 1024 * 1024 bytes (8MB). -# Datatype: long -# wal_memtable_snapshot_threshold_in_byte=8388608 - -# MemTable's max snapshot number in wal -# If one memTable's snapshot number in wal exceeds this value, it will be flushed to disk. -# Datatype: int -# max_wal_memtable_snapshot_num=1 - -# The period when outdated wal files are periodically deleted -# If this value is too large, outdated wal files may not able to be deleted in time. -# If it's a value smaller than 0, use the default value 20 * 1000 ms (20 seconds). -# Datatype: long -# delete_wal_files_period_in_ms=20000 - -# The minimum size of wal files when throttle down in IoTConsensus -# If this value is not set, it will be carefully chosen according to the available disk space. -# If this value is set smaller than 0, it will default to 50 * 1024 * 1024 * 1024 bytes (50GB). -# Datatype: long -# iot_consensus_throttle_threshold_in_byte=53687091200 - -# Maximum wait time of write cache in IoTConsensus -# If this value is less than or equal to 0, use the default value 10 * 1000 ms (10s) -# Datatype: long -# iot_consensus_cache_window_time_in_ms=-1 - -#################### -### IoTConsensus Configuration -#################### - -# The maximum log entries num in IoTConsensus Batch -# Datatype: int -# data_region_iot_max_log_entries_num_per_batch = 1024 - -# The maximum size in IoTConsensus Batch -# Datatype: int -# data_region_iot_max_size_per_batch = 16777216 - -# The maximum pending batches num in IoTConsensus -# Datatype: int -# data_region_iot_max_pending_batches_num = 5 - -# The maximum memory ratio for queue in IoTConsensus -# Datatype: double -# data_region_iot_max_memory_ratio_for_queue = 0.6 - -# The maximum transit size in byte per second for region migration -# values less than or equal to 0 means no limit -# Datatype: long -# region_migration_speed_limit_bytes_per_second = 33554432 - -#################### -### TsFile Configurations -#################### - -# Datatype: int -# group_size_in_byte=134217728 - -# The memory size for each series writer to pack page, default value is 64KB -# Datatype: int -# page_size_in_byte=65536 - -# The maximum number of data points in a page, default 10000 -# Datatype: int -# max_number_of_points_in_page=10000 - -# The threshold for pattern matching in regex -# Datatype: int -# pattern_matching_threshold=1000000 - -# Floating-point precision -# Datatype: int -# float_precision=2 - -# Encoder of value series. default value is PLAIN. -# For int, long data type, also supports TS_2DIFF and RLE(run-length encoding), GORILLA and ZIGZAG. -# value_encoder=PLAIN - -# Compression configuration -# Data compression method, supports UNCOMPRESSED, SNAPPY, ZSTD, LZMA2 or LZ4. Default value is LZ4 -# And it is also used as the default compressor of time column in aligned timeseries. -# compressor=LZ4 - -#################### -### Authorization Configuration -#################### - -# which class to serve for authorization. By default, it is LocalFileAuthorizer. -# Another choice is org.apache.iotdb.commons.auth.authorizer.OpenIdAuthorizer -# authorizer_provider_class=org.apache.iotdb.commons.auth.authorizer.LocalFileAuthorizer - -# If OpenIdAuthorizer is enabled, then openID_url must be set. -# openID_url= - -# encryption provider class -# iotdb_server_encrypt_decrypt_provider=org.apache.iotdb.commons.security.encrypt.MessageDigestEncrypt - -# encryption provided class parameter -# iotdb_server_encrypt_decrypt_provider_parameter= - -# Cache size of user and role -# Datatype: int -# author_cache_size=1000 - -# Cache expire time of user and role -# Datatype: int -# author_cache_expire_time=30 - -#################### -### UDF Configuration -#################### - -# Used to estimate the memory usage of text fields in a UDF query. -# It is recommended to set this value to be slightly larger than the average length of all text -# records. -# Datatype: int -# udf_initial_byte_array_length_for_memory_control=48 - -# How much memory may be used in ONE UDF query (in MB). -# The upper limit is 20% of allocated memory for read. -# Datatype: float -# udf_memory_budget_in_mb=30.0 - -# UDF memory allocation ratio. -# The parameter form is a:b:c, where a, b, and c are integers. -# udf_reader_transformer_collector_memory_proportion=1:1:1 - -# UDF lib dir -# If this property is unset, system will save the data in the default relative path directory under -# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/udf). -# -# If it is absolute, system will save the data in exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the -# UDF folder. -# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative -# path. -# -# For Windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# udf_lib_dir=ext\\udf -# -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# udf_lib_dir=ext/udf - -#################### -### Trigger Configuration -#################### - -# Uncomment the following field to configure the trigger lib directory. -# For Windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# trigger_lib_dir=ext\\trigger -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# trigger_lib_dir=ext/trigger - -# How many times will we retry to found an instance of stateful trigger on DataNodes -# 3 by default. -# Datatype: int -# stateful_trigger_retry_num_when_not_found=3 - - -#################### -### Select-Into Configuration -#################### - -# The maximum memory occupied by the data to be written when executing select-into statements. -# Datatype: long -# into_operation_buffer_size_in_byte=104857600 - -# The maximum number of rows can be processed in insert-tablet-plan when executing select-into statements. -# When <= 0, use 10000. -# Datatype: int -# select_into_insert_tablet_plan_row_limit=10000 - -# The number of threads in the thread pool that execute insert-tablet tasks -# Datatype: int -# into_operation_execution_thread_count=2 - -#################### -### Continuous Query Configuration -#################### - -# The number of threads in the scheduled thread pool that submit continuous query tasks periodically -# Datatype: int -# continuous_query_submit_thread_count=2 - -# The minimum value of the continuous query execution time interval -# Datatype: long(duration) -# continuous_query_min_every_interval_in_ms=1000 - -#################### -### Pipe Configuration -#################### - -# Uncomment the following field to configure the pipe lib directory. -# For Windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# pipe_lib_dir=ext\\pipe -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# pipe_lib_dir=ext/pipe - -# The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. -# The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). -# pipe_subtask_executor_max_thread_num=5 - -# The connection timeout (in milliseconds) for the thrift client. -# pipe_sink_timeout_ms=900000 - -# The maximum number of selectors that can be used in the sink. -# Recommend to set this value to less than or equal to pipe_sink_max_client_number. -# pipe_sink_selector_number=4 - -# The maximum number of clients that can be used in the sink. -# pipe_sink_max_client_number=16 - -# Whether to enable receiving pipe data through air gap. -# The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. -# pipe_air_gap_receiver_enabled=false - -# The port for the server to receive pipe data through air gap. -# pipe_air_gap_receiver_port=9780 - -#################### -### RatisConsensus Configuration -#################### - -# max payload size for a single log-sync-RPC from leader to follower(in byte, by default 16MB) -# config_node_ratis_log_appender_buffer_size_max=16777216 -# schema_region_ratis_log_appender_buffer_size_max=16777216 -# data_region_ratis_log_appender_buffer_size_max=16777216 - -# trigger a snapshot when snapshot_trigger_threshold logs are written -# config_node_ratis_snapshot_trigger_threshold=400000 -# schema_region_ratis_snapshot_trigger_threshold=400000 -# data_region_ratis_snapshot_trigger_threshold=400000 - -# allow flushing Raft Log asynchronously -# config_node_ratis_log_unsafe_flush_enable=false -# schema_region_ratis_log_unsafe_flush_enable=false -# data_region_ratis_log_unsafe_flush_enable=false - -# max capacity of a single Log segment file (in byte, by default 24MB) -# config_node_ratis_log_segment_size_max_in_byte=25165824 -# schema_region_ratis_log_segment_size_max_in_byte=25165824 -# data_region_ratis_log_segment_size_max_in_byte=25165824 -# config_node_simple_consensus_log_segment_size_max_in_byte=25165824 - -# flow control window for ratis grpc log appender -# config_node_ratis_grpc_flow_control_window=4194304 -# schema_region_ratis_grpc_flow_control_window=4194304 -# data_region_ratis_grpc_flow_control_window=4194304 -# config_node_ratis_grpc_leader_outstanding_appends_max=128 -# schema_region_ratis_grpc_leader_outstanding_appends_max=128 -# data_region_ratis_grpc_leader_outstanding_appends_max=128 -# config_node_ratis_log_force_sync_num=128 -# schema_region_ratis_log_force_sync_num=128 -# data_region_ratis_log_force_sync_num=128 - -# min election timeout for leader election -# config_node_ratis_rpc_leader_election_timeout_min_ms=2000 -# schema_region_ratis_rpc_leader_election_timeout_min_ms=2000 -# data_region_ratis_rpc_leader_election_timeout_min_ms=2000 - -# max election timeout for leader election -# config_node_ratis_rpc_leader_election_timeout_max_ms=4000 -# schema_region_ratis_rpc_leader_election_timeout_max_ms=4000 -# data_region_ratis_rpc_leader_election_timeout_max_ms=4000 - -# ratis client retry threshold -# config_node_ratis_request_timeout_ms=10000 -# schema_region_ratis_request_timeout_ms=10000 -# data_region_ratis_request_timeout_ms=10000 - -# currently we use exponential back-off retry policy for ratis -# config_node_ratis_max_retry_attempts=10 -# config_node_ratis_initial_sleep_time_ms=100 -# config_node_ratis_max_sleep_time_ms=10000 -# schema_region_ratis_max_retry_attempts=10 -# schema_region_ratis_initial_sleep_time_ms=100 -# schema_region_ratis_max_sleep_time_ms=10000 -# data_region_ratis_max_retry_attempts=10 -# data_region_ratis_initial_sleep_time_ms=100 -# data_region_ratis_max_sleep_time_ms=10000 - -# first election timeout -# ratis_first_election_timeout_min_ms=50 -# ratis_first_election_timeout_max_ms=150 - -# preserve certain logs when take snapshot and purge -# config_node_ratis_preserve_logs_num_when_purge=1000 -# schema_region_ratis_preserve_logs_num_when_purge=1000 -# data_region_ratis_preserve_logs_num_when_purge=1000 - -# Raft Log disk size control -# config_node_ratis_log_max_size = 2147483648 -# schema_region_ratis_log_max_size = 2147483648 -# data_region_ratis_log_max_size = 21474836480 - -# Raft periodic snapshot interval, time unit is second -# config_node_ratis_periodic_snapshot_interval=86400 -# schema_region_ratis_periodic_snapshot_interval=86400 -# data_region_ratis_periodic_snapshot_interval=86400 - -#################### -### Procedure Configuration -#################### - -# Default number of worker thread count -# Datatype: int -# procedure_core_worker_thread_count=4 - -# Default time interval of completed procedure cleaner work in, time unit is second -# Datatype: int -# procedure_completed_clean_interval=30 - -# Default ttl of completed procedure, time unit is second -# Datatype: int -# procedure_completed_evict_ttl=800 - -#################### -### MQTT Broker Configuration -#################### - -# whether to enable the mqtt service. -# Datatype: boolean -# enable_mqtt_service=false - -# the mqtt service binding host. -# Datatype: String -# mqtt_host=127.0.0.1 - -# the mqtt service binding port. -# Datatype: int -# mqtt_port=1883 - -# the handler pool size for handing the mqtt messages. -# Datatype: int -# mqtt_handler_pool_size=1 - -# the mqtt message payload formatter. -# Datatype: String -# mqtt_payload_formatter=json - -# max length of mqtt message in byte -# Datatype: int -# mqtt_max_message_size=1048576 - -#################### -### IoTDB-ML Configuration -#################### - -# The thread count which can be used for model inference operation. -# model_inference_execution_thread_count=5 - -#################### -### Load TsFile Configuration -#################### - -# Load clean up task is used to clean up the unsuccessful loaded tsfile after a certain period of time. -# The parameter is the delay time after an unsuccessful load operation (in seconds). -# load_clean_up_task_execution_delay_time_seconds=1800 From f08b91bece2927c8fa816c24958ae383f3673568 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 22 May 2024 11:17:32 +0800 Subject: [PATCH 09/45] fix bug --- .../assembly/resources/sbin/start-confignode.bat | 16 ++++++++-------- .../assembly/resources/sbin/stop-confignode.bat | 4 ++-- .../assembly/resources/sbin/stop-datanode.bat | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat index f73f13d21c003..98399a9990bff 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/start-confignode.bat @@ -84,14 +84,14 @@ IF EXIST "%CONFIGNODE_CONF%\confignode-env.bat" ( @REM CHECK THE PORT USAGES @REM SET CONFIG FILE -IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( - set CONFIG_FILE="%IOTDB_CONF%\iotdb-system.properties" -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-system.properties" ( - set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-system.properties" -) ELSE IF EXIST "%IOTDB_CONF%\iotdb-confignode.properties" ( - set CONFIG_FILE="%IOTDB_CONF%\iotdb-confignode.properties" -) ELSE IF EXIST "%IOTDB_HOME%\conf\iotdb-confignode.properties" ( - set CONFIG_FILE="%IOTDB_HOME%\conf\iotdb-confignode.properties" +IF EXIST "%CONFIGNODE_CONF%\iotdb-system.properties" ( + set CONFIG_FILE="%CONFIGNODE_CONF%\iotdb-system.properties" +) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-system.properties" ( + set CONFIG_FILE="%CONFIGNODE_HOME%\conf\iotdb-system.properties" +) ELSE IF EXIST "%CONFIGNODE_CONF%\iotdb-confignode.properties" ( + set CONFIG_FILE="%CONFIGNODE_CONF%\iotdb-confignode.properties" +) ELSE IF EXIST "%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" ( + set CONFIG_FILE="%CONFIGNODE_HOME%\conf\iotdb-confignode.properties" ) ELSE ( set CONFIG_FILE= ) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index f7db1e156d366..e697642668876 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -23,9 +23,9 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ IF EXIST "%superior%\conf\iotdb-system.properties" ( - set config_file="%superior_dir%\iotdb-system.properties" + set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\iotdb-confignode.properties" + set config_file="%superior_dir%\conf\iotdb-confignode.properties" ) for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index 27d09f156b1d6..d249d42756748 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -23,9 +23,9 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ IF EXIST "%superior%\conf\iotdb-system.properties" ( - set config_file="%superior_dir%\iotdb-system.properties" + set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\iotdb-datanode.properties" + set config_file="%superior_dir%\conf\iotdb-datanode.properties" ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" From ae8e574405ae8edd9d670944f18295a9ce28f624 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 22 May 2024 11:41:07 +0800 Subject: [PATCH 10/45] fix cli scripts --- iotdb-client/cli/src/assembly/resources/tools/backup.bat | 7 ++++++- iotdb-client/cli/src/assembly/resources/tools/backup.sh | 7 ++++++- .../cli/src/assembly/resources/tools/collect-info.bat | 6 +++++- .../cli/src/assembly/resources/tools/collect-info.sh | 6 +++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.bat b/iotdb-client/cli/src/assembly/resources/tools/backup.bat index 13178b7f9bb4c..01be45f9ca01c 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.bat @@ -38,7 +38,12 @@ IF EXIST "%IOTDB_CONF%\datanode-env.bat" ( echo Can't find datanode-env.bat ) -IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( +IF EXIST "%IOTDB_CONF%\iotdb-system.properties" ( + for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" + "%IOTDB_CONF%\iotdb-system.properties"') do ( + set dn_rpc_port=%%i + ) +) ELSE IF EXIST "%IOTDB_CONF%\iotdb-datanode.properties" ( for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" "%IOTDB_CONF%\iotdb-datanode.properties"') do ( set dn_rpc_port=%%i diff --git a/iotdb-client/cli/src/assembly/resources/tools/backup.sh b/iotdb-client/cli/src/assembly/resources/tools/backup.sh index a77a0e3c489c5..d05bc86385321 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/backup.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/backup.sh @@ -70,7 +70,12 @@ get_properties_value() { local file_name=$1 local property_name=$2 local default_value=$3 - local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${IOTDB_HOME}/conf/${file_name}.properties") + if [ -f "${IOTDB_HOME}/conf/iotdb-system.properties" ]; then + local file_path="${IOTDB_HOME}/conf/iotdb-system.properties" + else + local file_path="${IOTDB_HOME}/conf/${file_name}.properties" + fi + local property_value=$(sed "/^${property_name}=/!d;s/.*=//" "${file_path}") if [ -z "$property_value" ]; then property_value="$default_value" fi diff --git a/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat b/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat index 0c87d0cba9947..246228b9187e7 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/collect-info.bat @@ -46,7 +46,11 @@ set "passwd_param=root" set "host_param=127.0.0.1" set "port_param=6667" -set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +if exist "%IOTDB_HOME%\conf\iotdb-system.properties" ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-system.properties" +) else ( + set "properties_file=%IOTDB_HOME%\conf\iotdb-datanode.properties" +) set "key=dn_data_dirs" for /f "usebackq tokens=1,* delims==" %%a in ("%properties_file%") do ( diff --git a/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh b/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh index a01714b289d70..1926309374a8d 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/collect-info.sh @@ -85,7 +85,11 @@ choose_unit() { echo "$unit" } -properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties" +if [ -f "$IOTDB_HOME/conf/iotdb-system.properties" ]; then + properties_file="$IOTDB_HOME/conf/iotdb-system.properties" +else + properties_file="$IOTDB_HOME/conf/iotdb-datanode.properties" +fi data_dir_key="dn_data_dirs" value=$(get_property_value "$properties_file" "$data_dir_key") if [ -n "$value" ]; then From 13cf9ad392632a05fb7135bc86b6fc93ff214292 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 24 May 2024 09:36:00 +0800 Subject: [PATCH 11/45] fix it --- .../org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java | 2 +- .../apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java index 04e5dae887247..473d2f1b64c7f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java @@ -97,7 +97,7 @@ public final void updateProperties(@Nullable String filePath) throws IOException * @throws IOException if properties storage failed. */ public final void persistent(String filePath) throws IOException { - try (FileWriter confOutput = new FileWriter(filePath)) { + try (FileWriter confOutput = new FileWriter(filePath, true)) { properties.store(confOutput, null); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 45a1188a4ca9d..cdbee3d32ab15 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -301,6 +301,7 @@ public final void changeConfig( outputNodeConfig.updateProperties(immutableNodeProperties); // Persistent + Files.deleteIfExists(new File(getTargetNodeConfigPath()).toPath()); outputCommonConfig.persistent(getTargetCommonConfigPath()); outputNodeConfig.persistent(getTargetNodeConfigPath()); } catch (IOException ex) { From 2d086669819d49204a57414aa8111e4c02214fb9 Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 24 May 2024 10:52:29 +0800 Subject: [PATCH 12/45] Update client-go.yml --- .github/workflows/client-go.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/client-go.yml b/.github/workflows/client-go.yml index d4743f8434fee..bc85763f966d8 100644 --- a/.github/workflows/client-go.yml +++ b/.github/workflows/client-go.yml @@ -53,4 +53,6 @@ jobs: - name: Integration test shell: bash run: | - cd iotdb-client/client-go/ && make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo + cd iotdb-client/client-go/ + git pull origin main + make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo From eb3f24aee5ce830334be5e53f735890e3ae10670 Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 24 May 2024 10:58:38 +0800 Subject: [PATCH 13/45] Update client-go.yml --- .github/workflows/client-go.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/client-go.yml b/.github/workflows/client-go.yml index bc85763f966d8..8e658d7846abf 100644 --- a/.github/workflows/client-go.yml +++ b/.github/workflows/client-go.yml @@ -54,5 +54,6 @@ jobs: shell: bash run: | cd iotdb-client/client-go/ + git config pull.ff only git pull origin main make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo From c30b8af6ade1795d32ee2f3fa3d5cecfc2f64dbc Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 24 May 2024 11:04:56 +0800 Subject: [PATCH 14/45] Update client-go.yml --- .github/workflows/client-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client-go.yml b/.github/workflows/client-go.yml index 8e658d7846abf..6bf0bd60661cb 100644 --- a/.github/workflows/client-go.yml +++ b/.github/workflows/client-go.yml @@ -54,6 +54,6 @@ jobs: shell: bash run: | cd iotdb-client/client-go/ - git config pull.ff only + git config pull.rebase false git pull origin main make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo From 989953ef67f8c8b34b1f34d4d906c196ae1f88f4 Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 24 May 2024 11:13:52 +0800 Subject: [PATCH 15/45] Update client-go.yml --- .github/workflows/client-go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client-go.yml b/.github/workflows/client-go.yml index 6bf0bd60661cb..a12cd8e5d89fb 100644 --- a/.github/workflows/client-go.yml +++ b/.github/workflows/client-go.yml @@ -55,5 +55,5 @@ jobs: run: | cd iotdb-client/client-go/ git config pull.rebase false - git pull origin main + git pull origin main --allow-unrelated-histories make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo From 7e56c2c3fd764d8649e57a220d8b650249b6a1f1 Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 24 May 2024 11:36:41 +0800 Subject: [PATCH 16/45] Update client-go.yml --- .github/workflows/client-go.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/client-go.yml b/.github/workflows/client-go.yml index a12cd8e5d89fb..fddd89795a910 100644 --- a/.github/workflows/client-go.yml +++ b/.github/workflows/client-go.yml @@ -53,7 +53,7 @@ jobs: - name: Integration test shell: bash run: | - cd iotdb-client/client-go/ - git config pull.rebase false - git pull origin main --allow-unrelated-histories + cd iotdb-client + git clone https://github.com/apache/iotdb-client-go.git + cd iotdb-client-go make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo From 907e5d397cfae5f251c708ec072a1ce2103b5936 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 27 May 2024 09:54:28 +0800 Subject: [PATCH 17/45] fix it --- .../it/env/cluster/node/AbstractNodeWrapper.java | 9 +++------ .../it/env/cluster/node/ConfigNodeWrapper.java | 7 +------ .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 7 +------ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 -------------- 4 files changed, 5 insertions(+), 32 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index cdbee3d32ab15..80b78c4fc2dd9 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -301,9 +301,8 @@ public final void changeConfig( outputNodeConfig.updateProperties(immutableNodeProperties); // Persistent - Files.deleteIfExists(new File(getTargetNodeConfigPath()).toPath()); - outputCommonConfig.persistent(getTargetCommonConfigPath()); - outputNodeConfig.persistent(getTargetNodeConfigPath()); + outputCommonConfig.persistent(getSystemConfigPath()); + outputNodeConfig.persistent(getSystemConfigPath()); } catch (IOException ex) { throw new AssertionError("Change the config of node failed. " + ex); } @@ -662,9 +661,7 @@ private String getKillPoints() { protected abstract void renameFile(); - protected abstract String getTargetNodeConfigPath(); - - protected abstract String getTargetCommonConfigPath(); + protected abstract String getSystemConfigPath(); /** Return the node config file path specified through system variable */ protected abstract String getDefaultNodeConfigPath(); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index 6bd90c9d36cdc..a807fb2af2023 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -89,12 +89,7 @@ public ConfigNodeWrapper( } @Override - protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); - } - - @Override - protected String getTargetCommonConfigPath() { + protected String getSystemConfigPath() { return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 0a12cfae43c38..1135f5cd055c6 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -123,12 +123,7 @@ public DataNodeWrapper( } @Override - protected String getTargetNodeConfigPath() { - return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); - } - - @Override - protected String getTargetCommonConfigPath() { + protected String getSystemConfigPath() { return workDirFilePath("conf", IOTDB_SYSTEM_PROPERTIES_FILE); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 1ad67a99da5e1..d3f502160efa6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1752,20 +1752,6 @@ public void loadHotModifiedProps() throws QueryProcessException { } } - private void loadHotModifiedPropertiesFromUrl(Properties commonProperties, URL url) - throws QueryProcessException { - try (InputStream inputStream = url.openStream()) { - LOGGER.info("Start to reload config file {}", url); - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - commonProperties.putAll(properties); - } catch (Exception e) { - LOGGER.warn("Fail to reload config file {}", url, e); - throw new QueryProcessException( - String.format("Fail to reload config file %s because %s", url, e.getMessage())); - } - } - private void initMemoryAllocate(Properties properties) { String memoryAllocateProportion = properties.getProperty("datanode_memory_proportion", null); if (memoryAllocateProportion == null) { From 573a7b1af02075ebcfe79b1d40b4585baf6f5088 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 28 May 2024 12:27:06 +0800 Subject: [PATCH 18/45] fix metric it --- .../confignode/conf/ConfigNodeDescriptor.java | 2 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 5 +- .../config/MetricConfigDescriptor.java | 73 +++++++++++++------ .../metrics/config/MetricConfigTest.java | 4 +- 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index db777cfa26d73..ce713b2e0e894 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -130,7 +130,7 @@ private void loadProps() { commonDescriptor .getConfig() .updatePath(System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null)); - MetricConfigDescriptor.getInstance().loadProps(commonProperties); + MetricConfigDescriptor.getInstance().loadProps(commonProperties, true); MetricConfigDescriptor.getInstance() .getMetricConfig() .updateRpcInstance( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index fa804f60ba1a7..296030c03f711 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -211,7 +211,7 @@ private void loadProps() { // update all data seriesPath conf.updatePath(); commonDescriptor.getConfig().updatePath(System.getProperty(IoTDBConstant.IOTDB_HOME, null)); - MetricConfigDescriptor.getInstance().loadProps(commonProperties); + MetricConfigDescriptor.getInstance().loadProps(commonProperties, false); MetricConfigDescriptor.getInstance() .getMetricConfig() .updateRpcInstance( @@ -1791,7 +1791,8 @@ public void loadHotModifiedProps() throws QueryProcessException { throw new QueryProcessException( String.format("Fail to reload config file %s because %s", url, e.getMessage())); } - ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(commonProperties); + ReloadLevel reloadLevel = + MetricConfigDescriptor.getInstance().loadHotProps(commonProperties, false); LOGGER.info("Reload metric service in level {}", reloadLevel); if (reloadLevel == ReloadLevel.RESTART_INTERNAL_REPORTER) { IoTDBInternalReporter internalReporter; diff --git a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java index e053587713da0..f1ec288d47ad2 100644 --- a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java +++ b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java @@ -37,8 +37,8 @@ private MetricConfigDescriptor() { } /** Load properties into metric config. */ - public void loadProps(Properties properties) { - MetricConfig loadConfig = generateFromProperties(properties); + public void loadProps(Properties properties, boolean isConfigNode) { + MetricConfig loadConfig = generateFromProperties(properties, isConfigNode); metricConfig.copy(loadConfig); } @@ -47,8 +47,8 @@ public void loadProps(Properties properties) { * * @return reload level of metric service */ - public ReloadLevel loadHotProps(Properties properties) { - MetricConfig newMetricConfig = generateFromProperties(properties); + public ReloadLevel loadHotProps(Properties properties, boolean isConfigNode) { + MetricConfig newMetricConfig = generateFromProperties(properties, isConfigNode); ReloadLevel reloadLevel = ReloadLevel.NOTHING; if (!metricConfig.equals(newMetricConfig)) { if (!metricConfig.getMetricLevel().equals(newMetricConfig.getMetricLevel()) @@ -72,7 +72,7 @@ public ReloadLevel loadHotProps(Properties properties) { } /** Load properties into metric config. */ - private MetricConfig generateFromProperties(Properties properties) { + private MetricConfig generateFromProperties(Properties properties, boolean isConfigNode) { MetricConfig loadConfig = new MetricConfig(); String reporterList = @@ -83,73 +83,98 @@ private MetricConfig generateFromProperties(Properties properties) { loadConfig.getMetricReporterList().stream() .map(ReporterType::toString) .collect(Collectors.toSet())), - properties); + properties, + isConfigNode); loadConfig.setMetricReporterList(reporterList); loadConfig.setMetricLevel( MetricLevel.valueOf( - getProperty("metric_level", String.valueOf(loadConfig.getMetricLevel()), properties))); + getProperty( + "metric_level", + String.valueOf(loadConfig.getMetricLevel()), + properties, + isConfigNode))); loadConfig.setAsyncCollectPeriodInSecond( Integer.parseInt( getProperty( "metric_async_collect_period", String.valueOf(loadConfig.getAsyncCollectPeriodInSecond()), - properties))); + properties, + isConfigNode))); loadConfig.setPrometheusReporterPort( Integer.parseInt( getProperty( "metric_prometheus_reporter_port", String.valueOf(loadConfig.getPrometheusReporterPort()), - properties))); + properties, + isConfigNode))); IoTDBReporterConfig reporterConfig = loadConfig.getIoTDBReporterConfig(); reporterConfig.setHost( - getProperty("metric_iotdb_reporter_host", reporterConfig.getHost(), properties)); + getProperty( + "metric_iotdb_reporter_host", reporterConfig.getHost(), properties, isConfigNode)); reporterConfig.setPort( Integer.valueOf( getProperty( "metric_iotdb_reporter_port", String.valueOf(reporterConfig.getPort()), - properties))); + properties, + isConfigNode))); reporterConfig.setUsername( - getProperty("metric_iotdb_reporter_username", reporterConfig.getUsername(), properties)); + getProperty( + "metric_iotdb_reporter_username", + reporterConfig.getUsername(), + properties, + isConfigNode)); reporterConfig.setPassword( - getProperty("metric_iotdb_reporter_password", reporterConfig.getPassword(), properties)); + getProperty( + "metric_iotdb_reporter_password", + reporterConfig.getPassword(), + properties, + isConfigNode)); reporterConfig.setMaxConnectionNumber( Integer.valueOf( getProperty( "metric_iotdb_reporter_max_connection_number", String.valueOf(reporterConfig.getMaxConnectionNumber()), - properties))); + properties, + isConfigNode))); reporterConfig.setLocation( - getProperty("metric_iotdb_reporter_location", reporterConfig.getLocation(), properties)); + getProperty( + "metric_iotdb_reporter_location", + reporterConfig.getLocation(), + properties, + isConfigNode)); reporterConfig.setPushPeriodInSecond( Integer.valueOf( getProperty( "metric_iotdb_reporter_push_period", String.valueOf(reporterConfig.getPushPeriodInSecond()), - properties))); - loadConfig.setInternalReportType( - InternalReporterType.valueOf( - properties.getProperty( - "dn_metric_internal_reporter_type", - loadConfig.getInternalReportType().toString()))); + properties, + isConfigNode))); + if (!isConfigNode) { + loadConfig.setInternalReportType( + InternalReporterType.valueOf( + properties.getProperty( + "dn_metric_internal_reporter_type", + loadConfig.getInternalReportType().toString()))); + } return loadConfig; } /** Get property from confignode or datanode. */ - private String getProperty(String target, String defaultValue, Properties properties) { - return properties.getProperty( - "dn_" + target, properties.getProperty("cn_" + target, defaultValue)); + private String getProperty( + String target, String defaultValue, Properties properties, boolean isConfigNode) { + return properties.getProperty((isConfigNode ? "cn_" : "dn_") + target, defaultValue); } private static class MetricConfigDescriptorHolder { diff --git a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java index 3ce4b79c6b780..43382d00aeddc 100644 --- a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java +++ b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java @@ -47,7 +47,7 @@ public void testConfigNodeMetricConfig() { properties.setProperty("cn_metric_iotdb_reporter_location", "metric"); properties.setProperty("cn_metric_iotdb_reporter_push_period", "5"); - MetricConfigDescriptor.getInstance().loadProps(properties); + MetricConfigDescriptor.getInstance().loadProps(properties, true); MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); @@ -84,7 +84,7 @@ public void testDataNodeMetricConfig() { properties.setProperty("dn_metric_iotdb_reporter_push_period", "5"); properties.setProperty("dn_metric_internal_reporter_type", "IOTDB"); - MetricConfigDescriptor.getInstance().loadProps(properties); + MetricConfigDescriptor.getInstance().loadProps(properties, false); MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); From 750a4788d4ace495413fe01b020647313745e494 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 30 May 2024 22:56:08 +0800 Subject: [PATCH 19/45] set configuration on node id --- .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 11 ++- .../client/DataNodeRequestType.java | 1 + .../client/async/AsyncDataNodeClientPool.java | 7 ++ .../confignode/manager/ConfigManager.java | 9 +++ .../iotdb/confignode/manager/IManager.java | 4 ++ .../confignode/manager/node/NodeManager.java | 16 +++++ .../thrift/ConfigNodeRPCServiceProcessor.java | 6 ++ .../apache/iotdb/db/audit/AuditLogger.java | 1 + .../apache/iotdb/db/conf/IoTDBDescriptor.java | 7 +- .../db/protocol/client/ConfigNodeClient.java | 7 ++ .../impl/DataNodeInternalRPCServiceImpl.java | 6 ++ .../execution/config/ConfigTaskVisitor.java | 8 +++ .../executor/ClusterConfigTaskExecutor.java | 25 +++++++ .../config/executor/IConfigTaskExecutor.java | 3 + .../config/sys/SetConfigurationTask.java | 45 ++++++++++++ .../queryengine/plan/parser/ASTVisitor.java | 20 ++++++ .../plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 ++ .../sys/SetConfigurationStatement.java | 72 +++++++++++++++++++ .../iotdb/db/storageengine/StorageEngine.java | 42 +++++++++++ .../src/main/thrift/common.thrift | 5 ++ .../src/main/thrift/confignode.thrift | 2 + .../src/main/thrift/datanode.thrift | 2 + 23 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index c2086e8de8919..6d3ffe0b837d8 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -82,7 +82,7 @@ dclStatement ; utilityStatement - : flush | clearCache | settle | startRepairData | stopRepairData | explain + : flush | clearCache | setConfiguration | settle | startRepairData | stopRepairData | explain | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile @@ -973,6 +973,15 @@ clearCache : CLEAR CACHE (ON (LOCAL | CLUSTER))? ; +// Set Configuration +setConfiguration + : SET CONFIGURATION setConfigurationEntry+ (ON INTEGER_LITERAL)? + ; + +setConfigurationEntry + : STRING_LITERAL OPERATOR_SEQ STRING_LITERAL + ; + // Settle settle : SETTLE (prefixPath|tsFilePath=STRING_LITERAL) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java index 1eb2500eb83d1..99c9caee1c416 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java @@ -52,6 +52,7 @@ public enum DataNodeRequestType { INVALIDATE_PERMISSION_CACHE, INVALIDATE_SCHEMA_CACHE, CLEAR_CACHE, + SET_CONFIGURATION, // Function CREATE_FUNCTION, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java index dd201a66b5745..155597ceb43d1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -341,6 +342,12 @@ private void sendAsyncRequestToDataNode( (AsyncTSStatusRPCHandler) clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); break; + case SET_CONFIGURATION: + client.setConfiguration( + (TSetConfigurationReq) clientHandler.getRequest(requestId), + (AsyncTSStatusRPCHandler) + clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); + break; case START_REPAIR_DATA: client.startRepairData( (AsyncTSStatusRPCHandler) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 8238598e651b7..9f8f9813927a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; @@ -1496,6 +1497,14 @@ public TSStatus clearCache() { : status; } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? RpcUtils.squashResponseStatusList(nodeManager.setConfiguration(req)) + : status; + } + @Override public TSStatus startRepairData() { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 3ac0ee6ade41f..136223b259c7a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.path.PartialPath; @@ -479,6 +480,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Clear cache on all DataNodes. */ TSStatus clearCache(); + /** Set Configuration. */ + TSStatus setConfiguration(TSetConfigurationReq req); + /** Check and repair unsorted tsfile by compaction. */ TSStatus startRepairData(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 3c3c9358f81de..51ad31244dbf8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionRoleType; @@ -725,6 +726,21 @@ public List clearCache() { return clientHandler.getResponseList(); } + public List setConfiguration(TSetConfigurationReq req) { + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + int nodeId = req.getNodeId(); + if (nodeId >= 0 && dataNodeLocationMap.containsKey(nodeId)) { + TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); + dataNodeLocationMap = new HashMap<>(); + dataNodeLocationMap.put(nodeId, dataNodeLocation); + } + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + return clientHandler.getResponseList(); + } + public List startRpairData() { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 82cb2672752c2..98f4a9a00c8d4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -808,6 +809,11 @@ public TSStatus clearCache() { return configManager.clearCache(); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return configManager.setConfiguration(req); + } + @Override public TSStatus startRepairData() { return configManager.startRepairData(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java index 43ab611e54b8a..bb5f557ec0ef5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java @@ -228,6 +228,7 @@ private static AuditLogOperation judgeLogOperation(StatementType type) { case INTERNAL_CREATE_TIMESERIES: case START_REPAIR_DATA: case STOP_REPAIR_DATA: + case SET_CONFIGURATION: return AuditLogOperation.DML; case LIST_USER: case LIST_ROLE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 86ad6087fc083..ba7bff2af0b92 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1846,8 +1846,11 @@ public void loadHotModifiedProps() throws QueryProcessException { throw new QueryProcessException( String.format("Fail to reload config file %s because %s", url, e.getMessage())); } - ReloadLevel reloadLevel = - MetricConfigDescriptor.getInstance().loadHotProps(commonProperties, false); + reloadMetricProperties(commonProperties); + } + + public void reloadMetricProperties(Properties properties) { + ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(properties, false); LOGGER.info("Reload metric service in level {}", reloadLevel); if (reloadLevel == ReloadLevel.RESTART_INTERNAL_REPORTER) { IoTDBInternalReporter internalReporter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index c49b7e0b4451e..5dd59cf43df11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -647,6 +648,12 @@ public TSStatus clearCache() throws TException { () -> client.clearCache(), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.setConfiguration(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus startRepairData() throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 614956e961691..c862282a91af3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -1709,6 +1710,11 @@ public TSStatus clearCache() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + return StorageEngine.getInstance().setConfiguration(req); + } + @Override public TSStatus settle(TSettleReq req) throws TException { return SettleRequestHandler.getInstance().handleSettleRequest(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java index b024ea5a62a38..0786d2fb89386 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.MergeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; @@ -149,6 +150,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -259,6 +261,12 @@ public IConfigTask visitClearCache( return new ClearCacheTask(clearCacheStatement); } + @Override + public IConfigTask visitSetConfiguration( + SetConfigurationStatement setConfigurationStatement, MPPQueryContext context) { + return new SetConfigurationTask(setConfigurationStatement); + } + @Override public IConfigTask visitStartRepairData( StartRepairDataStatement startRepairDataStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index d2e2c1c10b84f..88be389b2bdba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -1022,6 +1023,30 @@ public SettableFuture clearCache(boolean onCluster) { return future; } + @Override + public SettableFuture setConfiguration(TSetConfigurationReq req) { + SettableFuture future = SettableFuture.create(); + TSStatus tsStatus = new TSStatus(); + boolean onLocal = IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == req.getNodeId(); + if (onLocal) { + tsStatus = StorageEngine.getInstance().setConfiguration(req); + } else { + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // Send request to some API server + tsStatus = client.setConfiguration(req); + } catch (ClientManagerException | TException e) { + future.setException(e); + } + } + if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); + } + return future; + } + @Override public SettableFuture startRepairData(boolean onCluster) { SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index df24fec47da68..5adb94a56ce96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; @@ -120,6 +121,8 @@ public interface IConfigTaskExecutor { SettableFuture clearCache(boolean onCluster); + SettableFuture setConfiguration(TSetConfigurationReq tSetConfigurationReq); + SettableFuture loadConfiguration(boolean onCluster); SettableFuture setSystemStatus(boolean onCluster, NodeStatus status); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java new file mode 100644 index 0000000000000..f03c0aed7bd54 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; + +import com.google.common.util.concurrent.ListenableFuture; + +public class SetConfigurationTask implements IConfigTask { + + private final SetConfigurationStatement setConfigurationStatement; + + public SetConfigurationTask(SetConfigurationStatement setConfigurationStatement) { + this.setConfigurationStatement = setConfigurationStatement; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.setConfiguration( + new TSetConfigurationReq( + setConfigurationStatement.getConfigItems(), setConfigurationStatement.getNodeId())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index f2d37f06d75a1..daae04a8eb479 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -196,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -3243,6 +3244,25 @@ public Statement visitClearCache(IoTDBSqlParser.ClearCacheContext ctx) { return clearCacheStatement; } + // Set Configuration + + @Override + public Statement visitSetConfiguration(IoTDBSqlParser.SetConfigurationContext ctx) { + SetConfigurationStatement setConfigurationStatement = + new SetConfigurationStatement(StatementType.SET_CONFIGURATION); + int nodeId = + Integer.parseInt(ctx.INTEGER_LITERAL() == null ? "-1" : ctx.INTEGER_LITERAL().getText()); + Map configItems = new HashMap<>(); + for (IoTDBSqlParser.SetConfigurationEntryContext entry : ctx.setConfigurationEntry()) { + String key = entry.STRING_LITERAL(0).getText().replace("\"", ""); + String value = entry.STRING_LITERAL(1).getText().replace("\"", ""); + configItems.put(key, value); + } + setConfigurationStatement.setNodeId(nodeId); + setConfigurationStatement.setConfigItems(configItems); + return setConfigurationStatement; + } + // Start Repair Data @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 88906ba83f4c9..4f49a5909e750 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -184,4 +184,5 @@ public enum StatementType { SHOW_TOPICS, SHOW_SUBSCRIPTIONS, + SET_CONFIGURATION } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8dba82f490b0..248cf881de12b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -391,6 +392,10 @@ public R visitClearCache(ClearCacheStatement clearCacheStatement, C context) { return visitStatement(clearCacheStatement, context); } + public R visitSetConfiguration(SetConfigurationStatement setConfigurationStatement, C context) { + return visitStatement(setConfigurationStatement, context); + } + public R visitStartRepairData(StartRepairDataStatement startRepairDataStatement, C context) { return visitStatement(startRepairDataStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java new file mode 100644 index 0000000000000..8f519de02807b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class SetConfigurationStatement extends Statement implements IConfigStatement { + + private int nodeId; + private Map configItems; + + public SetConfigurationStatement(StatementType setConfigurationType) { + this.statementType = setConfigurationType; + } + + public void setNodeId(int nodeId) { + this.nodeId = nodeId; + } + + public void setConfigItems(Map configItems) { + this.configItems = configItems; + } + + public int getNodeId() { + return nodeId; + } + + public Map getConfigItems() { + return configItems; + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitSetConfiguration(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index ef54989011939..c36bdd785b0cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -20,11 +20,13 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.commons.concurrent.ExceptionalCountDownLatch; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -80,12 +82,15 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -599,6 +604,43 @@ public void clearCache() { BloomFilterCache.getInstance().clear(); } + public TSStatus setConfiguration(TSetConfigurationReq req) { + Map newConfigItems = req.getConfigs(); + Properties properties = new Properties(); + properties.putAll(newConfigItems); + + if (newConfigItems.isEmpty()) { + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + URL configFileUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (configFileUrl == null || !(new File(configFileUrl.getFile()).exists())) { + // configuration file not exist, update in mem + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + IoTDBDescriptor.getInstance().reloadMetricProperties(properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + + // 1. append new configuration properties to configuration file + File configFile = new File(configFileUrl.getFile()); + try (FileWriter writer = new FileWriter(configFile, true)) { + properties.store(writer, null); + } catch (IOException e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + + // 2. load hot modified properties + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + /** * Add a listener to listen flush start/end events. Notice that this addition only applies to * TsFileProcessors created afterwards. diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index d81730337c627..f8c435b28491e 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -127,6 +127,11 @@ struct TSetTTLReq { 3: required bool isDataBase } +struct TSetConfigurationReq { + 1: required map configs + 2: required i32 nodeId +} + // for File struct TFile { 1: required string fileName diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 25ed9d3c64cf2..cac3d908a9239 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1327,6 +1327,8 @@ service IConfigNodeRPCService { /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ common.TSStatus clearCache() + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + /** Check and repair unsorted tsfile by compaction */ common.TSStatus startRepairData() diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 2b90240fce0fa..069172c055ff9 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -851,6 +851,8 @@ service IDataNodeRPCService { common.TSStatus clearCache() + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + common.TSStatus loadConfiguration() common.TSStatus setSystemStatus(string status) From 9929c4f0e7632d1019c859035eaee185d417cad8 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 4 Jun 2024 18:44:55 +0800 Subject: [PATCH 20/45] impl showConfiguration and showConfigurationTemplate --- .../client/ConfigNodeRequestType.java | 4 +- .../client/DataNodeRequestType.java | 3 +- .../client/sync/SyncConfigNodeClientPool.java | 3 + .../client/sync/SyncDataNodeClientPool.java | 22 ++++- .../confignode/manager/ConfigManager.java | 43 +++++++++ .../iotdb/confignode/manager/IManager.java | 4 + .../confignode/manager/node/NodeManager.java | 96 +++++++++++++++---- .../procedure/env/ConfigNodeProcedureEnv.java | 22 +++-- .../procedure/env/RegionMaintainHandler.java | 78 ++++++++------- .../impl/sync/AuthOperationProcedure.java | 11 ++- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 ++ .../db/protocol/client/ConfigNodeClient.java | 7 ++ .../thrift/impl/ClientRPCServiceImpl.java | 84 +++++++++++++++- .../impl/DataNodeInternalRPCServiceImpl.java | 20 ++++ .../src/main/thrift/common.thrift | 13 ++- .../src/main/thrift/confignode.thrift | 4 + .../src/main/thrift/client.thrift | 4 + .../src/main/thrift/datanode.thrift | 2 + 18 files changed, 352 insertions(+), 74 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java index e9aa1e93f9d72..5f3064b4eed73 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java @@ -27,5 +27,7 @@ public enum ConfigNodeRequestType { REMOVE_CONFIG_NODE, DELETE_CONFIG_NODE_PEER, REPORT_CONFIG_NODE_SHUTDOWN, - STOP_CONFIG_NODE + STOP_CONFIG_NODE, + SET_CONFIGURATION, + SHOW_CONFIGURATION, } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java index 99c9caee1c416..6f22341e3958b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java @@ -32,6 +32,8 @@ public enum DataNodeRequestType { STOP_REPAIR_DATA, LOAD_CONFIGURATION, SET_SYSTEM_STATUS, + SET_CONFIGURATION, + SHOW_CONFIGURATION, // Region Maintenance CREATE_DATA_REGION, @@ -52,7 +54,6 @@ public enum DataNodeRequestType { INVALIDATE_PERMISSION_CACHE, INVALIDATE_SCHEMA_CACHE, CLEAR_CACHE, - SET_CONFIGURATION, // Function CREATE_FUNCTION, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java index 04d2c0917698d..ab85a9badcb9e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -88,6 +89,8 @@ public Object sendSyncRequestToConfigNodeWithRetry( case STOP_CONFIG_NODE: // Only use stopConfigNode when the ConfigNode is removed. return client.stopConfigNode((TConfigNodeLocation) req); + case SET_CONFIGURATION: + return client.setConfiguration((TSetConfigurationReq) req); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index 1d0e564e43cd1..5f4d7c2f12974 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -64,7 +65,7 @@ private SyncDataNodeClientPool() { new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); } - public TSStatus sendSyncRequestToDataNodeWithRetry( + public Object sendSyncRequestToDataNodeWithRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType) { Throwable lastException = new TException(); for (int retry = 0; retry < DEFAULT_RETRY_NUM; retry++) { @@ -83,7 +84,7 @@ public TSStatus sendSyncRequestToDataNodeWithRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - public TSStatus sendSyncRequestToDataNodeWithGivenRetry( + public Object sendSyncRequestToDataNodeWithGivenRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType, int retryNum) { Throwable lastException = new TException(); for (int retry = 0; retry < retryNum; retry++) { @@ -102,7 +103,7 @@ public TSStatus sendSyncRequestToDataNodeWithGivenRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - private TSStatus executeSyncRequest( + private Object executeSyncRequest( DataNodeRequestType requestType, SyncDataNodeInternalServiceClient client, Object req) throws TException { switch (requestType) { @@ -138,6 +139,8 @@ private TSStatus executeSyncRequest( return client.deleteOldRegionPeer((TMaintainPeerReq) req); case RESET_PEER_LIST: return client.resetPeerList((TResetPeerListReq) req); + case SHOW_CONFIGURATION: + return client.showConfiguration(); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); @@ -187,6 +190,19 @@ public TRegionLeaderChangeResp changeRegionLeader( return new TRegionLeaderChangeResp(status, -1L); } + public TShowConfigurationResp showConfiguration(TEndPoint dataNode) { + TSStatus status; + try (SyncDataNodeInternalServiceClient client = clientManager.borrowClient(dataNode)) { + return client.showConfiguration(); + } catch (ClientManagerException e) { + LOGGER.error("Can't connect to Data node: {}", dataNode, e); + status = RpcUtils.getStatus(TSStatusCode.CAN_NOT_CONNECT_DATANODE, e.getMessage()); + } catch (TException e) { + status = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return new TShowConfigurationResp(status, ""); + } + // TODO: Is the ClientPool must be a singleton? private static class ClientPoolHolder { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 9f8f9813927a7..d57f702a8fa0c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -31,6 +31,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.cluster.NodeStatus; @@ -209,8 +210,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -220,6 +225,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -1499,6 +1505,21 @@ public TSStatus clearCache() { @Override public TSStatus setConfiguration(TSetConfigurationReq req) { + if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == req.getNodeId()) { + URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (url == null || !new File(url.getFile()).exists()) { + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + File file = new File(url.getFile()); + try (FileWriter fileWriter = new FileWriter(file, true)) { + Properties properties = new Properties(); + properties.putAll(req.getConfigs()); + properties.store(fileWriter, null); + } catch (IOException e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() ? RpcUtils.squashResponseStatusList(nodeManager.setConfiguration(req)) @@ -1529,6 +1550,28 @@ public TSStatus loadConfiguration() { : status; } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) { + if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == nodeId) { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (propsUrl != null && new File(propsUrl.getFile()).exists()) { + byte[] bytes = Files.readAllBytes(new File(propsUrl.getFile()).toPath()); + resp.setContent(new String(bytes)); + } + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? nodeManager.showConfiguration(nodeId) + : new TShowConfigurationResp(status, ""); + } + @Override public TSStatus setSystemStatus(String systemStatus) { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 136223b259c7a..919e876bac213 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; @@ -492,6 +493,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Load configuration on all DataNodes. */ TSStatus loadConfiguration(); + /** Show content of configuration file on specified node */ + TShowConfigurationResp showConfiguration(int nodeId); + /** Set system status on all DataNodes. */ TSStatus setSystemStatus(String status); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 51ad31244dbf8..3e140b1dc819d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -27,6 +27,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionRoleType; @@ -34,9 +35,11 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.confignode.client.ConfigNodeRequestType; import org.apache.iotdb.confignode.client.DataNodeRequestType; import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool; import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; @@ -727,18 +730,44 @@ public List clearCache() { } public List setConfiguration(TSetConfigurationReq req) { + List responseList = new ArrayList<>(); + Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); + Map targetDataNodes = new HashMap<>(); int nodeId = req.getNodeId(); - if (nodeId >= 0 && dataNodeLocationMap.containsKey(nodeId)) { - TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); - dataNodeLocationMap = new HashMap<>(); - dataNodeLocationMap.put(nodeId, dataNodeLocation); + // send to datanode + if (dataNodeLocationMap.containsKey(nodeId)) { + targetDataNodes.put(nodeId, dataNodeLocationMap.get(nodeId)); + } else if (nodeId < 0) { + targetDataNodes.putAll(dataNodeLocationMap); } - AsyncClientHandler clientHandler = - new AsyncClientHandler<>(DataNodeRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); - AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); - return clientHandler.getResponseList(); + if (!targetDataNodes.isEmpty()) { + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + responseList.addAll(clientHandler.getResponseList()); + } + + // send to config node + List configNodes = getRegisteredConfigNodes(); + for (TConfigNodeLocation configNode : configNodes) { + if (configNode.getConfigNodeId() == CONF.getConfigNodeId()) { + continue; + } + if (nodeId >= 0 && nodeId != configNode.getConfigNodeId()) { + continue; + } + TSStatus status = + (TSStatus) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + configNode.getInternalEndPoint(), + req, + ConfigNodeRequestType.SET_CONFIGURATION); + responseList.add(status); + } + return responseList; } public List startRpairData() { @@ -768,6 +797,35 @@ public List loadConfiguration() { return clientHandler.getResponseList(); } + public TShowConfigurationResp showConfiguration(int nodeId) { + TShowConfigurationResp resp = new TShowConfigurationResp(); + + // data node + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + if (dataNodeLocationMap.containsKey(nodeId)) { + TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); + return SyncDataNodeClientPool.getInstance() + .showConfiguration(dataNodeLocation.getInternalEndPoint()); + } + + // other config node + for (TConfigNodeLocation registeredConfigNode : getRegisteredConfigNodes()) { + if (registeredConfigNode.getConfigNodeId() != nodeId) { + continue; + } + resp = + (TShowConfigurationResp) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + registeredConfigNode.getInternalEndPoint(), + nodeId, + ConfigNodeRequestType.SHOW_CONFIGURATION); + return resp; + } + return resp; + } + public List setSystemStatus(String status) { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); @@ -779,11 +837,12 @@ public List setSystemStatus(String status) { } public TSStatus setDataNodeStatus(TSetDataNodeStatusReq setDataNodeStatusReq) { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), - setDataNodeStatusReq.getStatus(), - DataNodeRequestType.SET_SYSTEM_STATUS); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), + setDataNodeStatusReq.getStatus(), + DataNodeRequestType.SET_SYSTEM_STATUS); } /** @@ -816,11 +875,12 @@ private TSStatus killSpecificQuery(String queryId, TDataNodeLocation dataNodeLoc .setMessage( "The target DataNode is not existed, please ensure your input is correct"); } else { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeLocation.getInternalEndPoint(), - queryId, - DataNodeRequestType.KILL_QUERY_INSTANCE); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + queryId, + DataNodeRequestType.KILL_QUERY_INSTANCE); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 1e2a7d7f15019..00d9569c6593c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -181,18 +181,20 @@ public boolean invalidateCache(String storageGroupName) throws IOException, TExc if (nodeStatus == NodeStatus.Running) { // Always invalidate PartitionCache first final TSStatus invalidatePartitionStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_PARTITION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_PARTITION_CACHE); final TSStatus invalidateSchemaStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) { LOG.error( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java index 66a8ffd3001b3..745ef9074aa5c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java @@ -140,11 +140,12 @@ public void broadcastDisableDataNode(TDataNodeLocation disabledDataNode) { for (TDataNodeConfiguration node : otherOnlineDataNodes) { TDisableDataNodeReq disableReq = new TDisableDataNodeReq(disabledDataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - node.getLocation().getInternalEndPoint(), - disableReq, - DataNodeRequestType.DISABLE_DATA_NODE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + node.getLocation().getInternalEndPoint(), + disableReq, + DataNodeRequestType.DISABLE_DATA_NODE); if (!isSucceed(status)) { LOGGER.error( "{}, BroadcastDisableDataNode meets error, disabledDataNode: {}, error: {}", @@ -224,11 +225,12 @@ public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocatio TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, storageGroup); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - destDataNode.getInternalEndPoint(), - req, - DataNodeRequestType.CREATE_NEW_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + destDataNode.getInternalEndPoint(), + req, + DataNodeRequestType.CREATE_NEW_REGION_PEER); if (isSucceed(status)) { LOGGER.info( @@ -269,11 +271,12 @@ public TSStatus submitAddRegionPeerTask( // destDataNode is where the new RegionReplica is created TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.ADD_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.ADD_REGION_PEER); LOGGER.info( "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}", REGION_MIGRATE_PROCESS, @@ -305,11 +308,12 @@ public TSStatus submitRemoveRegionPeerTask( TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, originalDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.REMOVE_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.REMOVE_REGION_PEER); LOGGER.info( "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}", REGION_MIGRATE_PROCESS, @@ -338,17 +342,19 @@ public TSStatus submitDeleteOldRegionPeerTask( status = configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId()) == NodeStatus.Unknown - ? SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER, - 1) - : SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER); + ? (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER, + 1) + : (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER); LOGGER.info( "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}", REGION_MIGRATE_PROCESS, @@ -498,9 +504,13 @@ public void stopDataNode(TDataNodeLocation dataNode) { REMOVE_DATANODE_PROCESS, dataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - dataNode.getInternalEndPoint(), dataNode, DataNodeRequestType.STOP_DATA_NODE, 2); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getInternalEndPoint(), + dataNode, + DataNodeRequestType.STOP_DATA_NODE, + 2); configManager.getLoadManager().removeNodeCache(dataNode.getDataNodeId()); LOGGER.info( "{}, Stop Data Node result: {}, stoppedDataNode: {}", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java index b3f10181c2016..badc5d2aa96b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java @@ -107,11 +107,12 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, AuthOperationProcedu continue; } status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - pair.getLeft().getLocation().getInternalEndPoint(), - req, - DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + pair.getLeft().getLocation().getInternalEndPoint(), + req, + DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { it.remove(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 98f4a9a00c8d4..2a465f34eaf15 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; @@ -829,6 +830,11 @@ public TSStatus loadConfiguration() { return configManager.loadConfiguration(); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return configManager.showConfiguration(nodeId); + } + @Override public TSStatus setSystemStatus(String status) { return configManager.setSystemStatus(status); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 5dd59cf43df11..cda4f597284cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -28,6 +28,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.client.ClientManager; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.factory.ThriftClientFactory; @@ -672,6 +673,12 @@ public TSStatus loadConfiguration() throws TException { () -> client.loadConfiguration(), status -> !updateConfigNodeLeader(status)); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return executeRemoteCallWithRetry( + () -> client.showConfiguration(nodeId), resp -> !updateConfigNodeLeader(resp.getStatus())); + } + @Override public TSStatus setSystemStatus(String systemStatus) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 98c862f27b23f..bc73f135694c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -23,7 +23,11 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationTemplateResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -45,6 +49,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.protocol.thrift.OperationType; @@ -187,8 +194,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; @@ -2301,6 +2311,78 @@ public TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req) { } } + @Override + public TShowConfigurationTemplateResp showConfigurationTemplate() throws TException { + TShowConfigurationTemplateResp resp = new TShowConfigurationTemplateResp(); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + resp.setContent(getConfigurationFileContent(CommonConfig.SYSTEM_CONFIG_NAME + ".properties")); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + TShowConfigurationResp resp = new TShowConfigurationResp(); + resp.setContent(""); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + if (!clientSession.getUsername().equals(AuthorityChecker.SUPER_USER)) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.NO_PERMISSION)); + return resp; + } + + if (IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == nodeId) { + resp.setContent(getConfigurationFileContent(CommonConfig.SYSTEM_CONFIG_NAME)); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } + + try (ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + return client.showConfiguration(nodeId); + } catch (ClientManagerException e) { + throw new TException(e); + } + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + + private String getConfigurationFileContent(String configurationFileName) throws IOException { + String content = ""; + URL configurationFileUrl = IoTDBDescriptor.getPropsUrl(configurationFileName); + if (configurationFileUrl == null) { + return content; + } + File file = new File(configurationFileUrl.getFile()); + if (!file.exists()) { + return content; + } + return new String(Files.readAllBytes(file.toPath())); + } + private TSQueryTemplateResp executeTemplateQueryStatement( Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) { long startTime = System.nanoTime(); @@ -2674,7 +2756,7 @@ private TSExecuteStatementResp createResponse(DatasetHeader header, long queryId return resp; } - private TSStatus getNotLoggedInStatus() { + protected TSStatus getNotLoggedInStatus() { return RpcUtils.getStatus( TSStatusCode.NOT_LOGIN, "Log in failed. Either you are not authorized or the session has timed out."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index c862282a91af3..167a91a07d322 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -29,6 +29,7 @@ import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSettleReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; @@ -244,8 +245,11 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -1730,6 +1734,22 @@ public TSStatus loadConfiguration() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TShowConfigurationResp showConfiguration() { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (propsUrl != null && new File(propsUrl.getFile()).exists()) { + byte[] bytes = Files.readAllBytes(new File(propsUrl.getFile()).toPath()); + resp.setContent(new String(bytes)); + } + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + @Override public TSStatus setSystemStatus(String status) throws TException { try { diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index f8c435b28491e..49dc50d94c476 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -215,4 +215,15 @@ enum TAggregationType { MAX_BY, MIN_BY, UDAF -} \ No newline at end of file +} + +struct TShowConfigurationTemplateResp { + 1: required TSStatus status + 2: required string content +} + +struct TShowConfigurationResp { + 1: required TSStatus status + 2: required string content +} + diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index cac3d908a9239..ad75ba17c1d31 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1327,8 +1327,12 @@ service IConfigNodeRPCService { /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ common.TSStatus clearCache() + /** Set configuration on specified node */ common.TSStatus setConfiguration(common.TSetConfigurationReq req) + /** Show content of configuration file */ + common.TShowConfigurationResp showConfiguration(1: i32 nodeId) + /** Check and repair unsorted tsfile by compaction */ common.TSStatus startRepairData() diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index a99a8e599c937..6069199bd64c9 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -634,6 +634,10 @@ service IClientRPCService { TSQueryTemplateResp querySchemaTemplate(1:TSQueryTemplateReq req); + common.TShowConfigurationTemplateResp showConfigurationTemplate(); + + common.TShowConfigurationResp showConfiguration(1:i32 nodeId); + common.TSStatus setSchemaTemplate(1:TSSetSchemaTemplateReq req); common.TSStatus unsetSchemaTemplate(1:TSUnsetSchemaTemplateReq req); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 069172c055ff9..d4ef59eee2b79 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -851,6 +851,8 @@ service IDataNodeRPCService { common.TSStatus clearCache() + common.TShowConfigurationResp showConfiguration() + common.TSStatus setConfiguration(common.TSetConfigurationReq req) common.TSStatus loadConfiguration() From c6a77e987661f02cbc43543232497d94030b71ce Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 4 Jun 2024 21:45:10 +0800 Subject: [PATCH 21/45] add ConfigurationFileUtils --- .../confignode/conf/ConfigNodeDescriptor.java | 6 +- .../confignode/manager/ConfigManager.java | 34 +++++------ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +- .../thrift/impl/ClientRPCServiceImpl.java | 25 +++----- .../impl/DataNodeInternalRPCServiceImpl.java | 8 +-- .../iotdb/db/storageengine/StorageEngine.java | 20 ++++--- ...eTool.java => ConfigurationFileUtils.java} | 57 +++++++++++++++---- 7 files changed, 92 insertions(+), 64 deletions(-) rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/{ConfigFileAutoUpdateTool.java => ConfigurationFileUtils.java} (70%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index ce713b2e0e894..e14a0efd612b0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -55,13 +55,13 @@ public class ConfigNodeDescriptor { private final ConfigNodeConfig conf = new ConfigNodeConfig(); static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index d57f702a8fa0c..695c222aa76fe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; @@ -211,11 +212,9 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.net.URL; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1505,25 +1504,31 @@ public TSStatus clearCache() { @Override public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == req.getNodeId()) { URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url == null || !new File(url.getFile()).exists()) { - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + return tsStatus; } File file = new File(url.getFile()); - try (FileWriter fileWriter = new FileWriter(file, true)) { - Properties properties = new Properties(); - properties.putAll(req.getConfigs()); - properties.store(fileWriter, null); - } catch (IOException e) { + Properties properties = new Properties(); + properties.putAll(req.getConfigs()); + List ignoredConfigItems = + ConfigurationFileUtils.filterImmutableConfigItems(properties); + if (!ignoredConfigItems.isEmpty()) { + tsStatus.setMessage("ignored config items" + ignoredConfigItems); + } + try { + ConfigurationFileUtils.updateConfigurationFile(file, properties); + } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + return tsStatus; } - TSStatus status = confirmLeader(); - return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + tsStatus = confirmLeader(); + return tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() ? RpcUtils.squashResponseStatusList(nodeManager.setConfiguration(req)) - : status; + : tsStatus; } @Override @@ -1557,10 +1562,7 @@ public TShowConfigurationResp showConfiguration(int nodeId) { new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); try { URL propsUrl = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); - if (propsUrl != null && new File(propsUrl.getFile()).exists()) { - byte[] bytes = Files.readAllBytes(new File(propsUrl.getFile()).toPath()); - resp.setContent(new String(bytes)); - } + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); } catch (Exception e) { resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index ba7bff2af0b92..bc5f534f3469a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -101,13 +101,13 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index bc73f135694c4..ac1cdfd5c4a3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; @@ -194,11 +195,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; @@ -2320,7 +2318,9 @@ public TShowConfigurationTemplateResp showConfigurationTemplate() throws TExcept resp.setStatus(getNotLoggedInStatus()); return resp; } - resp.setContent(getConfigurationFileContent(CommonConfig.SYSTEM_CONFIG_NAME + ".properties")); + resp.setContent( + ConfigurationFileUtils.readConfigFileContent( + IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME + ".template"))); resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); return resp; } catch (Exception e) { @@ -2349,7 +2349,9 @@ public TShowConfigurationResp showConfiguration(int nodeId) throws TException { } if (IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == nodeId) { - resp.setContent(getConfigurationFileContent(CommonConfig.SYSTEM_CONFIG_NAME)); + resp.setContent( + ConfigurationFileUtils.readConfigFileContent( + IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME))); resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); return resp; } @@ -2370,19 +2372,6 @@ public TShowConfigurationResp showConfiguration(int nodeId) throws TException { } } - private String getConfigurationFileContent(String configurationFileName) throws IOException { - String content = ""; - URL configurationFileUrl = IoTDBDescriptor.getPropsUrl(configurationFileName); - if (configurationFileUrl == null) { - return content; - } - File file = new File(configurationFileUrl.getFile()); - if (!file.exists()) { - return content; - } - return new String(Files.readAllBytes(file.toPath())); - } - private TSQueryTemplateResp executeTemplateQueryStatement( Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) { long startTime = System.nanoTime(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 167a91a07d322..f0932c3450bde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -245,11 +246,9 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; -import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -1740,10 +1739,7 @@ public TShowConfigurationResp showConfiguration() { new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); try { URL propsUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); - if (propsUrl != null && new File(propsUrl.getFile()).exists()) { - byte[] bytes = Files.readAllBytes(new File(propsUrl.getFile()).toPath()); - resp.setContent(new String(bytes)); - } + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); } catch (Exception e) { resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index c36bdd785b0cc..8fb02915f1d35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; @@ -82,7 +83,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.net.URL; import java.util.ArrayList; @@ -608,9 +608,14 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { Map newConfigItems = req.getConfigs(); Properties properties = new Properties(); properties.putAll(newConfigItems); + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List ignoredConfigItems = ConfigurationFileUtils.filterImmutableConfigItems(properties); + if (!ignoredConfigItems.isEmpty()) { + tsStatus.setMessage("ignored config items: " + ignoredConfigItems); + } if (newConfigItems.isEmpty()) { - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + return tsStatus; } URL configFileUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (configFileUrl == null || !(new File(configFileUrl.getFile()).exists())) { @@ -621,14 +626,13 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + return tsStatus; } // 1. append new configuration properties to configuration file - File configFile = new File(configFileUrl.getFile()); - try (FileWriter writer = new FileWriter(configFile, true)) { - properties.store(writer, null); - } catch (IOException e) { + try { + ConfigurationFileUtils.updateConfigurationFile(new File(configFileUrl.getFile()), properties); + } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } @@ -638,7 +642,7 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + return tsStatus; } /** diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java similarity index 70% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 49b44f0bc02f6..9ade702d2251c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -23,19 +23,23 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.io.RandomAccessFile; import java.net.URL; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; import java.util.concurrent.TimeUnit; -public class ConfigFileAutoUpdateTool { +public class ConfigurationFileUtils { - private final String lockFileSuffix = ".lock"; - private final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); - private final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); - private Logger logger = LoggerFactory.getLogger(ConfigFileAutoUpdateTool.class); - private String license = + private static final String lockFileSuffix = ".lock"; + private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); + private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); + private static Logger logger = LoggerFactory.getLogger(ConfigurationFileUtils.class); + private static String license = "#\n" + "# Licensed to the Apache Software Foundation (ASF) under one\n" + "# or more contributor license agreements. See the NOTICE file\n" @@ -54,7 +58,8 @@ public class ConfigFileAutoUpdateTool { + "# specific language governing permissions and limitations\n" + "# under the License."; - public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) + public static void checkAndMayUpdate( + URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) throws IOException, InterruptedException { if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { return; @@ -94,7 +99,39 @@ public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, } } - private String readConfigLines(File file) throws IOException { + public static String readConfigFileContent(URL url) throws IOException { + if (url == null) { + return ""; + } + File f = new File(url.getFile()); + if (!f.exists()) { + return ""; + } + return readConfigLines(f); + } + + public static List filterImmutableConfigItems(Properties properties) { + List ignoredConfigItems = new ArrayList<>(); + return ignoredConfigItems; + } + + public static void updateConfigurationFile(File file, Properties newConfigItems) + throws IOException, InterruptedException { + File lockFile = new File(file.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + try { + String contentOfSourceFile = readConfigLines(file); + try (FileWriter writer = new FileWriter(file, true)) { + writer.write(contentOfSourceFile); + newConfigItems.store(writer, null); + } + Files.move(lockFile.toPath(), file.toPath()); + } finally { + releaseFileLock(lockFile); + } + } + + private static String readConfigLines(File file) throws IOException { if (!file.exists()) { return ""; } @@ -103,7 +140,7 @@ private String readConfigLines(File file) throws IOException { return content.replace(license, ""); } - private void acquireTargetFileLock(File file) throws IOException, InterruptedException { + private static void acquireTargetFileLock(File file) throws IOException, InterruptedException { long totalWaitTime = 0; while (totalWaitTime < maxTimeMillsToAcquireLock) { if (file.createNewFile()) { @@ -120,7 +157,7 @@ private void acquireTargetFileLock(File file) throws IOException, InterruptedExc file.getName()); } - private void releaseFileLock(File file) throws IOException { + private static void releaseFileLock(File file) throws IOException { Files.deleteIfExists(file.toPath()); } } From 0f0364310339ab02a8342f7afd1dc15436a45b4c Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 4 Jun 2024 21:51:25 +0800 Subject: [PATCH 22/45] configuration template file --- iotdb-core/node-commons/pom.xml | 67 +++++++++++++++++++ .../resources/conf/generate_properties.bat | 12 ++++ .../resources/conf/generate_properties.sh | 19 ++++++ 3 files changed, 98 insertions(+) create mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat create mode 100644 iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index c19924c2870df..b440c7b863b64 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -323,5 +323,72 @@ + + generate-properties-unix + + + unix + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + generate-config-unix + compile + + exec + + + sh + + ${project.basedir}/src/assembly/resources/conf/generate_properties.sh + + + + + + + + + + generate-properties-windows + + + windows + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + generate-config-unix + compile + + exec + + + cmd + + /c + ${project.basedir}\src\assembly\resources\conf\generate_properties.bat + + + + + + + + diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat new file mode 100644 index 0000000000000..7ee2c66a443dc --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat @@ -0,0 +1,12 @@ +@echo off +setlocal enabledelayedexpansion + +set "template_file=conf\iotdb-system.properties.template" +set "output_file=conf\iotdb-system.properties" +ren "%output_file%" "%template_file%" + +(for /f "usebackq delims=" %%A in ("%template_file%") do ( + set "line=%%A" + set "line=!line:~0,1!" + if not "!line!"=="#" if not "!line!"=="" echo %%A +)) > "%output_file%" diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh new file mode 100644 index 0000000000000..26bcdfac1456c --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +source_file="src/assembly/resources/conf/iotdb-system.properties" +target_template_file="target/conf/iotdb-system.properties.template" +target_properties_file="target/conf/iotdb-system.properties" + +if [ -f "$target_template_file" ]; then + rm "$target_template_file" +fi + +if [ -f "$target_properties_file" ]; then + rm "$target_properties_file" +fi + +mkdir -p "$(dirname "$target_template_file")" + +cp "$source_file" "$target_template_file" + +grep -v '^\s*#' "$target_template_file" | grep -v '^\s*$' > "$target_properties_file" From 5ec9fead7c1d3d0513e0ee1ca62c966fcb9e2c7d Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 10:28:06 +0800 Subject: [PATCH 23/45] add template configuration file to distribution --- distribution/src/assembly/all.xml | 9 +++++++++ distribution/src/assembly/confignode.xml | 9 +++++++++ distribution/src/assembly/datanode.xml | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index 028af358eac42..9a236d5725a10 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -57,6 +57,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin diff --git a/distribution/src/assembly/confignode.xml b/distribution/src/assembly/confignode.xml index 502b90758cae2..3944c62325ef8 100644 --- a/distribution/src/assembly/confignode.xml +++ b/distribution/src/assembly/confignode.xml @@ -54,6 +54,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf diff --git a/distribution/src/assembly/datanode.xml b/distribution/src/assembly/datanode.xml index 84697fda6d742..9b0ae34a2bbb5 100644 --- a/distribution/src/assembly/datanode.xml +++ b/distribution/src/assembly/datanode.xml @@ -45,6 +45,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin From 924b9ac6f233bea77704dca1d395bfda00275318 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 10:30:38 +0800 Subject: [PATCH 24/45] modify script for windows --- .../resources/conf/generate_properties.bat | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat index 7ee2c66a443dc..101babbed5d95 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat @@ -1,12 +1,21 @@ @echo off -setlocal enabledelayedexpansion -set "template_file=conf\iotdb-system.properties.template" -set "output_file=conf\iotdb-system.properties" -ren "%output_file%" "%template_file%" +set "source_file=src\assembly\resources\conf\iotdb-system.properties" +set "target_template_file=target\conf\iotdb-system.properties.template" +set "target_properties_file=target\conf\iotdb-system.properties" -(for /f "usebackq delims=" %%A in ("%template_file%") do ( - set "line=%%A" - set "line=!line:~0,1!" - if not "!line!"=="#" if not "!line!"=="" echo %%A -)) > "%output_file%" +if exist "%target_template_file%" ( + del "%target_template_file%" +) + +if exist "%target_properties_file%" ( + del "%target_properties_file%" +) + +mkdir "%target_template_file%\.." + +copy "%source_file%" "%target_template_file%" + +for /f "tokens=* delims=" %%i in ('findstr /v "^[ ]*#" "%target_template_file%" ^| findstr /r /v "^$"') do ( + echo %%i >> "%target_properties_file%" +) From 5476846e04a3bdbe5992ea8b23fba9b36c206df9 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 14:23:01 +0800 Subject: [PATCH 25/45] fix bug --- .../iotdb/db/it/IoTDBSetConfigurationIT.java | 59 +++++++++++++++++++ .../confignode/manager/ConfigManager.java | 3 +- .../confignode/manager/node/NodeManager.java | 2 +- .../executor/ClusterConfigTaskExecutor.java | 2 +- .../commons/conf/ConfigurationFileUtils.java | 51 ++++++++++++++-- 5 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java new file mode 100644 index 0000000000000..6decb4fbc2b47 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.sql.Connection; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSetConfigurationIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Test + public void testSetConfiguration() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + Assert.assertTrue(statement.execute("set configuration \"enable_seq_space_compaction\"=\"true\"")); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 695c222aa76fe..ae46825c389ae 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1505,7 +1505,8 @@ public TSStatus clearCache() { @Override public TSStatus setConfiguration(TSetConfigurationReq req) { TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == req.getNodeId()) { + if (req.getNodeId() < 0 + || ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == req.getNodeId()) { URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url == null || !new File(url.getFile()).exists()) { return tsStatus; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 3e140b1dc819d..4b3d6ce7b46cc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -763,7 +763,7 @@ public List setConfiguration(TSetConfigurationReq req) { SyncConfigNodeClientPool.getInstance() .sendSyncRequestToConfigNodeWithRetry( configNode.getInternalEndPoint(), - req, + new TSetConfigurationReq(req.getConfigs(), configNode.getConfigNodeId()), ConfigNodeRequestType.SET_CONFIGURATION); responseList.add(status); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 981eb15489ab6..bb2c846c34ca9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -1027,7 +1027,7 @@ public SettableFuture clearCache(boolean onCluster) { @Override public SettableFuture setConfiguration(TSetConfigurationReq req) { SettableFuture future = SettableFuture.create(); - TSStatus tsStatus = new TSStatus(); + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); boolean onLocal = IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == req.getNodeId(); if (onLocal) { tsStatus = StorageEngine.getInstance().setConfiguration(req); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 9ade702d2251c..125796a26ebc0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -22,14 +22,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.RandomAccessFile; import java.net.URL; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -117,20 +121,57 @@ public static List filterImmutableConfigItems(Properties properties) { public static void updateConfigurationFile(File file, Properties newConfigItems) throws IOException, InterruptedException { + try { + throw new RuntimeException(""); + } catch (Exception e) { + logger.error("update configuration file", e); + } File lockFile = new File(file.getPath() + lockFileSuffix); acquireTargetFileLock(lockFile); try { - String contentOfSourceFile = readConfigLines(file); - try (FileWriter writer = new FileWriter(file, true)) { - writer.write(contentOfSourceFile); - newConfigItems.store(writer, null); + List lines = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(lockFile))) { + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } + try (FileWriter writer = new FileWriter(lockFile)) { + for (String currentLine : lines) { + if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { + writer.write(currentLine + System.lineSeparator()); + continue; + } + int equalsIndex = currentLine.indexOf('='); + // replace old config + if (equalsIndex != -1) { + String key = currentLine.substring(0, equalsIndex).trim(); + if (newConfigItems.containsKey(key)) { + writer.write(key + "=" + newConfigItems.remove(key) + System.lineSeparator()); + } else { + writer.write(currentLine + System.lineSeparator()); + } + } + } + // add new config items + for (Map.Entry entry : newConfigItems.entrySet()) { + writer.write(entry.getKey() + "=" + entry.getValue()); + } } - Files.move(lockFile.toPath(), file.toPath()); + Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); } finally { releaseFileLock(lockFile); } } + private static Properties loadProperties(File file) throws IOException { + Properties properties = new Properties(); + try (FileReader reader = new FileReader(file)) { + properties.load(reader); + } + return properties; + } + private static String readConfigLines(File file) throws IOException { if (!file.exists()) { return ""; From b0fbfd57989f4bc3b7c6ddd48084cd032e133d78 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 14:25:27 +0800 Subject: [PATCH 26/45] fix bug --- .../apache/iotdb/commons/conf/ConfigurationFileUtils.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 125796a26ebc0..4d40f0f19ef6e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -33,7 +33,6 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -130,7 +129,7 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) acquireTargetFileLock(lockFile); try { List lines = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new FileReader(lockFile))) { + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { String line = null; while ((line = reader.readLine()) != null) { lines.add(line); @@ -154,9 +153,7 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) } } // add new config items - for (Map.Entry entry : newConfigItems.entrySet()) { - writer.write(entry.getKey() + "=" + entry.getValue()); - } + newConfigItems.store(writer, null); } Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); } finally { From 715af8348f12b69afb7355b14c273fd770c77de4 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 14:34:42 +0800 Subject: [PATCH 27/45] fix bug --- .../apache/iotdb/commons/conf/ConfigurationFileUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 4d40f0f19ef6e..6bbf5655cf2f2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -146,14 +146,14 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) if (equalsIndex != -1) { String key = currentLine.substring(0, equalsIndex).trim(); if (newConfigItems.containsKey(key)) { - writer.write(key + "=" + newConfigItems.remove(key) + System.lineSeparator()); - } else { - writer.write(currentLine + System.lineSeparator()); + writer.write("# " + currentLine + System.lineSeparator()); } } } // add new config items - newConfigItems.store(writer, null); + if (!newConfigItems.isEmpty()) { + newConfigItems.store(writer, null); + } } Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); } finally { From 68073e8ae5e6cc26f71a5f6063c3a01fba9d93db Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 15:19:32 +0800 Subject: [PATCH 28/45] fix bug --- .../apache/iotdb/confignode/manager/ConfigManager.java | 8 +++++--- .../iotdb/commons/conf/ConfigurationFileUtils.java | 10 +++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index ae46825c389ae..e3c00c202435d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1505,8 +1505,8 @@ public TSStatus clearCache() { @Override public TSStatus setConfiguration(TSetConfigurationReq req) { TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - if (req.getNodeId() < 0 - || ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == req.getNodeId()) { + int currentNodeId = CONF.getConfigNodeId(); + if (req.getNodeId() < 0 || currentNodeId == req.getNodeId()) { URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (url == null || !new File(url.getFile()).exists()) { return tsStatus; @@ -1524,7 +1524,9 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } - return tsStatus; + if (CONF.getConfigNodeId() == req.getNodeId()) { + return tsStatus; + } } tsStatus = confirmLeader(); return tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 6bbf5655cf2f2..23ae7cff482db 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -120,11 +120,6 @@ public static List filterImmutableConfigItems(Properties properties) { public static void updateConfigurationFile(File file, Properties newConfigItems) throws IOException, InterruptedException { - try { - throw new RuntimeException(""); - } catch (Exception e) { - logger.error("update configuration file", e); - } File lockFile = new File(file.getPath() + lockFileSuffix); acquireTargetFileLock(lockFile); try { @@ -142,12 +137,13 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) continue; } int equalsIndex = currentLine.indexOf('='); - // replace old config + // replace old config by '#' if (equalsIndex != -1) { String key = currentLine.substring(0, equalsIndex).trim(); if (newConfigItems.containsKey(key)) { - writer.write("# " + currentLine + System.lineSeparator()); + writer.write("# "); } + writer.write(currentLine + System.lineSeparator()); } } // add new config items From f78ff9e153a04a48cd6e1bc4697cf9cff6a56f4f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 15:56:33 +0800 Subject: [PATCH 29/45] fix update file --- .../commons/conf/ConfigurationFileUtils.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 23ae7cff482db..e83610504db96 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -137,13 +137,20 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) continue; } int equalsIndex = currentLine.indexOf('='); - // replace old config by '#' + // replace old config if (equalsIndex != -1) { String key = currentLine.substring(0, equalsIndex).trim(); - if (newConfigItems.containsKey(key)) { - writer.write("# "); + String value = currentLine.substring(equalsIndex + 1).trim(); + if (!newConfigItems.containsKey(key)) { + writer.write(currentLine + System.lineSeparator()); + continue; + } + if (newConfigItems.getProperty(key).equals(value)) { + writer.write(currentLine + System.lineSeparator()); + newConfigItems.remove(key); + } else { + writer.write("#" + currentLine + System.lineSeparator()); } - writer.write(currentLine + System.lineSeparator()); } } // add new config items From 9c62b0eddb0feae2e120cd2a05059244570ae5d9 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 16:17:14 +0800 Subject: [PATCH 30/45] add it --- .../org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java | 9 +++++---- .../iotdb/commons/conf/ConfigurationFileUtils.java | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java index 6decb4fbc2b47..871c839d46d0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -22,6 +22,7 @@ import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; + import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -29,10 +30,8 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.io.File; import java.sql.Connection; import java.sql.Statement; -import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class}) @@ -50,8 +49,10 @@ public static void tearDown() throws Exception { @Test public void testSetConfiguration() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - Assert.assertTrue(statement.execute("set configuration \"enable_seq_space_compaction\"=\"true\"")); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"=\"true\""); + statement.execute("set configuration \"enable_cross_space_compaction\"=\"true\" on 0"); + statement.execute("set configuration \"enable_cross_space_compaction\"=\"true\" on 1"); } catch (Exception e) { Assert.fail(e.getMessage()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index e83610504db96..b61e55659d0f7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -120,6 +120,7 @@ public static List filterImmutableConfigItems(Properties properties) { public static void updateConfigurationFile(File file, Properties newConfigItems) throws IOException, InterruptedException { + logger.info("Updating configuration file {}", file.getAbsolutePath()); File lockFile = new File(file.getPath() + lockFileSuffix); acquireTargetFileLock(lockFile); try { From 5854ecc460b0007a094924fa0e9607dda7cbd8c2 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 16:45:00 +0800 Subject: [PATCH 31/45] fix bug --- .../commons/conf/ConfigurationFileUtils.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index b61e55659d0f7..9f3cd1b45928d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -89,11 +89,11 @@ public static void checkAndMayUpdate( } try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { raf.write(license.getBytes()); - String configNodeContent = readConfigLines(configNodeFile); + String configNodeContent = readConfigLinesWithoutLicense(configNodeFile); raf.write(configNodeContent.getBytes()); - String dataNodeContent = readConfigLines(dataNodeFile); + String dataNodeContent = readConfigLinesWithoutLicense(dataNodeFile); raf.write(dataNodeContent.getBytes()); - String commonContent = readConfigLines(commonFile); + String commonContent = readConfigLinesWithoutLicense(commonFile); raf.write(commonContent.getBytes()); } Files.move(lockFile.toPath(), systemFile.toPath()); @@ -165,12 +165,13 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) } } - private static Properties loadProperties(File file) throws IOException { - Properties properties = new Properties(); - try (FileReader reader = new FileReader(file)) { - properties.load(reader); + private static String readConfigLinesWithoutLicense(File file) throws IOException { + if (!file.exists()) { + return ""; } - return properties; + byte[] bytes = Files.readAllBytes(file.toPath()); + String content = new String(bytes); + return content.replace(license, ""); } private static String readConfigLines(File file) throws IOException { @@ -178,8 +179,7 @@ private static String readConfigLines(File file) throws IOException { return ""; } byte[] bytes = Files.readAllBytes(file.toPath()); - String content = new String(bytes); - return content.replace(license, ""); + return new String(bytes); } private static void acquireTargetFileLock(File file) throws IOException, InterruptedException { From 661dd2c970fc654921467539dc2e1d4a3af5a4cc Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 18:01:57 +0800 Subject: [PATCH 32/45] fix it --- .../apache/iotdb/db/it/IoTDBRepairDataIT.java | 2 +- .../iotdb/db/it/IoTDBSetConfigurationIT.java | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java index 550b0d7744b0c..cac76888f3362 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java @@ -60,7 +60,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java index 871c839d46d0b..3ab24dd94f70a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -19,7 +19,10 @@ package org.apache.iotdb.db.it; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -30,6 +33,8 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.io.File; +import java.nio.file.Files; import java.sql.Connection; import java.sql.Statement; @@ -43,18 +48,42 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test public void testSetConfiguration() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { - statement.execute("set configuration \"enable_seq_space_compaction\"=\"true\""); - statement.execute("set configuration \"enable_cross_space_compaction\"=\"true\" on 0"); - statement.execute("set configuration \"enable_cross_space_compaction\"=\"true\" on 1"); + statement.execute("set configuration \"enable_seq_space_compaction\"=\"false\""); + statement.execute("set configuration \"enable_unseq_space_compaction\"=\"false\" on 0"); + statement.execute("set configuration \"enable_cross_space_compaction\"=\"false\" on 1"); } catch (Exception e) { Assert.fail(e.getMessage()); } + for (ConfigNodeWrapper configNodeWrapper : EnvFactory.getEnv().getConfigNodeWrapperList()) { + String systemPropertiesPath = + configNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_unseq_space_compaction=false")); + } + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String systemPropertiesPath = + dataNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_cross_space_compaction=false")); + } } } From 9f94c517b0dd3fedbd6df2d12a50d468f7239b99 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 18:02:49 +0800 Subject: [PATCH 33/45] fix it --- .../src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java index f173e2ff9d600..d800bb8a054a7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java @@ -56,7 +56,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { close(); - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test From ee5cda6718a0e8edc359a2586bcc610409525393 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 18:16:51 +0800 Subject: [PATCH 34/45] filter immutable config items --- .../confignode/manager/ConfigManager.java | 5 ++++- .../iotdb/db/storageengine/StorageEngine.java | 6 +++++- .../commons/conf/ConfigurationFileUtils.java | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index e3c00c202435d..3b9d48b8fe1bc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -53,6 +53,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.SystemPropertiesUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; @@ -1514,8 +1515,10 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { File file = new File(url.getFile()); Properties properties = new Properties(); properties.putAll(req.getConfigs()); + File systemPropertiesFile = + new File(CONF.getSystemDir() + File.separator + ConfigNodeConstant.SYSTEM_FILE_NAME); List ignoredConfigItems = - ConfigurationFileUtils.filterImmutableConfigItems(properties); + ConfigurationFileUtils.filterImmutableConfigItems(systemPropertiesFile, properties); if (!ignoredConfigItems.isEmpty()) { tsStatus.setMessage("ignored config items" + ignoredConfigItems); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 16bb54e0dff8d..bd7c4aeed41fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -45,6 +45,7 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.conf.IoTDBStartCheck; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataExecutionVisitor; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.exception.LoadReadOnlyException; @@ -612,7 +613,10 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { Properties properties = new Properties(); properties.putAll(newConfigItems); TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - List ignoredConfigItems = ConfigurationFileUtils.filterImmutableConfigItems(properties); + File datanodeSystemProperties = + new File(CONFIG.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME); + List ignoredConfigItems = + ConfigurationFileUtils.filterImmutableConfigItems(datanodeSystemProperties, properties); if (!ignoredConfigItems.isEmpty()) { tsStatus.setMessage("ignored config items: " + ignoredConfigItems); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 9f3cd1b45928d..ba98a4dceb88d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -113,8 +113,25 @@ public static String readConfigFileContent(URL url) throws IOException { return readConfigLines(f); } - public static List filterImmutableConfigItems(Properties properties) { + public static List filterImmutableConfigItems( + File systemPropertiesFile, Properties properties) { List ignoredConfigItems = new ArrayList<>(); + if (!systemPropertiesFile.exists()) { + return ignoredConfigItems; + } + Properties systemProperties = new Properties(); + try (FileReader reader = new FileReader(systemPropertiesFile)) { + properties.load(reader); + } catch (Exception e) { + logger.error("Failed to load system properties from {}", systemPropertiesFile, e); + return ignoredConfigItems; + } + for (String ignoredKey : systemProperties.stringPropertyNames()) { + if (properties.containsKey(ignoredKey)) { + properties.remove(ignoredKey); + ignoredConfigItems.add(ignoredKey); + } + } return ignoredConfigItems; } From 80f734a191c9b49a31c2ae122d661bc2aba44e26 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 18:38:47 +0800 Subject: [PATCH 35/45] fix bug --- .../apache/iotdb/commons/conf/ConfigurationFileUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index ba98a4dceb88d..64cddaee10dc6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -114,21 +114,21 @@ public static String readConfigFileContent(URL url) throws IOException { } public static List filterImmutableConfigItems( - File systemPropertiesFile, Properties properties) { + File systemPropertiesFile, Properties newConfigItems) { List ignoredConfigItems = new ArrayList<>(); if (!systemPropertiesFile.exists()) { return ignoredConfigItems; } Properties systemProperties = new Properties(); try (FileReader reader = new FileReader(systemPropertiesFile)) { - properties.load(reader); + systemProperties.load(reader); } catch (Exception e) { logger.error("Failed to load system properties from {}", systemPropertiesFile, e); return ignoredConfigItems; } for (String ignoredKey : systemProperties.stringPropertyNames()) { - if (properties.containsKey(ignoredKey)) { - properties.remove(ignoredKey); + if (newConfigItems.containsKey(ignoredKey)) { + newConfigItems.remove(ignoredKey); ignoredConfigItems.add(ignoredKey); } } From dfa08032663b0d295ebc962ae6c5e3a520845d9b Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 5 Jun 2024 19:26:44 +0800 Subject: [PATCH 36/45] fix error msg --- .../confignode/manager/ConfigManager.java | 6 +-- .../iotdb/db/storageengine/StorageEngine.java | 7 +-- .../commons/conf/ConfigurationFileUtils.java | 47 ++++++++++++++----- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 3b9d48b8fe1bc..d561284bb614f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -53,7 +53,6 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; -import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.SystemPropertiesUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; @@ -1515,12 +1514,11 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { File file = new File(url.getFile()); Properties properties = new Properties(); properties.putAll(req.getConfigs()); - File systemPropertiesFile = - new File(CONF.getSystemDir() + File.separator + ConfigNodeConstant.SYSTEM_FILE_NAME); List ignoredConfigItems = - ConfigurationFileUtils.filterImmutableConfigItems(systemPropertiesFile, properties); + ConfigurationFileUtils.filterImmutableConfigItems(properties); if (!ignoredConfigItems.isEmpty()) { tsStatus.setMessage("ignored config items" + ignoredConfigItems); + tsStatus.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); } try { ConfigurationFileUtils.updateConfigurationFile(file, properties); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index bd7c4aeed41fa..96743b26a89fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -45,7 +45,6 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.IoTDBStartCheck; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataExecutionVisitor; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.exception.LoadReadOnlyException; @@ -613,12 +612,10 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { Properties properties = new Properties(); properties.putAll(newConfigItems); TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - File datanodeSystemProperties = - new File(CONFIG.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME); - List ignoredConfigItems = - ConfigurationFileUtils.filterImmutableConfigItems(datanodeSystemProperties, properties); + List ignoredConfigItems = ConfigurationFileUtils.filterImmutableConfigItems(properties); if (!ignoredConfigItems.isEmpty()) { tsStatus.setMessage("ignored config items: " + ignoredConfigItems); + tsStatus.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); } if (newConfigItems.isEmpty()) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 64cddaee10dc6..6314837a1dc8a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -32,8 +32,11 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; public class ConfigurationFileUtils { @@ -61,6 +64,35 @@ public class ConfigurationFileUtils { + "# specific language governing permissions and limitations\n" + "# under the License."; + // This is a temporary implementations + private static final Set ignoreConfigKeys = + new HashSet<>( + Arrays.asList( + "cn_internal_address", + "cn_internal_port", + "cn_consensus_port", + "cn_seed_config_node", + "dn_internal_address", + "dn_internal_port", + "dn_mpp_data_exchange_port", + "dn_schema_region_consensus_port", + "dn_data_region_consensus_port", + "dn_seed_config_node", + "dn_session_timeout_threshold", + "config_node_consensus_protocol_class", + "schema_replication_factor", + "data_replication_factor", + "data_region_consensus_protocol_class", + "series_slot_num", + "series_partition_executor_class", + "time_partition_interval", + "schema_engine_mode", + "tag_attribute_flush_interval", + "tag_attribute_total_size", + "timestamp_precision", + "iotdb_server_encrypt_decrypt_provider", + "iotdb_server_encrypt_decrypt_provider_parameter")); + public static void checkAndMayUpdate( URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) throws IOException, InterruptedException { @@ -113,20 +145,9 @@ public static String readConfigFileContent(URL url) throws IOException { return readConfigLines(f); } - public static List filterImmutableConfigItems( - File systemPropertiesFile, Properties newConfigItems) { + public static List filterImmutableConfigItems(Properties newConfigItems) { List ignoredConfigItems = new ArrayList<>(); - if (!systemPropertiesFile.exists()) { - return ignoredConfigItems; - } - Properties systemProperties = new Properties(); - try (FileReader reader = new FileReader(systemPropertiesFile)) { - systemProperties.load(reader); - } catch (Exception e) { - logger.error("Failed to load system properties from {}", systemPropertiesFile, e); - return ignoredConfigItems; - } - for (String ignoredKey : systemProperties.stringPropertyNames()) { + for (String ignoredKey : ignoreConfigKeys) { if (newConfigItems.containsKey(ignoredKey)) { newConfigItems.remove(ignoredKey); ignoredConfigItems.add(ignoredKey); From 42a56529354c8ac45e81ba2b235d053f7c6eb1a8 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 6 Jun 2024 11:56:56 +0800 Subject: [PATCH 37/45] add template configuration file to jar --- .../confignode/manager/node/NodeManager.java | 21 ++++++++++++------- .../thrift/impl/ClientRPCServiceImpl.java | 4 +--- iotdb-core/node-commons/pom.xml | 9 ++++++++ .../commons/conf/ConfigurationFileUtils.java | 21 +++++++++++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 4b3d6ce7b46cc..0254bddad5f1d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -758,13 +758,20 @@ public List setConfiguration(TSetConfigurationReq req) { if (nodeId >= 0 && nodeId != configNode.getConfigNodeId()) { continue; } - TSStatus status = - (TSStatus) - SyncConfigNodeClientPool.getInstance() - .sendSyncRequestToConfigNodeWithRetry( - configNode.getInternalEndPoint(), - new TSetConfigurationReq(req.getConfigs(), configNode.getConfigNodeId()), - ConfigNodeRequestType.SET_CONFIGURATION); + TSStatus status = null; + try { + status = + (TSStatus) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + configNode.getInternalEndPoint(), + new TSetConfigurationReq(req.getConfigs(), configNode.getConfigNodeId()), + ConfigNodeRequestType.SET_CONFIGURATION); + } catch (Exception e) { + status = + RpcUtils.getStatus( + TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), e.getMessage()); + } responseList.add(status); } return responseList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index ac1cdfd5c4a3e..f7a2d3ffcd5fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -2318,9 +2318,7 @@ public TShowConfigurationTemplateResp showConfigurationTemplate() throws TExcept resp.setStatus(getNotLoggedInStatus()); return resp; } - resp.setContent( - ConfigurationFileUtils.readConfigFileContent( - IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME + ".template"))); + resp.setContent(ConfigurationFileUtils.readConfigurationTemplateFile()); resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); return resp; } catch (Exception e) { diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3ef5c6fbb8083..30f11341f9a4a 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -172,6 +172,15 @@ + + + ${project.basedir}/src/assembly/resources/conf + + iotdb-system.properties + + false + + - generate-config-unix + generate-properties-windows compile exec From 5bd5ffa09935504dad3fe57bcc3525333eedc29d Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 6 Jun 2024 16:29:34 +0800 Subject: [PATCH 41/45] fix pom.xml --- iotdb-core/node-commons/pom.xml | 2 +- .../src/assembly/resources/conf/generate_properties.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index fbe91c2849445..ed0e4b10ea655 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -384,7 +384,7 @@ exec-maven-plugin 3.1.0 - + generate-properties-windows compile diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh index e31a51244e7f9..f14329b76a6a0 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh @@ -54,7 +54,6 @@ cat < "$target_properties_file" # specific language governing permissions and limitations # under the License. # - EOL grep -v '^\s*#' "$target_template_file" | grep -v '^\s*$' >> "$target_properties_file" From 0319367be8568e53e280e891bb7d248361dbff3b Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 7 Jun 2024 18:34:41 +0800 Subject: [PATCH 42/45] fix scripts --- .../resources/sbin/stop-confignode.bat | 19 ++++- .../client/sync/SyncDataNodeClientPool.java | 14 ---- .../confignode/manager/ConfigManager.java | 6 -- .../confignode/manager/node/NodeManager.java | 8 +- .../assembly/resources/sbin/stop-datanode.bat | 24 +++++- .../executor/ClusterConfigTaskExecutor.java | 17 ++++ .../iotdb/db/storageengine/StorageEngine.java | 13 +-- .../commons/conf/ConfigurationFileUtils.java | 83 +++++++++++-------- 8 files changed, 113 insertions(+), 71 deletions(-) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index e697642668876..fd828384b1e56 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -22,10 +22,15 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-confignode.properties" + IF EXIST "%superior_dir%\conf\iotdb-confignode.properties" ( + set config_file="%superior_dir%\conf\iotdb-confignode.properties" + ) ELSE ( + echo "No configuration file found. Exiting." + exit /b 1 + ) ) for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" @@ -33,6 +38,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" set cn_internal_port=%%i ) +if not defined cn_internal_port ( + echo "cn_internal_port not found in the configuration file. Exiting." + exit /b 1 +) + echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" @@ -40,6 +50,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" set cn_internal_address=%%i ) +if not defined cn_internal_address ( + echo "cn_internal_address not found in the configuration file. Exiting." + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %cn_internal_address%:%cn_internal_port% ^| findstr LISTENING ') do ( taskkill /f /pid %%a echo "close ConfigNode, PID:" %%a diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index 5f4d7c2f12974..9628bc805094e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -190,19 +189,6 @@ public TRegionLeaderChangeResp changeRegionLeader( return new TRegionLeaderChangeResp(status, -1L); } - public TShowConfigurationResp showConfiguration(TEndPoint dataNode) { - TSStatus status; - try (SyncDataNodeInternalServiceClient client = clientManager.borrowClient(dataNode)) { - return client.showConfiguration(); - } catch (ClientManagerException e) { - LOGGER.error("Can't connect to Data node: {}", dataNode, e); - status = RpcUtils.getStatus(TSStatusCode.CAN_NOT_CONNECT_DATANODE, e.getMessage()); - } catch (TException e) { - status = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); - } - return new TShowConfigurationResp(status, ""); - } - // TODO: Is the ClientPool must be a singleton? private static class ClientPoolHolder { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index d561284bb614f..c3e97c960908a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1514,12 +1514,6 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { File file = new File(url.getFile()); Properties properties = new Properties(); properties.putAll(req.getConfigs()); - List ignoredConfigItems = - ConfigurationFileUtils.filterImmutableConfigItems(properties); - if (!ignoredConfigItems.isEmpty()) { - tsStatus.setMessage("ignored config items" + ignoredConfigItems); - tsStatus.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - } try { ConfigurationFileUtils.updateConfigurationFile(file, properties); } catch (Exception e) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 0254bddad5f1d..c85f0fbaba5a2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -812,8 +812,12 @@ public TShowConfigurationResp showConfiguration(int nodeId) { configManager.getNodeManager().getRegisteredDataNodeLocations(); if (dataNodeLocationMap.containsKey(nodeId)) { TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); - return SyncDataNodeClientPool.getInstance() - .showConfiguration(dataNodeLocation.getInternalEndPoint()); + return (TShowConfigurationResp) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + null, + DataNodeRequestType.SHOW_CONFIGURATION); } // other config node diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index d249d42756748..ddc430c305aff 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -22,10 +22,20 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-datanode.properties" + IF EXIST "%superior_dir%\conf\iotdb-datanode.properties" ( + set config_file=%superior_dir%\conf\iotdb-datanode.properties + ) ELSE ( + echo No configuration file found. Exiting. + exit /b 1 + ) +) + +if not defined config_file ( + echo No configuration file found. Exiting. + exit /b 1 ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" @@ -33,6 +43,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" set dn_rpc_port=%%i ) +if not defined dn_rpc_port ( + echo dn_rpc_port not found in the configuration file. Exiting. + exit /b 1 +) + echo Check whether the rpc_port is used..., port is %dn_rpc_port% for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" @@ -40,6 +55,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" set dn_rpc_address=%%i ) +if not defined dn_rpc_address ( + echo dn_rpc_address not found in the configuration file. Exiting. + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %dn_rpc_address%:%dn_rpc_port%') do ( taskkill /f /pid %%a echo Close DataNode, PID: %%a diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index bb2c846c34ca9..8b5f8655b4b18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; @@ -1027,7 +1028,20 @@ public SettableFuture clearCache(boolean onCluster) { @Override public SettableFuture setConfiguration(TSetConfigurationReq req) { SettableFuture future = SettableFuture.create(); + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List ignoredConfigItems = + ConfigurationFileUtils.filterImmutableConfigItems(req.getConfigs()); + TSStatus warningTsStatus = nll; + if (!ignoredConfigItems.isEmpty()) { + warningTsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + warningTsStatus.setMessage("ignored config items: " + ignoredConfigItems); + if (req.getConfigs().isEmpty()) { + future.setException(new IoTDBException(warningTsStatus.message, warningTsStatus.code)); + return future; + } + } + boolean onLocal = IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == req.getNodeId(); if (onLocal) { tsStatus = StorageEngine.getInstance().setConfiguration(req); @@ -1040,6 +1054,9 @@ public SettableFuture setConfiguration(TSetConfigurationReq re future.setException(e); } } + if (warningTsStatus != null) { + tsStatus = warningTsStatus; + } if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 96743b26a89fa..d26304e113cce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -608,19 +608,14 @@ public void clearCache() { } public TSStatus setConfiguration(TSetConfigurationReq req) { - Map newConfigItems = req.getConfigs(); - Properties properties = new Properties(); - properties.putAll(newConfigItems); TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - List ignoredConfigItems = ConfigurationFileUtils.filterImmutableConfigItems(properties); - if (!ignoredConfigItems.isEmpty()) { - tsStatus.setMessage("ignored config items: " + ignoredConfigItems); - tsStatus.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - } - + Map newConfigItems = req.getConfigs(); if (newConfigItems.isEmpty()) { return tsStatus; } + Properties properties = new Properties(); + properties.putAll(newConfigItems); + URL configFileUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); if (configFileUrl == null || !(new File(configFileUrl.getFile()).exists())) { // configuration file not exist, update in mem diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 960e864f311fd..0bd422d462021 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -166,11 +167,11 @@ public static String readConfigurationTemplateFile() throws IOException { return content.toString(); } - public static List filterImmutableConfigItems(Properties newConfigItems) { + public static List filterImmutableConfigItems(Map configItems) { List ignoredConfigItems = new ArrayList<>(); for (String ignoredKey : ignoreConfigKeys) { - if (newConfigItems.containsKey(ignoredKey)) { - newConfigItems.remove(ignoredKey); + if (configItems.containsKey(ignoredKey)) { + configItems.remove(ignoredKey); ignoredConfigItems.add(ignoredKey); } } @@ -179,44 +180,54 @@ public static List filterImmutableConfigItems(Properties newConfigItems) public static void updateConfigurationFile(File file, Properties newConfigItems) throws IOException, InterruptedException { - logger.info("Updating configuration file {}", file.getAbsolutePath()); + // read configuration file + List lines = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } + // generate new configuration file content in memory + StringBuilder contentsOfNewConfigurationFile = new StringBuilder(); + for (String currentLine : lines) { + if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + int equalsIndex = currentLine.indexOf('='); + // replace old config + if (equalsIndex != -1) { + String key = currentLine.substring(0, equalsIndex).trim(); + String value = currentLine.substring(equalsIndex + 1).trim(); + if (!newConfigItems.containsKey(key)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + if (newConfigItems.getProperty(key).equals(value)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + newConfigItems.remove(key); + } else { + contentsOfNewConfigurationFile + .append("#") + .append(currentLine) + .append(System.lineSeparator()); + } + } + } + if (newConfigItems.isEmpty()) { + // No configuration needs to be modified + return; + } File lockFile = new File(file.getPath() + lockFileSuffix); acquireTargetFileLock(lockFile); + logger.info("Updating configuration file {}", file.getAbsolutePath()); try { - List lines = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new FileReader(file))) { - String line = null; - while ((line = reader.readLine()) != null) { - lines.add(line); - } - } try (FileWriter writer = new FileWriter(lockFile)) { - for (String currentLine : lines) { - if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { - writer.write(currentLine + System.lineSeparator()); - continue; - } - int equalsIndex = currentLine.indexOf('='); - // replace old config - if (equalsIndex != -1) { - String key = currentLine.substring(0, equalsIndex).trim(); - String value = currentLine.substring(equalsIndex + 1).trim(); - if (!newConfigItems.containsKey(key)) { - writer.write(currentLine + System.lineSeparator()); - continue; - } - if (newConfigItems.getProperty(key).equals(value)) { - writer.write(currentLine + System.lineSeparator()); - newConfigItems.remove(key); - } else { - writer.write("#" + currentLine + System.lineSeparator()); - } - } - } + writer.write(contentsOfNewConfigurationFile.toString()); // add new config items - if (!newConfigItems.isEmpty()) { - newConfigItems.store(writer, null); - } + newConfigItems.store(writer, null); + writer.flush(); } Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); } finally { From 9f500267a7a3376c2c266ec373fc065c65df4659 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 7 Jun 2024 18:36:14 +0800 Subject: [PATCH 43/45] fix format --- .../src/assembly/resources/sbin/stop-confignode.bat | 10 +++++----- .../src/assembly/resources/sbin/stop-datanode.bat | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index fd828384b1e56..8c0e392f71bd0 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -26,11 +26,11 @@ IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( IF EXIST "%superior_dir%\conf\iotdb-confignode.properties" ( - set config_file="%superior_dir%\conf\iotdb-confignode.properties" - ) ELSE ( - echo "No configuration file found. Exiting." - exit /b 1 - ) + set config_file="%superior_dir%\conf\iotdb-confignode.properties" + ) ELSE ( + echo "No configuration file found. Exiting." + exit /b 1 + ) ) for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index ddc430c305aff..fec6924123462 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -26,11 +26,11 @@ IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( IF EXIST "%superior_dir%\conf\iotdb-datanode.properties" ( - set config_file=%superior_dir%\conf\iotdb-datanode.properties - ) ELSE ( - echo No configuration file found. Exiting. - exit /b 1 - ) + set config_file=%superior_dir%\conf\iotdb-datanode.properties + ) ELSE ( + echo No configuration file found. Exiting. + exit /b 1 + ) ) if not defined config_file ( From f6547176bfaa7b5d79c1765179088cebafa4c2e4 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 7 Jun 2024 18:43:24 +0800 Subject: [PATCH 44/45] fix spell --- .../execution/config/executor/ClusterConfigTaskExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 8b5f8655b4b18..c2e8b404e6f59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -1032,7 +1032,7 @@ public SettableFuture setConfiguration(TSetConfigurationReq re TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); List ignoredConfigItems = ConfigurationFileUtils.filterImmutableConfigItems(req.getConfigs()); - TSStatus warningTsStatus = nll; + TSStatus warningTsStatus = null; if (!ignoredConfigItems.isEmpty()) { warningTsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); warningTsStatus.setMessage("ignored config items: " + ignoredConfigItems); From 8d80e599ce5995f9c0a4f4526f5f24a4350e9654 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 7 Jun 2024 21:18:32 +0800 Subject: [PATCH 45/45] fix windows script --- .../resources/conf/generate_properties.bat | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat index 7ad3c021ebcba..f158d7fe6d2f6 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat @@ -17,6 +17,7 @@ @REM under the License. @REM @echo off +setlocal enabledelayedexpansion set "source_file=src\assembly\resources\conf\iotdb-system.properties" set "target_template_file=target\conf\iotdb-system.properties.template" @@ -34,27 +35,30 @@ mkdir "%target_template_file%\.." copy "%source_file%" "%target_template_file%" -( - echo # - echo # Licensed to the Apache Software Foundation (ASF) under one - echo # or more contributor license agreements. See the NOTICE file - echo # distributed with this work for additional information - echo # regarding copyright ownership. The ASF licenses this file - echo # to you under the Apache License, Version 2.0 (the - echo # "License"); you may not use this file except in compliance - echo # with the License. You may obtain a copy of the License at - echo # - echo # http://www.apache.org/licenses/LICENSE-2.0 - echo # - echo # Unless required by applicable law or agreed to in writing, - echo # software distributed under the License is distributed on an - echo # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - echo # KIND, either express or implied. See the License for the - echo # specific language governing permissions and limitations - echo # under the License. - echo # -) > "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # Licensed to the Apache Software Foundation (ASF) under one >> "%target_properties_file%" +echo # or more contributor license agreements. See the NOTICE file >> "%target_properties_file%" +echo # distributed with this work for additional information >> "%target_properties_file%" +echo # regarding copyright ownership. The ASF licenses this file >> "%target_properties_file%" +echo # to you under the Apache License, Version 2.0 (the >> "%target_properties_file%" +echo # "License"); you may not use this file except in compliance >> "%target_properties_file%" +echo # with the License. You may obtain a copy of the License at >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # http://www.apache.org/licenses/LICENSE-2.0 >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # Unless required by applicable law or agreed to in writing, >> "%target_properties_file%" +echo # software distributed under the License is distributed on an >> "%target_properties_file%" +echo # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY >> "%target_properties_file%" +echo # KIND, either express or implied. See the License for the >> "%target_properties_file%" +echo # specific language governing permissions and limitations >> "%target_properties_file%" +echo # under the License. >> "%target_properties_file%" +echo # >> "%target_properties_file%" -for /f "tokens=* delims=" %%i in ('findstr /v "^[ ]*#" "%target_template_file%" ^| findstr /r /v "^$"') do ( - echo %%i >> "%target_properties_file%" -) +for /f "usebackq tokens=*" %%i in ("%target_template_file%") do ( + set "line=%%i" + setlocal enabledelayedexpansion + if not "!line!"=="" if "!line:~0,1!" NEQ "#" ( + echo !line!>>"%target_properties_file%" + ) + endlocal +) \ No newline at end of file