From f47b700652e6ace3cc67733eeca6513c1358fdca Mon Sep 17 00:00:00 2001 From: liyuheng Date: Thu, 13 Jun 2024 18:20:01 +0800 Subject: [PATCH 01/12] save --- .../datanode/DataNodeRegisterResp.java | 8 ++++-- .../confignode/manager/ConfigManager.java | 3 ++- .../manager/node/ClusterNodeStartUtils.java | 18 ++++++++++--- .../confignode/manager/node/NodeManager.java | 15 +++-------- .../apache/iotdb/db/conf/IoTDBStartCheck.java | 26 +++++++------------ .../db/protocol/client/ConfigNodeInfo.java | 4 +++ .../org/apache/iotdb/db/service/DataNode.java | 24 ++++++++++++----- .../src/main/thrift/confignode.thrift | 1 + 8 files changed, 59 insertions(+), 40 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java index ef7d4f212b261..44bbee8c721d1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java @@ -38,8 +38,8 @@ public class DataNodeRegisterResp implements DataSet { private TSStatus status; private List configNodeList; private Integer dataNodeId; - private TRuntimeConfiguration runtimeConfiguration; + private Boolean preCheck; public DataNodeRegisterResp() { this.dataNodeId = null; @@ -65,6 +65,10 @@ public void setRuntimeConfiguration(TRuntimeConfiguration runtimeConfiguration) this.runtimeConfiguration = runtimeConfiguration; } + public void setPreCheck(Boolean preCheck) { + this.preCheck = preCheck; + } + public static byte[] convertAllTTLInformation(Map allTTLInformation) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { @@ -84,7 +88,7 @@ public TDataNodeRegisterResp convertToRpcDataNodeRegisterResp() { resp.setStatus(status); resp.setConfigNodeList(configNodeList); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (!preCheck && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { resp.setDataNodeId(dataNodeId); resp.setRuntimeConfiguration(runtimeConfiguration); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index c3e97c960908a..f314e983fe17d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -407,11 +407,12 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { req.getClusterName(), req.getDataNodeConfiguration().getLocation(), this); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() && !req.isIsPreCheck()) { return nodeManager.registerDataNode(req); } } DataNodeRegisterResp resp = new DataNodeRegisterResp(); + resp.setPreCheck(req.isIsPreCheck()); resp.setStatus(status); resp.setConfigNodeList(getNodeManager().getRegisteredConfigNodes()); return resp; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java index 13cd42393fab2..1b6da0947c97c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java @@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.rpc.TSStatusCode; @@ -112,10 +113,21 @@ public static TSStatus confirmNodeRegistration( nodeType.getNodeType(), CommonConfig.SYSTEM_CONFIG_NAME)); return status; - } else { - /* Accept registration if all TEndPoints aren't conflict */ - return ACCEPT_NODE_REGISTRATION; } + + // Check cluster id + final String clusterId = + configManager + .getClusterManager() + .getClusterIdWithRetry( + CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); + if (clusterId == null) { + status.setCode(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()).setMessage("clusterId has not generated, please try again later"); + return status; + } + + /* Accept registration if all TEndPoints aren't conflict */ + return ACCEPT_NODE_REGISTRATION; } public static TSStatus confirmNodeRestart( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 0877acd336c24..25c9a52f73b2e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -265,17 +265,6 @@ private TRuntimeConfiguration getRuntimeConfiguration() { public DataSet registerDataNode(TDataNodeRegisterReq req) { DataNodeRegisterResp resp = new DataNodeRegisterResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); - final String clusterId = - configManager - .getClusterManager() - .getClusterIdWithRetry( - CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); - if (clusterId == null) { - resp.setStatus( - new TSStatus(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()) - .setMessage("clusterId has not generated, please try again later")); - return resp; - } // Create a new DataNodeHeartbeatCache and force update NodeStatus int dataNodeId = nodeInfo.generateNextNodeId(); @@ -311,6 +300,10 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { resp.setStatus(ClusterNodeStartUtils.ACCEPT_NODE_REGISTRATION); resp.setDataNodeId( registerDataNodePlan.getDataNodeConfiguration().getLocation().getDataNodeId()); + String clusterId = configManager + .getClusterManager() + .getClusterIdWithRetry( + CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); resp.setRuntimeConfiguration(getRuntimeConfiguration().setClusterId(clusterId)); return resp; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index c5d22f36e3f98..68a9c0015c0e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -177,12 +177,6 @@ public boolean checkIsFirstStart() throws IOException { System.exit(-1); } - // write properties to system.properties - try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) { - systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); - properties.store( - new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - } isFirstStart = true; return true; } @@ -269,11 +263,7 @@ public void checkSystemConfig() throws ConfigurationException, IOException { if (isFirstStart) { // overwrite system.properties when first start - try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) { - systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); - properties.store( - new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - } + generateOrRewriteSystemPropertiesFile(); if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS) && config.getWalMode().equals(WALMode.DISABLE)) { throw new ConfigurationException( @@ -423,15 +413,19 @@ public void serializeMutableSystemPropertiesIfNecessary() throws IOException { } if (needsSerialize) { - try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) { - systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); - properties.store( - new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - } + generateOrRewriteSystemPropertiesFile(); } long endTime = System.currentTimeMillis(); logger.info( "Serialize mutable system properties successfully, which takes {} ms.", (endTime - startTime)); } + + public void generateOrRewriteSystemPropertiesFile() throws IOException { + try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) { + systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); + properties.store( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index d9fe03d80bc1e..6dca7d41f677d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -126,6 +126,10 @@ public boolean updateConfigNodeList(List latestConfigNodes) { * @throws IOException if properties deserialization or configNode list serialization failed. */ private void storeConfigNode() throws IOException { + if (!propertiesFile.exists()) { + logger.info("{} not exist, not necessary to store ConfigNode list", propertiesFile); + return; + } Properties properties = new Properties(); try (FileInputStream inputStream = new FileInputStream(propertiesFile)) { properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 255349cab8971..e2af13c004b9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -195,7 +195,6 @@ protected void doAddNode() { isFirstStart = prepareDataNode(); if (isFirstStart) { - // Set target ConfigNodeList from iotdb-system.properties file ConfigNodeInfo.getInstance() .updateConfigNodeList(Collections.singletonList(config.getSeedConfigNode())); } else { @@ -207,8 +206,12 @@ protected void doAddNode() { pullAndCheckSystemConfigurations(); if (isFirstStart) { + // Do pre-check before formal registration + sendRegisterRequestToConfigNode(true); + // Generate system.properties + IoTDBStartCheck.getInstance().generateOrRewriteSystemPropertiesFile(); // Register this DataNode to the cluster when first start - sendRegisterRequestToConfigNode(); + sendRegisterRequestToConfigNode(false); } else { // Send restart request of this DataNode sendRestartRequestToConfigNode(); @@ -278,7 +281,7 @@ private void pullAndCheckSystemConfigurations() throws StartupException { logger.info("Pulling system configurations from the ConfigNode-leader..."); long startTime = System.currentTimeMillis(); /* Pull system configurations */ - int retry = DEFAULT_RETRY; + int retry = 3; TSystemConfigurationResp configurationResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -397,15 +400,18 @@ private void storeRuntimeConfigurations( /** * Register this DataNode into cluster. * + * @param isPreCheck do pre-check before formal registration * @throws StartupException if register failed. - * @throws IOException if serialize cluster name and dataNode Id failed. + * @throws IOException if serialize cluster name and DataNode id failed. */ - private void sendRegisterRequestToConfigNode() throws StartupException, IOException { + private void sendRegisterRequestToConfigNode(boolean isPreCheck) + throws StartupException, IOException { logger.info("Sending register request to ConfigNode-leader..."); long startTime = System.currentTimeMillis(); /* Send register request */ - int retry = DEFAULT_RETRY; + int retry = 5; TDataNodeRegisterReq req = new TDataNodeRegisterReq(); + req.setIsPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); @@ -438,7 +444,10 @@ private void sendRegisterRequestToConfigNode() throws StartupException, IOExcept } if (dataNodeRegisterResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - + if (isPreCheck) { + logger.info("Successfully pass the pre-check, will do the formal registration soon."); + return; + } /* Store runtime configurations when register success */ int dataNodeID = dataNodeRegisterResp.getDataNodeId(); config.setDataNodeId(dataNodeID); @@ -448,6 +457,7 @@ private void sendRegisterRequestToConfigNode() throws StartupException, IOExcept storeRuntimeConfigurations( dataNodeRegisterResp.getConfigNodeList(), dataNodeRegisterResp.getRuntimeConfiguration()); long endTime = System.currentTimeMillis(); + logger.info( "Successfully register to the cluster: {} , which takes {} ms.", config.getClusterName(), diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index e11cf7540fa8a..ba077cc8bd9d6 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -111,6 +111,7 @@ struct TDataNodeRegisterReq { 1: required string clusterName 2: required common.TDataNodeConfiguration dataNodeConfiguration 3: optional TNodeVersionInfo versionInfo + 4: optional bool preCheck } struct TDataNodeRegisterResp { From a8ef1f367700f6aa4c9220661f80ced7e3b5992f Mon Sep 17 00:00:00 2001 From: liyuheng Date: Mon, 17 Jun 2024 14:30:17 +0800 Subject: [PATCH 02/12] isPreCheck -> preCheck --- .../iotdb/confignode/manager/ConfigManager.java | 4 ++-- .../manager/node/ClusterNodeStartUtils.java | 12 +++++++----- .../iotdb/confignode/manager/node/NodeManager.java | 5 +++-- .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f314e983fe17d..4898a64aae7a1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -407,12 +407,12 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { req.getClusterName(), req.getDataNodeConfiguration().getLocation(), this); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() && !req.isIsPreCheck()) { + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() && !req.isPreCheck()) { return nodeManager.registerDataNode(req); } } DataNodeRegisterResp resp = new DataNodeRegisterResp(); - resp.setPreCheck(req.isIsPreCheck()); + resp.setPreCheck(req.isPreCheck()); resp.setStatus(status); resp.setConfigNodeList(getNodeManager().getRegisteredConfigNodes()); return resp; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java index 1b6da0947c97c..6b809928fa45b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java @@ -117,12 +117,14 @@ public static TSStatus confirmNodeRegistration( // Check cluster id final String clusterId = - configManager - .getClusterManager() - .getClusterIdWithRetry( - CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); + configManager + .getClusterManager() + .getClusterIdWithRetry( + CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); if (clusterId == null) { - status.setCode(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()).setMessage("clusterId has not generated, please try again later"); + status + .setCode(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()) + .setMessage("clusterId has not generated, please try again later"); return status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 25c9a52f73b2e..f2dca78dd97af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -300,10 +300,11 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { resp.setStatus(ClusterNodeStartUtils.ACCEPT_NODE_REGISTRATION); resp.setDataNodeId( registerDataNodePlan.getDataNodeConfiguration().getLocation().getDataNodeId()); - String clusterId = configManager + String clusterId = + configManager .getClusterManager() .getClusterIdWithRetry( - CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); + CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); resp.setRuntimeConfiguration(getRuntimeConfiguration().setClusterId(clusterId)); return resp; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index e2af13c004b9b..b1c0258a67a6a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -411,7 +411,7 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) /* Send register request */ int retry = 5; TDataNodeRegisterReq req = new TDataNodeRegisterReq(); - req.setIsPreCheck(isPreCheck); + req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); From 4bfa8b69d9de34ca331a2b60c4fd32edf159202c Mon Sep 17 00:00:00 2001 From: liyuheng Date: Mon, 17 Jun 2024 17:00:39 +0800 Subject: [PATCH 03/12] use SystemPropertiesUtils --- .../iotdb/it/env/cluster/ClusterConstant.java | 3 - .../env/cluster/node/ConfigNodeWrapper.java | 4 +- .../it/env/cluster/node/DataNodeWrapper.java | 4 +- .../conf/SystemPropertiesUtils.java | 79 ++------ .../iotdb/confignode/service/ConfigNode.java | 5 + .../apache/iotdb/db/conf/IoTDBStartCheck.java | 151 +++------------ .../db/protocol/client/ConfigNodeInfo.java | 62 +------ .../org/apache/iotdb/db/service/DataNode.java | 25 +-- iotdb-core/node-commons/pom.xml | 4 + .../iotdb/commons/file/SystemFileFactory.java | 4 - .../file/SystemPropertiesFileHandler.java | 175 ++++++++++++++++++ 11 files changed, 243 insertions(+), 273 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index d688f2169d5d1..12af0d025f75c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -120,9 +120,6 @@ public class ClusterConstant { public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties"; public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties"; - public static final String SYSTEM_PROPERTIES_FILE = "system.properties"; - public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties"; - // Properties' keys // Common public static final String CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS = diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java index a457149f5985e..a93c54c52b82a 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java @@ -20,6 +20,7 @@ package org.apache.iotdb.it.env.cluster.node; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; @@ -37,7 +38,6 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES; @@ -105,7 +105,7 @@ protected String getDefaultCommonConfigPath() { @Override public String getSystemPropertiesPath() { - return workDirFilePath("data/confignode/system", CONFIG_NODE_SYSTEM_PROPERTIES_FILE); + return workDirFilePath("data/confignode/system", ConfigNodeConstant.SYSTEM_FILE_NAME); } @Override diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 1135f5cd055c6..647bd9b9fa003 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -20,6 +20,7 @@ package org.apache.iotdb.it.env.cluster.node; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBStartCheck; import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; import org.apache.iotdb.it.env.cluster.config.MppJVMConfig; @@ -61,7 +62,6 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.REST_SERVICE_PORT; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS; import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR; -import static org.apache.iotdb.it.env.cluster.ClusterConstant.SYSTEM_PROPERTIES_FILE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET; import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR; @@ -139,7 +139,7 @@ protected String getDefaultCommonConfigPath() { @Override public String getSystemPropertiesPath() { - return workDirFilePath("data/datanode/system/schema", SYSTEM_PROPERTIES_FILE); + return workDirFilePath("data/datanode/system/schema", IoTDBStartCheck.PROPERTIES_FILE_NAME); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index ba9d74a641cd4..e6482e027771c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -24,18 +24,13 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -48,11 +43,8 @@ public class SystemPropertiesUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesUtils.class); - private static File systemPropertiesFile = - new File( - ConfigNodeDescriptor.getInstance().getConf().getSystemDir() - + File.separator - + ConfigNodeConstant.SYSTEM_FILE_NAME); + private static SystemPropertiesFileHandler systemPropertiesHandler = + SystemPropertiesFileHandler.getInstance(); private static final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); @@ -74,11 +66,7 @@ private SystemPropertiesUtils() { // TODO: This needs removal of statics ... public static void reinitializeStatics() { - systemPropertiesFile = - new File( - ConfigNodeDescriptor.getInstance().getConf().getSystemDir() - + File.separator - + ConfigNodeConstant.SYSTEM_FILE_NAME); + systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); } /** @@ -87,7 +75,7 @@ public static void reinitializeStatics() { * @return True if confignode-system.properties file exist. */ public static boolean isRestarted() { - return systemPropertiesFile.exists(); + return systemPropertiesHandler.isRestart(); } /** @@ -97,7 +85,7 @@ public static boolean isRestarted() { * @throws IOException When read the confignode-system.properties file failed */ public static void checkSystemProperties() throws IOException { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); boolean needReWrite = false; final String format = "[SystemProperties] The parameter \"{}\" can't be modified after first startup." @@ -251,7 +239,7 @@ public static void checkSystemProperties() throws IOException { */ public static List loadConfigNodeList() throws IOException, BadNodeUrlException { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); String addresses = systemProperties.getProperty("config_node_list", null); if (addresses != null && !addresses.isEmpty()) { @@ -265,10 +253,10 @@ public static List loadConfigNodeList() * The system parameters can't be changed after the ConfigNode first started. Therefore, store * them in confignode-system.properties during the first startup. * - * @throws IOException getSystemProperties() + * @throws IOException systemPropertiesHandler.read() */ public static void storeSystemParameters() throws IOException { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); systemProperties.setProperty("iotdb_version", IoTDBConstant.VERSION); systemProperties.setProperty("commit_id", IoTDBConstant.BUILD_INFO); @@ -311,7 +299,7 @@ public static void storeSystemParameters() throws IOException { systemProperties.setProperty( "tag_attribute_total_size", String.valueOf(COMMON_CONFIG.getTagAttributeTotalSize())); - storeSystemProperties(systemProperties); + systemPropertiesHandler.write(systemProperties); } /** @@ -321,7 +309,7 @@ public static void storeSystemParameters() throws IOException { * @throws IOException When store confignode-system.properties file failed */ public static void storeConfigNodeList(List configNodes) throws IOException { - if (!systemPropertiesFile.exists()) { + if (!systemPropertiesHandler.fileExist()) { // Avoid creating confignode-system.properties files during // synchronizing the ApplyConfigNode logs from the ConsensusLayer. // 1. For the Non-Seed-ConfigNode, We don't need to create confignode-system.properties file @@ -330,12 +318,8 @@ public static void storeConfigNodeList(List configNodes) th // in which case the latest config_node_list will be updated. return; } - - Properties systemProperties = getSystemProperties(); - systemProperties.setProperty( + systemPropertiesHandler.put( "config_node_list", NodeUrlUtils.convertTConfigNodeUrls(configNodes)); - - storeSystemProperties(systemProperties); } /** @@ -346,7 +330,7 @@ public static void storeConfigNodeList(List configNodes) th * @throws IOException When load confignode-system.properties file failed */ public static String loadClusterNameWhenRestarted() throws IOException { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); String clusterName = systemProperties.getProperty(CLUSTER_NAME, null); if (clusterName == null) { LOGGER.warn( @@ -366,7 +350,7 @@ public static String loadClusterNameWhenRestarted() throws IOException { * @throws IOException When load confignode-system.properties file failed */ public static int loadConfigNodeIdWhenRestarted() throws IOException { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); try { return Integer.parseInt(systemProperties.getProperty("config_node_id", null)); } catch (NumberFormatException e) { @@ -387,7 +371,7 @@ public static int loadConfigNodeIdWhenRestarted() throws IOException { */ public static boolean isSeedConfigNode() { try { - Properties systemProperties = getSystemProperties(); + Properties systemProperties = systemPropertiesHandler.read(); boolean isSeedConfigNode = Boolean.parseBoolean(systemProperties.getProperty("is_seed_config_node", null)); if (isSeedConfigNode) { @@ -399,37 +383,4 @@ public static boolean isSeedConfigNode() { return false; } } - - private static synchronized Properties getSystemProperties() throws IOException { - // Create confignode-system.properties file if necessary - if (!systemPropertiesFile.exists()) { - if (systemPropertiesFile.createNewFile()) { - LOGGER.info( - "System properties file {} for ConfigNode is created.", - systemPropertiesFile.getAbsolutePath()); - } else { - LOGGER.error( - "Can't create the system properties file {} for ConfigNode. " - + "IoTDB-ConfigNode is shutdown.", - systemPropertiesFile.getAbsolutePath()); - throw new IOException("Can't create system properties file"); - } - } - - Properties systemProperties = new Properties(); - try (FileInputStream inputStream = new FileInputStream(systemPropertiesFile)) { - systemProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } - return systemProperties; - } - - private static synchronized void storeSystemProperties(Properties systemProperties) - throws IOException { - try (FileOutputStream fileOutputStream = new FileOutputStream(systemPropertiesFile)) { - systemProperties.store( - new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8), - " THIS FILE IS AUTOMATICALLY GENERATED. PLEASE DO NOT MODIFY THIS FILE !!!"); - fileOutputStream.getFD().sync(); - } - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 3d537cdec61f7..abba812d02442 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.commons.service.JMXService; import org.apache.iotdb.commons.service.RegisterManager; import org.apache.iotdb.commons.service.ServiceType; @@ -130,6 +131,10 @@ public void active() { initConfigManager(); /* Restart */ + SystemPropertiesFileHandler.init( + ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + + File.separator + + ConfigNodeConstant.SYSTEM_FILE_NAME); if (SystemPropertiesUtils.isRestarted()) { LOGGER.info("{} is in restarting process...", ConfigNodeConstant.GLOBAL_NAME); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index 68a9c0015c0e2..3a6c7d5c348bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -24,23 +24,17 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker; import org.apache.commons.io.FileUtils; -import org.apache.tsfile.common.conf.TSFileConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -61,13 +55,10 @@ public class IoTDBStartCheck { private boolean isFirstStart = false; - private final File propertiesFile; - private final File oldPropertiesFile; - private final File tmpPropertiesFile; - - private final Properties properties = new Properties(); + private Properties properties = new Properties(); private final Map> systemProperties = new HashMap<>(); + private final SystemPropertiesFileHandler systemPropertiesHandler; // region params need checking, determined when first start private static final String SYSTEM_PROPERTIES_STRING = "System properties:"; @@ -148,15 +139,7 @@ private IoTDBStartCheck() { } } - oldPropertiesFile = - SystemFileFactory.INSTANCE.getFile( - IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME); - propertiesFile = - SystemFileFactory.INSTANCE.getFile( - config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); - tmpPropertiesFile = - SystemFileFactory.INSTANCE.getFile( - IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME + ".tmp"); + systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); systemProperties.put(IOTDB_VERSION_STRING, () -> IoTDBConstant.VERSION); systemProperties.put(COMMIT_ID_STRING, () -> IoTDBConstant.BUILD_INFO); @@ -165,37 +148,6 @@ private IoTDBStartCheck() { } } - /** check configuration in system.properties when starting IoTDB */ - public boolean checkIsFirstStart() throws IOException { - // system init first time, no need to check, write system.properties and return - if (!propertiesFile.exists() && !tmpPropertiesFile.exists()) { - // create system.properties - if (propertiesFile.createNewFile()) { - logger.info(" {} has been created.", propertiesFile.getAbsolutePath()); - } else { - logger.error("can not create {}", propertiesFile.getAbsolutePath()); - System.exit(-1); - } - - isFirstStart = true; - return true; - } - - if (!propertiesFile.exists() && tmpPropertiesFile.exists()) { - // rename tmp file to system.properties, no need to check - FileUtils.moveFile(tmpPropertiesFile, propertiesFile); - logger.info("rename {} to {}", tmpPropertiesFile, propertiesFile); - isFirstStart = false; - return false; - } else if (propertiesFile.exists() && tmpPropertiesFile.exists()) { - // both files exist, remove tmp file - FileUtils.forceDelete(tmpPropertiesFile); - logger.info("remove {}", tmpPropertiesFile); - } - isFirstStart = false; - return false; - } - /** * check and create directory before start IoTDB. * @@ -235,13 +187,19 @@ public void checkDirectory() throws ConfigurationException, IOException { * @throws IOException If copy fail or delete fail */ public void checkOldSystemConfig() throws IOException { + File oldPropertiesFile = + SystemFileFactory.INSTANCE.getFile( + IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME); if (oldPropertiesFile.exists()) { - FileUtils.copyFile(oldPropertiesFile, propertiesFile); + File correctPropertiesFile = + SystemFileFactory.INSTANCE.getFile( + config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); + FileUtils.copyFile(oldPropertiesFile, correctPropertiesFile); FileUtils.delete(oldPropertiesFile); logger.info( "system.properties file has been moved successfully: {} -> {}", oldPropertiesFile.getAbsolutePath(), - propertiesFile.getAbsolutePath()); + correctPropertiesFile.getAbsolutePath()); } } @@ -255,11 +213,7 @@ public void checkOldSystemConfig() throws IOException { */ public void checkSystemConfig() throws ConfigurationException, IOException { // read properties from system.properties - try (FileInputStream inputStream = new FileInputStream(propertiesFile); - InputStreamReader inputStreamReader = - new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) { - properties.load(inputStreamReader); - } + properties = systemPropertiesHandler.read(); if (isFirstStart) { // overwrite system.properties when first start @@ -288,32 +242,15 @@ public void checkSystemConfig() throws ConfigurationException, IOException { /** repair broken properties */ private void upgradePropertiesFileFromBrokenFile() throws IOException { - // create an empty tmpPropertiesFile - if (tmpPropertiesFile.createNewFile()) { - logger.info("Create system.properties.tmp {}.", tmpPropertiesFile); - } else { - logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile); - System.exit(-1); - } - - try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) { - systemProperties.forEach( - (k, v) -> { - if (!properties.containsKey(k)) { - properties.setProperty(k, v.get()); - } - }); - properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION); - properties.setProperty(COMMIT_ID_STRING, IoTDBConstant.BUILD_INFO); - properties.store( - new OutputStreamWriter(tmpFOS, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - // upgrade finished, delete old system.properties file - if (propertiesFile.exists()) { - Files.delete(propertiesFile.toPath()); - } - } - // rename system.properties.tmp to system.properties - FileUtils.moveFile(tmpPropertiesFile, propertiesFile); + systemProperties.forEach( + (k, v) -> { + if (!properties.containsKey(k)) { + properties.setProperty(k, v.get()); + } + }); + properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION); + properties.setProperty(COMMIT_ID_STRING, IoTDBConstant.BUILD_INFO); + systemPropertiesHandler.write(properties); } /** Check all immutable properties */ @@ -352,44 +289,11 @@ private void throwException(String parameter, Object badValue) throws Configurat parameter + "can't be modified after first startup"); } - // reload properties from system.properties - private void reloadProperties() throws IOException { - try (FileInputStream inputStream = new FileInputStream(propertiesFile); - InputStreamReader inputStreamReader = - new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) { - properties.load(inputStreamReader); - } - } - /** call this method to serialize ClusterName and DataNodeId */ public void serializeClusterNameAndDataNodeId(String clusterName, int dataNodeId) throws IOException { - // create an empty tmpPropertiesFile - if (tmpPropertiesFile.createNewFile()) { - logger.info("Create system.properties.tmp {}.", tmpPropertiesFile); - } else { - logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile); - System.exit(-1); - } - - reloadProperties(); - FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString()); - try { - properties.setProperty(IoTDBConstant.CLUSTER_NAME, clusterName); - properties.setProperty(DATA_NODE_ID, String.valueOf(dataNodeId)); - properties.store( - new OutputStreamWriter(tmpFOS, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - // serialize finished, delete old system.properties file - if (propertiesFile.exists()) { - Files.delete(propertiesFile.toPath()); - } - } finally { - tmpFOS.flush(); - tmpFOS.getFD().sync(); - tmpFOS.close(); - } - // rename system.properties.tmp to system.properties - FileUtils.moveFile(tmpPropertiesFile, propertiesFile); + systemPropertiesHandler.putAll( + IoTDBConstant.CLUSTER_NAME, clusterName, DATA_NODE_ID, String.valueOf(dataNodeId)); } public boolean checkConsensusProtocolExists(TConsensusGroupType type) { @@ -422,10 +326,7 @@ public void serializeMutableSystemPropertiesIfNecessary() throws IOException { } public void generateOrRewriteSystemPropertiesFile() throws IOException { - try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) { - systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); - properties.store( - new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING); - } + systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); + systemPropertiesHandler.write(properties); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 6dca7d41f677d..43838e67338f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -22,21 +22,13 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.BadNodeUrlException; -import org.apache.iotdb.commons.file.SystemFileFactory; -import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.commons.utils.NodeUrlUtils; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -50,8 +42,6 @@ public class ConfigNodeInfo { private static final String CONFIG_NODE_LIST = "config_node_list"; - private static final String PROPERTIES_FILE_NAME = "system.properties"; - private final ReentrantReadWriteLock configNodeInfoReadWriteLock; /** latest config nodes. */ @@ -59,26 +49,11 @@ public class ConfigNodeInfo { public static final ConfigRegionId CONFIG_REGION_ID = new ConfigRegionId(0); - private final File propertiesFile; - private final File propertiesFileTmp; + SystemPropertiesFileHandler systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); private ConfigNodeInfo() { this.configNodeInfoReadWriteLock = new ReentrantReadWriteLock(); this.onlineConfigNodes = new HashSet<>(); - propertiesFile = - SystemFileFactory.INSTANCE.getFile( - IoTDBDescriptor.getInstance().getConfig().getSystemDir() - + File.separator - + PROPERTIES_FILE_NAME); - propertiesFileTmp = - SystemFileFactory.INSTANCE.getFile(propertiesFile.getAbsolutePath() + ".tmp"); - if (propertiesFileTmp.exists()) { - try { - updatePropertiesFile(); - } catch (IOException e) { - logger.error("Update properties file fail", e); - } - } } // TODO: This needs removal of statics ... @@ -126,30 +101,14 @@ public boolean updateConfigNodeList(List latestConfigNodes) { * @throws IOException if properties deserialization or configNode list serialization failed. */ private void storeConfigNode() throws IOException { - if (!propertiesFile.exists()) { - logger.info("{} not exist, not necessary to store ConfigNode list", propertiesFile); + if (!systemPropertiesHandler.fileExist()) { + logger.info("System properties file not exist, not necessary to store ConfigNode list"); return; } - Properties properties = new Properties(); - try (FileInputStream inputStream = new FileInputStream(propertiesFile)) { - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } + Properties properties = systemPropertiesHandler.read(); properties.setProperty( CONFIG_NODE_LIST, NodeUrlUtils.convertTEndPointUrls(new ArrayList<>(onlineConfigNodes))); - try (FileOutputStream fileOutputStream = new FileOutputStream(propertiesFileTmp)) { - properties.store(new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8), ""); - } - updatePropertiesFile(); - } - - private void updatePropertiesFile() throws IOException { - if (!propertiesFile.delete()) { - String msg = - String.format( - "Update %s file fail: %s", PROPERTIES_FILE_NAME, propertiesFile.getAbsoluteFile()); - throw new IOException(msg); - } - FileUtils.moveFileSafe(propertiesFileTmp, propertiesFile); + systemPropertiesHandler.write(properties); } public void loadConfigNodeList() { @@ -157,12 +116,7 @@ public void loadConfigNodeList() { // properties contain CONFIG_NODE_LIST only when start as Data node try { configNodeInfoReadWriteLock.writeLock().lock(); - Properties properties = new Properties(); - try (FileInputStream inputStream = new FileInputStream(propertiesFile)) { - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new RuntimeException(e); - } + Properties properties = systemPropertiesHandler.read(); if (properties.containsKey(CONFIG_NODE_LIST)) { onlineConfigNodes.clear(); @@ -174,6 +128,8 @@ public void loadConfigNodeList() { "Load ConfigNode successfully: {}, which takes {} ms.", onlineConfigNodes, (endTime - startTime)); + } catch (IOException e) { + throw new RuntimeException(e); } catch (BadNodeUrlException e) { logger.error("Cannot parse config node list in system.properties"); } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index b1c0258a67a6a..3cead8a395986 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -33,7 +33,7 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.service.JMXService; @@ -123,6 +123,7 @@ import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.DEFAULT_CLUSTER_NAME; +import static org.apache.iotdb.db.conf.IoTDBStartCheck.PROPERTIES_FILE_NAME; public class DataNode implements DataNodeMBean { @@ -136,10 +137,6 @@ public class DataNode implements DataNodeMBean { IoTDBConstant.JMX_TYPE, ServiceType.DATA_NODE.getJmxName()); - private static File SYSTEM_PROPERTIES = - SystemFileFactory.INSTANCE.getFile( - config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME); - /** * When joining a cluster or getting configuration this node will retry at most "DEFAULT_RETRY" * times before returning a failure to the client. @@ -168,14 +165,6 @@ private DataNode() { // We do not init anything here, so that we can re-initialize the instance in IT. } - // TODO: This needs removal of statics ... - public static void reinitializeStatics() { - SYSTEM_PROPERTIES = - SystemFileFactory.INSTANCE.getFile( - config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME); - registerManager = new RegisterManager(); - } - private static RegisterManager registerManager = new RegisterManager(); public static DataNode getInstance() { @@ -236,11 +225,6 @@ protected void doAddNode() { } catch (StartupException | IOException e) { logger.error("Fail to start server", e); - if (isFirstStart) { - // Delete the system.properties file when first start failed. - // Therefore, the next time this DataNode is start will still be seen as the first time. - SYSTEM_PROPERTIES.deleteOnExit(); - } stop(); System.exit(-1); } @@ -250,9 +234,10 @@ protected void doAddNode() { private boolean prepareDataNode() throws StartupException, IOException { long startTime = System.currentTimeMillis(); - // Notice: Consider this DataNode as first start if the system.properties file doesn't exist IoTDBStartCheck.getInstance().checkOldSystemConfig(); - boolean isFirstStart = IoTDBStartCheck.getInstance().checkIsFirstStart(); + // Notice: Consider this DataNode as first start if the system.properties file doesn't exist + SystemPropertiesFileHandler.init(config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); + boolean isFirstStart = !SystemPropertiesFileHandler.getInstance().isRestart(); // Set this node thisNode.setIp(config.getInternalAddress()); diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 5e5a1d251dc65..9d9175ebfda02 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -174,6 +174,10 @@ awaitility test + + org.apache.ratis + ratis-common + diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java index fe7973bff960a..f9d2a3bb56770 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java @@ -38,7 +38,6 @@ public enum SystemFileFactory { public File getFile(String pathname) { if (fsType.equals(FSType.HDFS)) { throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name()); - // return new HDFSFile(pathname); } else { return new File(pathname); } @@ -47,7 +46,6 @@ public File getFile(String pathname) { public File getFile(String parent, String child) { if (fsType.equals(FSType.HDFS)) { throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name()); - // return new HDFSFile(parent, child); } else { return new File(parent, child); } @@ -56,7 +54,6 @@ public File getFile(String parent, String child) { public File getFile(File parent, String child) { if (fsType.equals(FSType.HDFS)) { throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name()); - // return new HDFSFile(parent, child); } else { return new File(parent, child); } @@ -65,7 +62,6 @@ public File getFile(File parent, String child) { public File getFile(URI uri) { if (fsType.equals(FSType.HDFS)) { throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name()); - // return new HDFSFile(uri); } else { return new File(uri); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java new file mode 100644 index 0000000000000..8909a87217bf3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.file; + +import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.utils.FileUtils; + +import org.apache.ratis.util.AutoCloseableLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Properties; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class SystemPropertiesFileHandler { + private static Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesFileHandler.class); + + private final File formalFile; + private final File tmpFile; + // private ImmutableProperties cache; + ReadWriteLock lock = new ReentrantReadWriteLock(); + + private SystemPropertiesFileHandler(String filePath) { + this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath); + this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp"); + } + + public void put(Object key, Object value) throws IOException { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock()); + FileInputStream inputStream = new FileInputStream(formalFile); + FileOutputStream outputStream = new FileOutputStream(tmpFile)) { + Properties properties = new Properties(); + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + properties.put(key, value); + properties.store(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), ""); + outputStream.getFD().sync(); + } + } + + public void putAll(Object... keyOrValue) throws IOException { + if (keyOrValue.length % 2 != 0) { + throw new IllegalArgumentException( + "Length of parameters should be evenly divided by 2, but actually they are: " + + Arrays.toString(keyOrValue)); + } + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock()); + FileInputStream inputStream = new FileInputStream(formalFile); + FileOutputStream outputStream = new FileOutputStream(tmpFile)) { + Properties properties = new Properties(); + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + for (int i = 0; i < keyOrValue.length; i += 2) { + properties.put(keyOrValue[i], keyOrValue[i + 1]); + } + properties.store(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), ""); + outputStream.getFD().sync(); + } + } + + public void write(Properties properties) throws IOException { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) { + if (!formalFile.exists()) { + writeImpl(properties, formalFile); + return; + } + writeImpl(properties, tmpFile); + if (!formalFile.delete()) { + String msg = + String.format( + "Update %s file fail: %s", formalFile.getName(), formalFile.getAbsoluteFile()); + throw new IOException(msg); + } + FileUtils.moveFileSafe(tmpFile, formalFile); + } + } + + public Properties read() throws IOException { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock()); + FileInputStream inputStream = new FileInputStream(formalFile)) { + Properties properties = new Properties(); + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + return properties; + } + } + + public boolean fileExist() { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock())) { + return formalFile.exists() || tmpFile.exists(); + } + } + + public boolean isRestart() { + return fileExist(); + } + + private void recover() throws StartupException { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) { + if (formalFile.exists() && !tmpFile.exists()) { + // No need to recover + return; + } + if (!formalFile.exists() && !tmpFile.exists()) { + // First start + return; + } + if (formalFile.exists() && tmpFile.exists()) { + if (!tmpFile.delete()) { + LOGGER.warn( + "Delete system.properties tmp file fail, you may manually delete it: {}", + tmpFile.getAbsoluteFile()); + } + return; + } + if (!formalFile.exists() && tmpFile.exists()) { + if (!tmpFile.renameTo(formalFile)) { + String msg = + String.format( + "Cannot rename system properties tmp file, you should manually rename it, then restart the node: %s -> %s", + tmpFile.getAbsolutePath(), formalFile.getAbsolutePath()); + throw new StartupException(msg); + } + return; + } + } + throw new StartupException("Should never touch here"); + } + + private void writeImpl(Properties properties, File file) throws IOException { + try (FileOutputStream fileOutputStream = new FileOutputStream(file)) { + properties.store(new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8), ""); + } + } + + public static void init(String filePath) throws StartupException { + Holder.INSTANCE = new SystemPropertiesFileHandler(filePath); + Holder.INSTANCE.recover(); + } + + public static SystemPropertiesFileHandler getInstance() { + if (Holder.INSTANCE == null) { + throw new RuntimeException( + "You should call SystemPropertiesFileHandler.init before getInstance"); + } + return Holder.INSTANCE; + } + + private static class Holder { + private static SystemPropertiesFileHandler INSTANCE = null; + } +} From efa07a29b6d6a37b8499c7b130242e97ded364bd Mon Sep 17 00:00:00 2001 From: liyuheng Date: Mon, 17 Jun 2024 18:20:07 +0800 Subject: [PATCH 04/12] rename method --- .../conf/SystemPropertiesUtils.java | 2 +- .../apache/iotdb/db/conf/IoTDBStartCheck.java | 6 +- .../db/protocol/client/ConfigNodeInfo.java | 2 +- .../file/SystemPropertiesFileHandler.java | 65 ++++++++++--------- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index e6482e027771c..1eeceac583e35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -299,7 +299,7 @@ public static void storeSystemParameters() throws IOException { systemProperties.setProperty( "tag_attribute_total_size", String.valueOf(COMMON_CONFIG.getTagAttributeTotalSize())); - systemPropertiesHandler.write(systemProperties); + systemPropertiesHandler.overwrite(systemProperties); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index 3a6c7d5c348bc..ea38ce91141ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -250,7 +250,7 @@ private void upgradePropertiesFileFromBrokenFile() throws IOException { }); properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION); properties.setProperty(COMMIT_ID_STRING, IoTDBConstant.BUILD_INFO); - systemPropertiesHandler.write(properties); + systemPropertiesHandler.overwrite(properties); } /** Check all immutable properties */ @@ -292,7 +292,7 @@ private void throwException(String parameter, Object badValue) throws Configurat /** call this method to serialize ClusterName and DataNodeId */ public void serializeClusterNameAndDataNodeId(String clusterName, int dataNodeId) throws IOException { - systemPropertiesHandler.putAll( + systemPropertiesHandler.put( IoTDBConstant.CLUSTER_NAME, clusterName, DATA_NODE_ID, String.valueOf(dataNodeId)); } @@ -327,6 +327,6 @@ public void serializeMutableSystemPropertiesIfNecessary() throws IOException { public void generateOrRewriteSystemPropertiesFile() throws IOException { systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); - systemPropertiesHandler.write(properties); + systemPropertiesHandler.overwrite(properties); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 43838e67338f5..1c39e2e1fb5f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -108,7 +108,7 @@ private void storeConfigNode() throws IOException { Properties properties = systemPropertiesHandler.read(); properties.setProperty( CONFIG_NODE_LIST, NodeUrlUtils.convertTEndPointUrls(new ArrayList<>(onlineConfigNodes))); - systemPropertiesHandler.write(properties); + systemPropertiesHandler.overwrite(properties); } public void loadConfigNodeList() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java index 8909a87217bf3..ade286c4e00f1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java @@ -20,7 +20,6 @@ package org.apache.iotdb.commons.file; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.utils.FileUtils; import org.apache.ratis.util.AutoCloseableLock; import org.slf4j.Logger; @@ -28,6 +27,7 @@ import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; @@ -51,51 +51,39 @@ private SystemPropertiesFileHandler(String filePath) { this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp"); } - public void put(Object key, Object value) throws IOException { - try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock()); - FileInputStream inputStream = new FileInputStream(formalFile); - FileOutputStream outputStream = new FileOutputStream(tmpFile)) { - Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - properties.put(key, value); - properties.store(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), ""); - outputStream.getFD().sync(); - } - } - - public void putAll(Object... keyOrValue) throws IOException { + public void put(Object... keyOrValue) throws IOException { if (keyOrValue.length % 2 != 0) { throw new IllegalArgumentException( "Length of parameters should be evenly divided by 2, but actually they are: " + Arrays.toString(keyOrValue)); } try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock()); - FileInputStream inputStream = new FileInputStream(formalFile); FileOutputStream outputStream = new FileOutputStream(tmpFile)) { Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + // read old properties + if (formalFile.exists()) { + try (FileInputStream inputStream = new FileInputStream(formalFile)) { + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + } + // store new properties for (int i = 0; i < keyOrValue.length; i += 2) { properties.put(keyOrValue[i], keyOrValue[i + 1]); } properties.store(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), ""); outputStream.getFD().sync(); + replaceFormalFile(); } } - public void write(Properties properties) throws IOException { + public void overwrite(Properties properties) throws IOException { try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) { if (!formalFile.exists()) { writeImpl(properties, formalFile); return; } writeImpl(properties, tmpFile); - if (!formalFile.delete()) { - String msg = - String.format( - "Update %s file fail: %s", formalFile.getName(), formalFile.getAbsoluteFile()); - throw new IOException(msg); - } - FileUtils.moveFileSafe(tmpFile, formalFile); + replaceFormalFile(); } } @@ -137,17 +125,13 @@ private void recover() throws StartupException { return; } if (!formalFile.exists() && tmpFile.exists()) { - if (!tmpFile.renameTo(formalFile)) { - String msg = - String.format( - "Cannot rename system properties tmp file, you should manually rename it, then restart the node: %s -> %s", - tmpFile.getAbsolutePath(), formalFile.getAbsolutePath()); - throw new StartupException(msg); - } + replaceFormalFile(); return; } + } catch (IOException e) { + throw new StartupException(e); } - throw new StartupException("Should never touch here"); + throw new StartupException("Should never touch here"); } private void writeImpl(Properties properties, File file) throws IOException { @@ -156,6 +140,23 @@ private void writeImpl(Properties properties, File file) throws IOException { } } + private void replaceFormalFile() throws IOException { + if (!tmpFile.exists()) { + throw new FileNotFoundException("Tmp system properties file must exist when call replaceFormalFile"); + } + if (formalFile.exists() && !formalFile.delete()) { + String msg = + String.format( + "Delete formal system properties file fail: %s", formalFile.getAbsoluteFile()); + throw new IOException(msg); + } + if (!tmpFile.renameTo(formalFile)) { + String msg = String.format("Failed to replace formal system properties file, you may manually rename it: %s -> %s", + tmpFile.getAbsolutePath(), formalFile.getAbsolutePath()); + throw new IOException(msg); + } + } + public static void init(String filePath) throws StartupException { Holder.INSTANCE = new SystemPropertiesFileHandler(filePath); Holder.INSTANCE.recover(); From 6cf9f5fcc9416d9fb1de94e44f663d92b195c032 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Mon, 17 Jun 2024 19:18:44 +0800 Subject: [PATCH 05/12] confignode done --- .../conf/SystemPropertiesUtils.java | 7 +---- .../datanode/DataNodeRegisterResp.java | 2 +- .../confignode/manager/ConfigManager.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 18 +++++++----- .../service/ConfigNodeCommandLine.java | 8 ++++++ .../apache/iotdb/db/conf/IoTDBStartCheck.java | 10 +++---- .../org/apache/iotdb/db/service/DataNode.java | 11 ++++---- .../file/SystemPropertiesFileHandler.java | 28 +++++++++++-------- .../src/main/thrift/confignode.thrift | 1 + 9 files changed, 50 insertions(+), 37 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index 1eeceac583e35..e1ee3b328fd1e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -75,7 +75,7 @@ public static void reinitializeStatics() { * @return True if confignode-system.properties file exist. */ public static boolean isRestarted() { - return systemPropertiesHandler.isRestart(); + return !systemPropertiesHandler.isFirstStart(); } /** @@ -222,11 +222,6 @@ public static void checkSystemProperties() throws IOException { COMMON_CONFIG.setTimePartitionInterval(timePartitionInterval); } } - - if (needReWrite) { - // Re-write special parameters if necessary - storeSystemParameters(); - } } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java index 44bbee8c721d1..3a1001984ab42 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java @@ -39,7 +39,7 @@ public class DataNodeRegisterResp implements DataSet { private List configNodeList; private Integer dataNodeId; private TRuntimeConfiguration runtimeConfiguration; - private Boolean preCheck; + private boolean preCheck = false; public DataNodeRegisterResp() { this.dataNodeId = null; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 4898a64aae7a1..364c930f3d24b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1218,7 +1218,7 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { req.getClusterParameters().getClusterName(), req.getConfigNodeLocation(), this); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (!req.isPreCheck() && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return nodeManager.registerConfigNode(req); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index abba812d02442..7eaa9e5484fc1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -31,7 +31,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; import org.apache.iotdb.commons.service.JMXService; import org.apache.iotdb.commons.service.RegisterManager; import org.apache.iotdb.commons.service.ServiceType; @@ -131,10 +130,6 @@ public void active() { initConfigManager(); /* Restart */ - SystemPropertiesFileHandler.init( - ConfigNodeDescriptor.getInstance().getConf().getSystemDir() - + File.separator - + ConfigNodeConstant.SYSTEM_FILE_NAME); if (SystemPropertiesUtils.isRestarted()) { LOGGER.info("{} is in restarting process...", ConfigNodeConstant.GLOBAL_NAME); @@ -210,7 +205,9 @@ public void active() { // We set up Non-Seed ConfigNode's RPC service before sending the register request // in order to facilitate the scheduling of capacity expansion process in ConfigNode-leader setUpRPCService(); - sendRegisterConfigNodeRequest(); + sendRegisterConfigNodeRequest(true); + SystemPropertiesUtils.storeSystemParameters(); + sendRegisterConfigNodeRequest(false); // The initial startup of Non-Seed-ConfigNode is not yet finished, // we should wait for leader's scheduling LOGGER.info(CONFIGURATION, CONF.getConfigMessage()); @@ -324,12 +321,15 @@ protected void setConfigManager() throws Exception { * @throws StartupException if register failed. * @throws IOException if {@link ConsensusManager} init failed. */ - private void sendRegisterConfigNodeRequest() throws StartupException, IOException { + private void sendRegisterConfigNodeRequest(boolean isPreCheck) + throws StartupException, IOException { TConfigNodeRegisterReq req = new TConfigNodeRegisterReq( configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); + req.setPreCheck(isPreCheck); + req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); @@ -358,6 +358,10 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio LOGGER.error("The result of register ConfigNode is empty!"); throw new StartupException("The result of register ConfigNode is empty!"); } + if (isPreCheck) { + LOGGER.info("Successfully pass the precheck, will do the formal registration soon."); + return; + } /* Always set ConfigNodeId before initConsensusManager */ CONF.setConfigNodeId(resp.getConfigNodeId()); configManager.initConsensusManager(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java index 21ccbab34219b..3465d2f88ff47 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java @@ -24,12 +24,16 @@ import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeConstant; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeRemoveCheck; import org.apache.iotdb.confignode.conf.ConfigNodeStartupCheck; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REMOVE_CONFIGNODE_USAGE; @@ -70,6 +74,10 @@ protected int run(String[] args) { try { // Do ConfigNode startup checks LOGGER.info("Starting IoTDB {}", IoTDBConstant.VERSION_WITH_BUILD); + SystemPropertiesFileHandler.init( + ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + + File.separator + + ConfigNodeConstant.SYSTEM_FILE_NAME); ConfigNodeStartupCheck checks = new ConfigNodeStartupCheck(IoTDBConstant.CN_ROLE); checks.startUpCheck(); } catch (StartupException | ConfigurationException | IOException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index ea38ce91141ba..d71aea21e7f6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -215,9 +215,9 @@ public void checkSystemConfig() throws ConfigurationException, IOException { // read properties from system.properties properties = systemPropertiesHandler.read(); - if (isFirstStart) { + if (systemPropertiesHandler.isFirstStart()) { // overwrite system.properties when first start - generateOrRewriteSystemPropertiesFile(); + generateOrOverwriteSystemPropertiesFile(); if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS) && config.getWalMode().equals(WALMode.DISABLE)) { throw new ConfigurationException( @@ -311,13 +311,13 @@ public void serializeMutableSystemPropertiesIfNecessary() throws IOException { long startTime = System.currentTimeMillis(); boolean needsSerialize = false; for (String param : variableParamValueTable.keySet()) { - if (!(properties.getProperty(param).equals(getVal(param)))) { + if (!properties.getProperty(param).equals(getVal(param))) { needsSerialize = true; } } if (needsSerialize) { - generateOrRewriteSystemPropertiesFile(); + generateOrOverwriteSystemPropertiesFile(); } long endTime = System.currentTimeMillis(); logger.info( @@ -325,7 +325,7 @@ public void serializeMutableSystemPropertiesIfNecessary() throws IOException { (endTime - startTime)); } - public void generateOrRewriteSystemPropertiesFile() throws IOException { + public void generateOrOverwriteSystemPropertiesFile() throws IOException { systemProperties.forEach((k, v) -> properties.setProperty(k, v.get())); systemPropertiesHandler.overwrite(properties); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 3cead8a395986..3ee8bf65e48fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -198,7 +198,7 @@ protected void doAddNode() { // Do pre-check before formal registration sendRegisterRequestToConfigNode(true); // Generate system.properties - IoTDBStartCheck.getInstance().generateOrRewriteSystemPropertiesFile(); + IoTDBStartCheck.getInstance().generateOrOverwriteSystemPropertiesFile(); // Register this DataNode to the cluster when first start sendRegisterRequestToConfigNode(false); } else { @@ -234,10 +234,9 @@ protected void doAddNode() { private boolean prepareDataNode() throws StartupException, IOException { long startTime = System.currentTimeMillis(); - IoTDBStartCheck.getInstance().checkOldSystemConfig(); - // Notice: Consider this DataNode as first start if the system.properties file doesn't exist SystemPropertiesFileHandler.init(config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); - boolean isFirstStart = !SystemPropertiesFileHandler.getInstance().isRestart(); + // Notice: Consider this DataNode as first start if the system.properties file doesn't exist + IoTDBStartCheck.getInstance().checkOldSystemConfig(); // Set this node thisNode.setIp(config.getInternalAddress()); @@ -248,7 +247,7 @@ private boolean prepareDataNode() throws StartupException, IOException { checks.startUpCheck(); long endTime = System.currentTimeMillis(); logger.info("The DataNode is prepared successfully, which takes {} ms", (endTime - startTime)); - return isFirstStart; + return SystemPropertiesFileHandler.getInstance().isFirstStart(); } /** @@ -430,7 +429,7 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) if (dataNodeRegisterResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { if (isPreCheck) { - logger.info("Successfully pass the pre-check, will do the formal registration soon."); + logger.info("Successfully pass the precheck, will do the formal registration soon."); return; } /* Store runtime configurations when register success */ diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java index ade286c4e00f1..4d8abdbfb47ff 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java @@ -88,10 +88,13 @@ public void overwrite(Properties properties) throws IOException { } public Properties read() throws IOException { - try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock()); - FileInputStream inputStream = new FileInputStream(formalFile)) { + try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock())) { Properties properties = new Properties(); - properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + if (formalFile.exists()) { + try (FileInputStream inputStream = new FileInputStream(formalFile)) { + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + } return properties; } } @@ -102,8 +105,8 @@ public boolean fileExist() { } } - public boolean isRestart() { - return fileExist(); + public boolean isFirstStart() { + return !fileExist(); } private void recover() throws StartupException { @@ -129,9 +132,9 @@ private void recover() throws StartupException { return; } } catch (IOException e) { - throw new StartupException(e); + throw new StartupException(e); } - throw new StartupException("Should never touch here"); + throw new StartupException("Should never touch here"); } private void writeImpl(Properties properties, File file) throws IOException { @@ -142,16 +145,19 @@ private void writeImpl(Properties properties, File file) throws IOException { private void replaceFormalFile() throws IOException { if (!tmpFile.exists()) { - throw new FileNotFoundException("Tmp system properties file must exist when call replaceFormalFile"); + throw new FileNotFoundException( + "Tmp system properties file must exist when call replaceFormalFile"); } if (formalFile.exists() && !formalFile.delete()) { String msg = - String.format( - "Delete formal system properties file fail: %s", formalFile.getAbsoluteFile()); + String.format( + "Delete formal system properties file fail: %s", formalFile.getAbsoluteFile()); throw new IOException(msg); } if (!tmpFile.renameTo(formalFile)) { - String msg = String.format("Failed to replace formal system properties file, you may manually rename it: %s -> %s", + String msg = + String.format( + "Failed to replace formal system properties file, you may manually rename it: %s -> %s", tmpFile.getAbsolutePath(), formalFile.getAbsolutePath()); throw new IOException(msg); } diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index ba077cc8bd9d6..6944fc819331c 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -405,6 +405,7 @@ struct TConfigNodeRegisterReq { 1: required TClusterParameters clusterParameters 2: required common.TConfigNodeLocation configNodeLocation 3: optional TNodeVersionInfo versionInfo + 4: optional bool preCheck } struct TConfigNodeRegisterResp { From 2d79ae9e13d8613b6e3ffc3656ca2f5cf08ee88d Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 18 Jun 2024 10:55:53 +0800 Subject: [PATCH 06/12] self review --- .../conf/SystemPropertiesUtils.java | 8 +++---- .../confignode/manager/ConfigManager.java | 2 +- .../confignode/manager/node/NodeManager.java | 6 +----- .../service/ConfigNodeCommandLine.java | 4 ++-- .../confignode/persistence/NodeInfoTest.java | 5 ++++- .../apache/iotdb/db/conf/IoTDBStartCheck.java | 8 +++---- .../db/protocol/client/ConfigNodeInfo.java | 4 ++-- .../org/apache/iotdb/db/service/DataNode.java | 19 ++++++++++------- ...dler.java => SystemPropertiesHandler.java} | 21 ++++++++++++------- 9 files changed, 44 insertions(+), 33 deletions(-) rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/{SystemPropertiesFileHandler.java => SystemPropertiesHandler.java} (92%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index e1ee3b328fd1e..2e5a9b755bde8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.slf4j.Logger; @@ -43,8 +43,8 @@ public class SystemPropertiesUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesUtils.class); - private static SystemPropertiesFileHandler systemPropertiesHandler = - SystemPropertiesFileHandler.getInstance(); + private static SystemPropertiesHandler systemPropertiesHandler = + SystemPropertiesHandler.getInstance(); private static final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); @@ -66,7 +66,7 @@ private SystemPropertiesUtils() { // TODO: This needs removal of statics ... public static void reinitializeStatics() { - systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); + systemPropertiesHandler = SystemPropertiesHandler.getInstance(); } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 364c930f3d24b..6f3ebe417b876 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -407,7 +407,7 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { req.getClusterName(), req.getDataNodeConfiguration().getLocation(), this); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() && !req.isPreCheck()) { + if (!req.isPreCheck() && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return nodeManager.registerDataNode(req); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index f2dca78dd97af..c8b946cf22630 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -300,11 +300,7 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { resp.setStatus(ClusterNodeStartUtils.ACCEPT_NODE_REGISTRATION); resp.setDataNodeId( registerDataNodePlan.getDataNodeConfiguration().getLocation().getDataNodeId()); - String clusterId = - configManager - .getClusterManager() - .getClusterIdWithRetry( - CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2); + String clusterId = configManager.getClusterManager().getClusterId(); resp.setRuntimeConfiguration(getRuntimeConfiguration().setClusterId(clusterId)); return resp; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java index 3465d2f88ff47..422570ec3bcbe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeRemoveCheck; @@ -74,7 +74,7 @@ protected int run(String[] args) { try { // Do ConfigNode startup checks LOGGER.info("Starting IoTDB {}", IoTDBConstant.VERSION_WITH_BUILD); - SystemPropertiesFileHandler.init( + SystemPropertiesHandler.init( ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + File.separator + ConfigNodeConstant.SYSTEM_FILE_NAME); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index 959eeba114d9f..560b621e0992d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -24,6 +24,8 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TNodeResource; +import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; @@ -46,7 +48,8 @@ public class NodeInfoTest { private static final File snapshotDir = new File(BASE_OUTPUT_PATH, "snapshot"); @BeforeClass - public static void setup() { + public static void setup() throws StartupException { + SystemPropertiesHandler.init(""); nodeInfo = new NodeInfo(); if (!snapshotDir.exists()) { snapshotDir.mkdirs(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index d71aea21e7f6e..496da5cd68aaf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.file.SystemFileFactory; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker; @@ -58,7 +58,7 @@ public class IoTDBStartCheck { private Properties properties = new Properties(); private final Map> systemProperties = new HashMap<>(); - private final SystemPropertiesFileHandler systemPropertiesHandler; + private final SystemPropertiesHandler systemPropertiesHandler; // region params need checking, determined when first start private static final String SYSTEM_PROPERTIES_STRING = "System properties:"; @@ -139,7 +139,7 @@ private IoTDBStartCheck() { } } - systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); + systemPropertiesHandler = SystemPropertiesHandler.getInstance(); systemProperties.put(IOTDB_VERSION_STRING, () -> IoTDBConstant.VERSION); systemProperties.put(COMMIT_ID_STRING, () -> IoTDBConstant.BUILD_INFO); @@ -186,7 +186,7 @@ public void checkDirectory() throws ConfigurationException, IOException { * * @throws IOException If copy fail or delete fail */ - public void checkOldSystemConfig() throws IOException { + public static void checkOldSystemConfig() throws IOException { File oldPropertiesFile = SystemFileFactory.INSTANCE.getFile( IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 1c39e2e1fb5f8..0e45e9d60e510 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.BadNodeUrlException; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.slf4j.Logger; @@ -49,7 +49,7 @@ public class ConfigNodeInfo { public static final ConfigRegionId CONFIG_REGION_ID = new ConfigRegionId(0); - SystemPropertiesFileHandler systemPropertiesHandler = SystemPropertiesFileHandler.getInstance(); + SystemPropertiesHandler systemPropertiesHandler = SystemPropertiesHandler.getInstance(); private ConfigNodeInfo() { this.configNodeInfoReadWriteLock = new ReentrantReadWriteLock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 3ee8bf65e48fa..a01ac9904eb9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -33,7 +33,7 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesFileHandler; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.service.JMXService; @@ -165,6 +165,11 @@ private DataNode() { // We do not init anything here, so that we can re-initialize the instance in IT. } + // TODO: This needs removal of statics ... + public static void reinitializeStatics() { + registerManager = new RegisterManager(); + } + private static RegisterManager registerManager = new RegisterManager(); public static DataNode getInstance() { @@ -234,9 +239,9 @@ protected void doAddNode() { private boolean prepareDataNode() throws StartupException, IOException { long startTime = System.currentTimeMillis(); - SystemPropertiesFileHandler.init(config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); - // Notice: Consider this DataNode as first start if the system.properties file doesn't exist - IoTDBStartCheck.getInstance().checkOldSystemConfig(); + // Init system properties handler + IoTDBStartCheck.checkOldSystemConfig(); + SystemPropertiesHandler.init(config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); // Set this node thisNode.setIp(config.getInternalAddress()); @@ -247,7 +252,7 @@ private boolean prepareDataNode() throws StartupException, IOException { checks.startUpCheck(); long endTime = System.currentTimeMillis(); logger.info("The DataNode is prepared successfully, which takes {} ms", (endTime - startTime)); - return SystemPropertiesFileHandler.getInstance().isFirstStart(); + return SystemPropertiesHandler.getInstance().isFirstStart(); } /** @@ -265,7 +270,7 @@ private void pullAndCheckSystemConfigurations() throws StartupException { logger.info("Pulling system configurations from the ConfigNode-leader..."); long startTime = System.currentTimeMillis(); /* Pull system configurations */ - int retry = 3; + int retry = DEFAULT_RETRY; TSystemConfigurationResp configurationResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -393,7 +398,7 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) logger.info("Sending register request to ConfigNode-leader..."); long startTime = System.currentTimeMillis(); /* Send register request */ - int retry = 5; + int retry = DEFAULT_RETRY; TDataNodeRegisterReq req = new TDataNodeRegisterReq(); req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java similarity index 92% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java index 4d8abdbfb47ff..4972da7b25241 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesFileHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java @@ -38,23 +38,30 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -public class SystemPropertiesFileHandler { - private static Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesFileHandler.class); +public class SystemPropertiesHandler { + private static Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesHandler.class); private final File formalFile; private final File tmpFile; // private ImmutableProperties cache; ReadWriteLock lock = new ReentrantReadWriteLock(); - private SystemPropertiesFileHandler(String filePath) { + private SystemPropertiesHandler(String filePath) { this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath); this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp"); } + /** + * Put key and value into properties + * + * @param keyOrValue [key0, value0, key1, value1, ... ] + */ public void put(Object... keyOrValue) throws IOException { if (keyOrValue.length % 2 != 0) { throw new IllegalArgumentException( - "Length of parameters should be evenly divided by 2, but actually they are: " + "Length of parameters should be evenly divided by 2, but the actual length is " + + keyOrValue.length + + " : " + Arrays.toString(keyOrValue)); } try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock()); @@ -164,11 +171,11 @@ private void replaceFormalFile() throws IOException { } public static void init(String filePath) throws StartupException { - Holder.INSTANCE = new SystemPropertiesFileHandler(filePath); + Holder.INSTANCE = new SystemPropertiesHandler(filePath); Holder.INSTANCE.recover(); } - public static SystemPropertiesFileHandler getInstance() { + public static SystemPropertiesHandler getInstance() { if (Holder.INSTANCE == null) { throw new RuntimeException( "You should call SystemPropertiesFileHandler.init before getInstance"); @@ -177,6 +184,6 @@ public static SystemPropertiesFileHandler getInstance() { } private static class Holder { - private static SystemPropertiesFileHandler INSTANCE = null; + private static SystemPropertiesHandler INSTANCE = null; } } From 3407e12ccd6fe36d5cfe4df76895061f6b80209d Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 18 Jun 2024 11:40:53 +0800 Subject: [PATCH 07/12] fix ci --- .../test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java | 2 +- .../apache/iotdb/db/protocol/client/ConfigNodeInfo.java | 8 +++----- .../main/java/org/apache/iotdb/db/service/DataNode.java | 7 ++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java index 7c4d87511d813..41bcc24a375a3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java @@ -98,7 +98,7 @@ public void tearDown() throws Exception { } @Test - public void RecoverTest1() { + public void recoverTest1() { // stop cluster EnvFactory.getEnv().shutdownAllDataNodes(); logger.info("All DataNodes are shut down"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 0e45e9d60e510..308bb1fbb3bc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -80,7 +80,7 @@ public boolean updateConfigNodeList(List latestConfigNodes) { try { onlineConfigNodes.clear(); onlineConfigNodes.addAll(latestConfigNodes); - storeConfigNode(); + storeConfigNodeList(); long endTime = System.currentTimeMillis(); logger.info( "Update ConfigNode Successfully: {}, which takes {} ms.", @@ -100,15 +100,13 @@ public boolean updateConfigNodeList(List latestConfigNodes) { * * @throws IOException if properties deserialization or configNode list serialization failed. */ - private void storeConfigNode() throws IOException { + public void storeConfigNodeList() throws IOException { if (!systemPropertiesHandler.fileExist()) { logger.info("System properties file not exist, not necessary to store ConfigNode list"); return; } - Properties properties = systemPropertiesHandler.read(); - properties.setProperty( + systemPropertiesHandler.put( CONFIG_NODE_LIST, NodeUrlUtils.convertTEndPointUrls(new ArrayList<>(onlineConfigNodes))); - systemPropertiesHandler.overwrite(properties); } public void loadConfigNodeList() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index a01ac9904eb9e..3a8b39254ec75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -200,10 +200,9 @@ protected void doAddNode() { pullAndCheckSystemConfigurations(); if (isFirstStart) { - // Do pre-check before formal registration sendRegisterRequestToConfigNode(true); - // Generate system.properties IoTDBStartCheck.getInstance().generateOrOverwriteSystemPropertiesFile(); + ConfigNodeInfo.getInstance().storeConfigNodeList(); // Register this DataNode to the cluster when first start sendRegisterRequestToConfigNode(false); } else { @@ -278,9 +277,7 @@ private void pullAndCheckSystemConfigurations() throws StartupException { configurationResp = configNodeClient.getSystemConfiguration(); break; } catch (TException | ClientManagerException e) { - logger.warn( - "Cannot pull system configurations from ConfigNode-leader, because: {}", - e.getMessage()); + logger.warn("Cannot pull system configurations from ConfigNode-leader", e); retry--; } From eebfb6d7985fdb082b8fafbb321df152ad78f08a Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 18 Jun 2024 14:43:02 +0800 Subject: [PATCH 08/12] fix ci --- .../org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java | 1 - .../apache/iotdb/commons/file/SystemPropertiesHandler.java | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 308bb1fbb3bc2..6f56a5698a728 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -37,7 +37,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; public class ConfigNodeInfo { - private static final Logger logger = LoggerFactory.getLogger(ConfigNodeInfo.class); private static final String CONFIG_NODE_LIST = "config_node_list"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java index 4972da7b25241..38c92861cc75a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java @@ -39,7 +39,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; public class SystemPropertiesHandler { - private static Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesHandler.class); private final File formalFile; private final File tmpFile; @@ -177,8 +177,8 @@ public static void init(String filePath) throws StartupException { public static SystemPropertiesHandler getInstance() { if (Holder.INSTANCE == null) { - throw new RuntimeException( - "You should call SystemPropertiesFileHandler.init before getInstance"); + LOGGER.warn( + "The SystemPropertiesHandler's instance has not been inited yet, call getInstance may cause NullPointerException"); } return Holder.INSTANCE; } From 23ffc42d08adcce97ad2f5037e22aee9fb012dda Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 18 Jun 2024 18:14:01 +0800 Subject: [PATCH 09/12] tan review --- .../conf/SystemPropertiesUtils.java | 8 +++-- .../datanode/DataNodeRegisterResp.java | 12 +++---- .../confignode/manager/ConfigManager.java | 3 +- .../iotdb/confignode/service/ConfigNode.java | 13 ++----- .../service/ConfigNodeCommandLine.java | 8 ----- .../confignode/persistence/NodeInfoTest.java | 2 -- .../apache/iotdb/db/conf/IoTDBStartCheck.java | 2 +- .../db/protocol/client/ConfigNodeInfo.java | 3 +- .../org/apache/iotdb/db/service/DataNode.java | 12 ++++--- .../commons/file/SystemPropertiesHandler.java | 35 ++++++------------- .../src/main/thrift/confignode.thrift | 1 - 11 files changed, 34 insertions(+), 65 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index 2e5a9b755bde8..a56f4a63f7296 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -44,7 +45,7 @@ public class SystemPropertiesUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesUtils.class); private static SystemPropertiesHandler systemPropertiesHandler = - SystemPropertiesHandler.getInstance(); + ConfigNodeSystemPropertiesHandler.getInstance(); private static final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); @@ -66,7 +67,10 @@ private SystemPropertiesUtils() { // TODO: This needs removal of statics ... public static void reinitializeStatics() { - systemPropertiesHandler = SystemPropertiesHandler.getInstance(); + systemPropertiesHandler.resetFilePath( + ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + + File.separator + + ConfigNodeConstant.SYSTEM_FILE_NAME); } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java index 3a1001984ab42..2cbcb15a90129 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; public class DataNodeRegisterResp implements DataSet { @@ -39,7 +40,6 @@ public class DataNodeRegisterResp implements DataSet { private List configNodeList; private Integer dataNodeId; private TRuntimeConfiguration runtimeConfiguration; - private boolean preCheck = false; public DataNodeRegisterResp() { this.dataNodeId = null; @@ -65,10 +65,6 @@ public void setRuntimeConfiguration(TRuntimeConfiguration runtimeConfiguration) this.runtimeConfiguration = runtimeConfiguration; } - public void setPreCheck(Boolean preCheck) { - this.preCheck = preCheck; - } - public static byte[] convertAllTTLInformation(Map allTTLInformation) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { @@ -88,9 +84,9 @@ public TDataNodeRegisterResp convertToRpcDataNodeRegisterResp() { resp.setStatus(status); resp.setConfigNodeList(configNodeList); - if (!preCheck && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - resp.setDataNodeId(dataNodeId); - resp.setRuntimeConfiguration(runtimeConfiguration); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + Optional.ofNullable(dataNodeId).ifPresent(resp::setDataNodeId); + Optional.ofNullable(runtimeConfiguration).ifPresent(resp::setRuntimeConfiguration); } return resp; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 6f3ebe417b876..49eb4599a800a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -412,7 +412,6 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { } } DataNodeRegisterResp resp = new DataNodeRegisterResp(); - resp.setPreCheck(req.isPreCheck()); resp.setStatus(status); resp.setConfigNodeList(getNodeManager().getRegisteredConfigNodes()); return resp; @@ -1218,7 +1217,7 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { req.getClusterParameters().getClusterName(), req.getConfigNodeLocation(), this); - if (!req.isPreCheck() && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return nodeManager.registerConfigNode(req); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 7eaa9e5484fc1..3d537cdec61f7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -205,9 +205,7 @@ public void active() { // We set up Non-Seed ConfigNode's RPC service before sending the register request // in order to facilitate the scheduling of capacity expansion process in ConfigNode-leader setUpRPCService(); - sendRegisterConfigNodeRequest(true); - SystemPropertiesUtils.storeSystemParameters(); - sendRegisterConfigNodeRequest(false); + sendRegisterConfigNodeRequest(); // The initial startup of Non-Seed-ConfigNode is not yet finished, // we should wait for leader's scheduling LOGGER.info(CONFIGURATION, CONF.getConfigMessage()); @@ -321,15 +319,12 @@ protected void setConfigManager() throws Exception { * @throws StartupException if register failed. * @throws IOException if {@link ConsensusManager} init failed. */ - private void sendRegisterConfigNodeRequest(boolean isPreCheck) - throws StartupException, IOException { + private void sendRegisterConfigNodeRequest() throws StartupException, IOException { TConfigNodeRegisterReq req = new TConfigNodeRegisterReq( configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); - req.setPreCheck(isPreCheck); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); @@ -358,10 +353,6 @@ private void sendRegisterConfigNodeRequest(boolean isPreCheck) LOGGER.error("The result of register ConfigNode is empty!"); throw new StartupException("The result of register ConfigNode is empty!"); } - if (isPreCheck) { - LOGGER.info("Successfully pass the precheck, will do the formal registration soon."); - return; - } /* Always set ConfigNodeId before initConsensusManager */ CONF.setConfigNodeId(resp.getConfigNodeId()); configManager.initConsensusManager(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java index 422570ec3bcbe..21ccbab34219b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeCommandLine.java @@ -24,16 +24,12 @@ import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.exception.ConfigurationException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesHandler; -import org.apache.iotdb.confignode.conf.ConfigNodeConstant; -import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeRemoveCheck; import org.apache.iotdb.confignode.conf.ConfigNodeStartupCheck; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REMOVE_CONFIGNODE_USAGE; @@ -74,10 +70,6 @@ protected int run(String[] args) { try { // Do ConfigNode startup checks LOGGER.info("Starting IoTDB {}", IoTDBConstant.VERSION_WITH_BUILD); - SystemPropertiesHandler.init( - ConfigNodeDescriptor.getInstance().getConf().getSystemDir() - + File.separator - + ConfigNodeConstant.SYSTEM_FILE_NAME); ConfigNodeStartupCheck checks = new ConfigNodeStartupCheck(IoTDBConstant.CN_ROLE); checks.startUpCheck(); } catch (StartupException | ConfigurationException | IOException e) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index 560b621e0992d..5ce94932036df 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -25,7 +25,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TNodeResource; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; @@ -49,7 +48,6 @@ public class NodeInfoTest { @BeforeClass public static void setup() throws StartupException { - SystemPropertiesHandler.init(""); nodeInfo = new NodeInfo(); if (!snapshotDir.exists()) { snapshotDir.mkdirs(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index 496da5cd68aaf..64cdbe20e76a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -139,7 +139,7 @@ private IoTDBStartCheck() { } } - systemPropertiesHandler = SystemPropertiesHandler.getInstance(); + systemPropertiesHandler = DataNodeSystemPropertiesHandler.getInstance(); systemProperties.put(IOTDB_VERSION_STRING, () -> IoTDBConstant.VERSION); systemProperties.put(COMMIT_ID_STRING, () -> IoTDBConstant.BUILD_INFO); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 6f56a5698a728..256b8a0be178f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.commons.utils.NodeUrlUtils; +import org.apache.iotdb.db.conf.DataNodeSystemPropertiesHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,7 @@ public class ConfigNodeInfo { public static final ConfigRegionId CONFIG_REGION_ID = new ConfigRegionId(0); - SystemPropertiesHandler systemPropertiesHandler = SystemPropertiesHandler.getInstance(); + SystemPropertiesHandler systemPropertiesHandler = DataNodeSystemPropertiesHandler.getInstance(); private ConfigNodeInfo() { this.configNodeInfoReadWriteLock = new ReentrantReadWriteLock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 3a8b39254ec75..c057c3392e579 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -33,7 +33,6 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.file.SystemPropertiesHandler; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.service.JMXService; @@ -60,6 +59,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.DataNodeStartupCheck; +import org.apache.iotdb.db.conf.DataNodeSystemPropertiesHandler; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.IoTDBStartCheck; @@ -168,6 +168,11 @@ private DataNode() { // TODO: This needs removal of statics ... public static void reinitializeStatics() { registerManager = new RegisterManager(); + DataNodeSystemPropertiesHandler.getInstance() + .resetFilePath( + IoTDBDescriptor.getInstance().getConfig().getSystemDir() + + File.separator + + PROPERTIES_FILE_NAME); } private static RegisterManager registerManager = new RegisterManager(); @@ -240,7 +245,6 @@ private boolean prepareDataNode() throws StartupException, IOException { // Init system properties handler IoTDBStartCheck.checkOldSystemConfig(); - SystemPropertiesHandler.init(config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME); // Set this node thisNode.setIp(config.getInternalAddress()); @@ -251,7 +255,7 @@ private boolean prepareDataNode() throws StartupException, IOException { checks.startUpCheck(); long endTime = System.currentTimeMillis(); logger.info("The DataNode is prepared successfully, which takes {} ms", (endTime - startTime)); - return SystemPropertiesHandler.getInstance().isFirstStart(); + return DataNodeSystemPropertiesHandler.getInstance().isFirstStart(); } /** @@ -388,7 +392,7 @@ private void storeRuntimeConfigurations( * * @param isPreCheck do pre-check before formal registration * @throws StartupException if register failed. - * @throws IOException if serialize cluster name and DataNode id failed. + * @throws IOException if serialize cluster name and datanode id failed. */ private void sendRegisterRequestToConfigNode(boolean isPreCheck) throws StartupException, IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java index 38c92861cc75a..61bcbb49b0cb9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java @@ -19,8 +19,6 @@ package org.apache.iotdb.commons.file; -import org.apache.iotdb.commons.exception.StartupException; - import org.apache.ratis.util.AutoCloseableLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,15 +36,14 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -public class SystemPropertiesHandler { +public abstract class SystemPropertiesHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesHandler.class); - private final File formalFile; - private final File tmpFile; - // private ImmutableProperties cache; + private File formalFile; + private File tmpFile; ReadWriteLock lock = new ReentrantReadWriteLock(); - private SystemPropertiesHandler(String filePath) { + public SystemPropertiesHandler(String filePath) { this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath); this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp"); } @@ -116,7 +113,7 @@ public boolean isFirstStart() { return !fileExist(); } - private void recover() throws StartupException { + protected void recover() { try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) { if (formalFile.exists() && !tmpFile.exists()) { // No need to recover @@ -139,9 +136,9 @@ private void recover() throws StartupException { return; } } catch (IOException e) { - throw new StartupException(e); + throw new RuntimeException(e); } - throw new StartupException("Should never touch here"); + throw new UnsupportedOperationException("Should never touch here"); } private void writeImpl(Properties properties, File file) throws IOException { @@ -170,20 +167,8 @@ private void replaceFormalFile() throws IOException { } } - public static void init(String filePath) throws StartupException { - Holder.INSTANCE = new SystemPropertiesHandler(filePath); - Holder.INSTANCE.recover(); - } - - public static SystemPropertiesHandler getInstance() { - if (Holder.INSTANCE == null) { - LOGGER.warn( - "The SystemPropertiesHandler's instance has not been inited yet, call getInstance may cause NullPointerException"); - } - return Holder.INSTANCE; - } - - private static class Holder { - private static SystemPropertiesHandler INSTANCE = null; + public void resetFilePath(String filePath) { + this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath); + this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp"); } } diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 6944fc819331c..ba077cc8bd9d6 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -405,7 +405,6 @@ struct TConfigNodeRegisterReq { 1: required TClusterParameters clusterParameters 2: required common.TConfigNodeLocation configNodeLocation 3: optional TNodeVersionInfo versionInfo - 4: optional bool preCheck } struct TConfigNodeRegisterResp { From 1bddd63ee830c252eab6133e2da844e851d328b3 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Tue, 18 Jun 2024 18:51:01 +0800 Subject: [PATCH 10/12] tan review --- .../ConfigNodeSystemPropertiesHandler.java | 52 ++++++++++++++++++ .../conf/DataNodeSystemPropertiesHandler.java | 54 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java new file mode 100644 index 0000000000000..9024cbef39dda --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.conf; + +import org.apache.iotdb.commons.file.SystemPropertiesHandler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class ConfigNodeSystemPropertiesHandler extends SystemPropertiesHandler { + private static final Logger LOGGER = + LoggerFactory.getLogger(ConfigNodeSystemPropertiesHandler.class); + + private ConfigNodeSystemPropertiesHandler(String filePath) { + super(filePath); + } + + public static SystemPropertiesHandler getInstance() { + if (Holder.INSTANCE == null) { + Holder.INSTANCE = + new ConfigNodeSystemPropertiesHandler( + ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + + File.separator + + ConfigNodeConstant.SYSTEM_FILE_NAME); + Holder.INSTANCE.recover(); + } + return Holder.INSTANCE; + } + + private static class Holder { + private static ConfigNodeSystemPropertiesHandler INSTANCE; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java new file mode 100644 index 0000000000000..71eeea7a9f1db --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.conf; + +import org.apache.iotdb.commons.file.SystemPropertiesHandler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +import static org.apache.iotdb.db.conf.IoTDBStartCheck.PROPERTIES_FILE_NAME; + +public class DataNodeSystemPropertiesHandler extends SystemPropertiesHandler { + private static final Logger LOGGER = + LoggerFactory.getLogger(DataNodeSystemPropertiesHandler.class); + + private DataNodeSystemPropertiesHandler(String filePath) { + super(filePath); + } + + public static SystemPropertiesHandler getInstance() { + if (Holder.INSTANCE == null) { + Holder.INSTANCE = + new DataNodeSystemPropertiesHandler( + IoTDBDescriptor.getInstance().getConfig().getSystemDir() + + File.separator + + PROPERTIES_FILE_NAME); + Holder.INSTANCE.recover(); + } + return Holder.INSTANCE; + } + + private static class Holder { + private static DataNodeSystemPropertiesHandler INSTANCE; + } +} From f37ccdee0a5c42e3e5e04ae647eddc016d31eb54 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Wed, 19 Jun 2024 10:00:03 +0800 Subject: [PATCH 11/12] synchronized --- .../confignode/conf/ConfigNodeSystemPropertiesHandler.java | 2 +- .../apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java index 9024cbef39dda..ef35b871070e7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java @@ -34,7 +34,7 @@ private ConfigNodeSystemPropertiesHandler(String filePath) { super(filePath); } - public static SystemPropertiesHandler getInstance() { + public static synchronized SystemPropertiesHandler getInstance() { if (Holder.INSTANCE == null) { Holder.INSTANCE = new ConfigNodeSystemPropertiesHandler( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java index 71eeea7a9f1db..bb0321f00a117 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java @@ -36,7 +36,7 @@ private DataNodeSystemPropertiesHandler(String filePath) { super(filePath); } - public static SystemPropertiesHandler getInstance() { + public static synchronized SystemPropertiesHandler getInstance() { if (Holder.INSTANCE == null) { Holder.INSTANCE = new DataNodeSystemPropertiesHandler( From 11fdf3d6607ca11d76a00225554c47db9c25ea14 Mon Sep 17 00:00:00 2001 From: liyuheng Date: Wed, 19 Jun 2024 10:08:06 +0800 Subject: [PATCH 12/12] instance --- .../conf/ConfigNodeSystemPropertiesHandler.java | 14 ++++++-------- .../db/conf/DataNodeSystemPropertiesHandler.java | 14 ++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java index ef35b871070e7..b92e2e57e05d6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java @@ -30,23 +30,21 @@ public class ConfigNodeSystemPropertiesHandler extends SystemPropertiesHandler { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigNodeSystemPropertiesHandler.class); + private static ConfigNodeSystemPropertiesHandler INSTANCE; + private ConfigNodeSystemPropertiesHandler(String filePath) { super(filePath); } public static synchronized SystemPropertiesHandler getInstance() { - if (Holder.INSTANCE == null) { - Holder.INSTANCE = + if (INSTANCE == null) { + INSTANCE = new ConfigNodeSystemPropertiesHandler( ConfigNodeDescriptor.getInstance().getConf().getSystemDir() + File.separator + ConfigNodeConstant.SYSTEM_FILE_NAME); - Holder.INSTANCE.recover(); + INSTANCE.recover(); } - return Holder.INSTANCE; - } - - private static class Holder { - private static ConfigNodeSystemPropertiesHandler INSTANCE; + return INSTANCE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java index bb0321f00a117..2c9d6e5e5358f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java @@ -32,23 +32,21 @@ public class DataNodeSystemPropertiesHandler extends SystemPropertiesHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeSystemPropertiesHandler.class); + private static DataNodeSystemPropertiesHandler INSTANCE; + private DataNodeSystemPropertiesHandler(String filePath) { super(filePath); } public static synchronized SystemPropertiesHandler getInstance() { - if (Holder.INSTANCE == null) { - Holder.INSTANCE = + if (INSTANCE == null) { + INSTANCE = new DataNodeSystemPropertiesHandler( IoTDBDescriptor.getInstance().getConfig().getSystemDir() + File.separator + PROPERTIES_FILE_NAME); - Holder.INSTANCE.recover(); + INSTANCE.recover(); } - return Holder.INSTANCE; - } - - private static class Holder { - private static DataNodeSystemPropertiesHandler INSTANCE; + return INSTANCE; } }