From 83c87126c924d873fb0654565571b79e09f7c2bb Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 18 Jun 2024 15:42:00 +0800 Subject: [PATCH 1/4] check duplicate key in start node script --- .../assembly/resources/sbin/iotdb-common.sh | 23 +++++++ .../commons/conf/ConfigurationFileUtils.java | 61 +++++++++++-------- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh index 753d1bccb0517..0236fef3f4139 100755 --- a/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh +++ b/iotdb-core/node-commons/src/assembly/resources/sbin/iotdb-common.sh @@ -150,6 +150,18 @@ checkAllConfigNodeVariables() fi } +check_config_unique() { + local key=$1 + local values=$2 + + line_count=$(echo "$values" | wc -l) + + if [ "$line_count" -gt 1 ]; then + echo "Error: Duplicate $key entries found" + exit 1 + fi +} + checkDataNodePortUsages () { echo "Checking whether the ports are already occupied..." if [ "$(id -u)" -ne 0 ]; then @@ -183,6 +195,13 @@ checkDataNodePortUsages () { else echo "Warning: cannot find iotdb-system.properties or iotdb-datanode.properties, check the default configuration" fi + + check_config_unique "dn_rpc_port" "$dn_rpc_port" + check_config_unique "dn_internal_port" "$dn_internal_port" + check_config_unique "dn_mpp_data_exchange_port" "$dn_mpp_data_exchange_port" + check_config_unique "dn_schema_region_consensus_port" "$dn_schema_region_consensus_port" + check_config_unique "dn_data_region_consensus_port" "$dn_data_region_consensus_port" + dn_rpc_port=${dn_rpc_port:-6667} dn_internal_port=${dn_internal_port:-10730} dn_mpp_data_exchange_port=${dn_mpp_data_exchange_port:-10740} @@ -271,6 +290,10 @@ checkConfigNodePortUsages () { else echo "Cannot find iotdb-system.properties or iotdb-confignode.properties, check the default configuration" fi + + check_config_unique "cn_internal_port" "$cn_internal_port" + check_config_unique "cn_consensus_port" "$cn_consensus_port" + cn_internal_port=${cn_internal_port:-10710} cn_consensus_port=${cn_consensus_port:-10720} if type lsof >/dev/null 2>&1; then diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index acccffb98bc9a..3f02f24112144 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -35,11 +36,13 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.StringJoiner; import java.util.concurrent.TimeUnit; public class ConfigurationFileUtils { @@ -49,23 +52,25 @@ public class ConfigurationFileUtils { private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); private static Logger logger = LoggerFactory.getLogger(ConfigurationFileUtils.class); private static String license = - "#\n" - + "# Licensed to the Apache Software Foundation (ASF) under one\n" - + "# or more contributor license agreements. See the NOTICE file\n" - + "# distributed with this work for additional information\n" - + "# regarding copyright ownership. The ASF licenses this file\n" - + "# to you under the Apache License, Version 2.0 (the\n" - + "# \"License\"); you may not use this file except in compliance\n" - + "# with the License. You may obtain a copy of the License at\n" - + "#\n" - + "# http://www.apache.org/licenses/LICENSE-2.0\n" - + "#\n" - + "# Unless required by applicable law or agreed to in writing,\n" - + "# software distributed under the License is distributed on an\n" - + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" - + "# KIND, either express or implied. See the License for the\n" - + "# specific language governing permissions and limitations\n" - + "# under the License."; + new StringJoiner("#") + .add("# Licensed to the Apache Software Foundation (ASF) under one") + .add("# or more contributor license agreements. See the NOTICE file") + .add("# distributed with this work for additional information") + .add("# regarding copyright ownership. The ASF licenses this file") + .add("# to you under the Apache License, Version 2.0 (the") + .add("# \"License\"); you may not use this file except in compliance") + .add("# with the License. You may obtain a copy of the License at") + .add("#") + .add("# http://www.apache.org/licenses/LICENSE-2.0") + .add("#") + .add("# Unless required by applicable law or agreed to in writing,") + .add("# software distributed under the License is distributed on an") + .add("# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY") + .add("# KIND, either express or implied. See the License for the") + .add("# specific language governing permissions and limitations") + .add("# under the License.") + .toString(); + ; // This is a temporary implementations private static final Set ignoreConfigKeys = @@ -160,7 +165,7 @@ public static String readConfigurationTemplateFile() throws IOException { BufferedReader reader = new BufferedReader(isr)) { String line; while ((line = reader.readLine()) != null) { - content.append(line).append("\n"); + content.append(line).append(System.lineSeparator()); } } catch (IOException e) { logger.warn("Failed to read configuration template", e); @@ -194,7 +199,7 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) StringBuilder contentsOfNewConfigurationFile = new StringBuilder(); for (String currentLine : lines) { if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { - contentsOfNewConfigurationFile.append(currentLine).append("\n"); + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); continue; } int equalsIndex = currentLine.indexOf('='); @@ -203,14 +208,17 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) String key = currentLine.substring(0, equalsIndex).trim(); String value = currentLine.substring(equalsIndex + 1).trim(); if (!newConfigItems.containsKey(key)) { - contentsOfNewConfigurationFile.append(currentLine).append("\n"); + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); continue; } if (newConfigItems.getProperty(key).equals(value)) { - contentsOfNewConfigurationFile.append(currentLine).append("\n"); + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); newConfigItems.remove(key); } else { - contentsOfNewConfigurationFile.append("#").append(currentLine).append("\n"); + contentsOfNewConfigurationFile + .append("#") + .append(currentLine) + .append(System.lineSeparator()); } } } @@ -222,10 +230,13 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) acquireTargetFileLock(lockFile); logger.info("Updating configuration file {}", file.getAbsolutePath()); try { - try (FileWriter writer = new FileWriter(lockFile)) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter(lockFile))) { writer.write(contentsOfNewConfigurationFile.toString()); - // add new config items - newConfigItems.store(writer, null); + // Properties.store is not used as Properties.store may generate '\' automatically + writer.write("#" + new Date().toString() + System.lineSeparator()); + for (String key : newConfigItems.stringPropertyNames()) { + writer.write(key + "=" + newConfigItems.get(key) + System.lineSeparator()); + } writer.flush(); } Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); From 3b1f29aeb939d7f074be088fb0ebc5f1adb694dd Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 18 Jun 2024 15:50:41 +0800 Subject: [PATCH 2/4] modify stop node scripts --- .../confignode/src/assembly/resources/sbin/stop-confignode.sh | 2 ++ .../datanode/src/assembly/resources/sbin/stop-datanode.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh index 5604afd3d32fa..5b2ccef01a514 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.sh @@ -18,6 +18,7 @@ # under the License. # +source "$(dirname "$0")/iotdb-common.sh" CONFIGNODE_CONF="$(dirname "$0")/../conf" if [ -f "${CONFIGNODE_CONF}/iotdb-system.properties" ]; then @@ -26,6 +27,7 @@ else cn_internal_port=$(sed '/^cn_internal_port=/!d;s/.*=//' "${CONFIGNODE_CONF}"/iotdb-confignode.properties) fi +check_config_unique "cn_internal_port" "$cn_internal_port" echo Check whether the internal_port is used..., port is "$cn_internal_port" diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh index 71bdcef0a82b5..94441cd6252a0 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.sh @@ -18,6 +18,7 @@ # under the License. # +source "$(dirname "$0")/iotdb-common.sh" DATANODE_CONF="`dirname "$0"`/../conf" if [ -f "${DATANODE_CONF}/iotdb-system.properties" ]; then @@ -26,6 +27,8 @@ else dn_rpc_port=`sed '/^dn_rpc_port=/!d;s/.*=//' ${DATANODE_CONF}/iotdb-datanode.properties` fi +check_config_unique "dn_rpc_port" "$dn_rpc_port" + force="" while true; do From f4d5d71a0e28b102d30045fee34b079fc31ed842 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 18 Jun 2024 16:13:53 +0800 Subject: [PATCH 3/4] use \n as line separator --- .../commons/conf/ConfigurationFileUtils.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java index 3f02f24112144..d4c50a92abf8e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -51,8 +51,9 @@ public class ConfigurationFileUtils { private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); private static Logger logger = LoggerFactory.getLogger(ConfigurationFileUtils.class); - private static String license = - new StringJoiner("#") + private static final String lineSeparator = "\n"; + private static final String license = + new StringJoiner(lineSeparator) .add("# Licensed to the Apache Software Foundation (ASF) under one") .add("# or more contributor license agreements. See the NOTICE file") .add("# distributed with this work for additional information") @@ -165,7 +166,7 @@ public static String readConfigurationTemplateFile() throws IOException { BufferedReader reader = new BufferedReader(isr)) { String line; while ((line = reader.readLine()) != null) { - content.append(line).append(System.lineSeparator()); + content.append(line).append(lineSeparator); } } catch (IOException e) { logger.warn("Failed to read configuration template", e); @@ -199,7 +200,7 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) StringBuilder contentsOfNewConfigurationFile = new StringBuilder(); for (String currentLine : lines) { if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { - contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + contentsOfNewConfigurationFile.append(currentLine).append(lineSeparator); continue; } int equalsIndex = currentLine.indexOf('='); @@ -208,17 +209,14 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) String key = currentLine.substring(0, equalsIndex).trim(); String value = currentLine.substring(equalsIndex + 1).trim(); if (!newConfigItems.containsKey(key)) { - contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + contentsOfNewConfigurationFile.append(currentLine).append(lineSeparator); continue; } if (newConfigItems.getProperty(key).equals(value)) { - contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + contentsOfNewConfigurationFile.append(currentLine).append(lineSeparator); newConfigItems.remove(key); } else { - contentsOfNewConfigurationFile - .append("#") - .append(currentLine) - .append(System.lineSeparator()); + contentsOfNewConfigurationFile.append("#").append(currentLine).append(lineSeparator); } } } @@ -233,9 +231,9 @@ public static void updateConfigurationFile(File file, Properties newConfigItems) try (BufferedWriter writer = new BufferedWriter(new FileWriter(lockFile))) { writer.write(contentsOfNewConfigurationFile.toString()); // Properties.store is not used as Properties.store may generate '\' automatically - writer.write("#" + new Date().toString() + System.lineSeparator()); + writer.write("#" + new Date().toString() + lineSeparator); for (String key : newConfigItems.stringPropertyNames()) { - writer.write(key + "=" + newConfigItems.get(key) + System.lineSeparator()); + writer.write(key + "=" + newConfigItems.get(key) + lineSeparator); } writer.flush(); } From 85fcfec24c5ee04dc245d1bf38aa1365d711b463 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 18 Jun 2024 18:28:44 +0800 Subject: [PATCH 4/4] fix config node send showConfiguration to config node classCastException --- .../iotdb/confignode/client/sync/SyncConfigNodeClientPool.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java index 302e587af0531..cc26be000a8eb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java @@ -92,6 +92,8 @@ public Object sendSyncRequestToConfigNodeWithRetry( return client.stopConfigNode((TConfigNodeLocation) req); case SET_CONFIGURATION: return client.setConfiguration((TSetConfigurationReq) req); + case SHOW_CONFIGURATION: + return client.showConfiguration((int) req); case SUBMIT_TEST_CONNECTION_TASK: return client.submitTestConnectionTask((TNodeLocations) req); case TEST_CONNECTION: