diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java new file mode 100644 index 0000000000000..fb9907dd65a2d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tools.it; + +import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class ExportSchemaTestIT extends AbstractScript { + private static String ip; + + private static String port; + + private static String toolsPath; + + private static String libPath; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + ip = EnvFactory.getEnv().getIP(); + port = EnvFactory.getEnv().getPort(); + toolsPath = EnvFactory.getEnv().getToolsPath(); + libPath = EnvFactory.getEnv().getLibPath(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws IOException { + String os = System.getProperty("os.name").toLowerCase(); + if (os.startsWith("windows")) { + testOnWindows(); + } else { + testOnUnix(); + } + } + + @Override + protected void testOnWindows() throws IOException { + prepareSchema(); + final String[] output = {"Export completely!"}; + ProcessBuilder builder = + new ProcessBuilder( + "cmd.exe", + "/c", + toolsPath + File.separator + "export-schema.bat", + "-h", + ip, + "-p", + port, + "-u", + "root", + "-pw", + "root", + "-td", + "target", + "-path", + "root.**", + "&", + "exit", + "%^errorlevel%"); + builder.environment().put("CLASSPATH", libPath); + testOutput(builder, output, 0); + } + + @Override + protected void testOnUnix() throws IOException { + prepareSchema(); + final String[] output = {"Export completely!"}; + ProcessBuilder builder = + new ProcessBuilder( + "bash", + toolsPath + File.separator + "export-schema.sh", + "-h", + ip, + "-p", + port, + "-u", + "root", + "-pw", + "root", + "-td", + "target", + "-path", + "root.**"); + builder.environment().put("CLASSPATH", libPath); + testOutput(builder, output, 0); + } + + public void prepareSchema() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.open(); + session.createTimeseries( + "root.schema.t2.c1", + TSDataType.DOUBLE, + TSEncoding.GORILLA, + CompressionType.LZ4, + null, + null, + null, + null); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new RuntimeException(e); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java new file mode 100644 index 0000000000000..d02927d9ce9d1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tools.it; + +import org.apache.iotdb.cli.it.AbstractScript; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class ImportSchemaTestIT extends AbstractScript { + + private static String ip; + + private static String port; + + private static String toolsPath; + + private static String libPath; + + @BeforeClass + public static void setUp() { + EnvFactory.getEnv().initClusterEnvironment(); + ip = EnvFactory.getEnv().getIP(); + port = EnvFactory.getEnv().getPort(); + toolsPath = EnvFactory.getEnv().getToolsPath(); + libPath = EnvFactory.getEnv().getLibPath(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws IOException { + String os = System.getProperty("os.name").toLowerCase(); + if (os.startsWith("windows")) { + testOnWindows(); + } else { + testOnUnix(); + } + } + + @Override + protected void testOnWindows() throws IOException { + final String[] output = { + "The file name must end with \"csv\"!", + }; + ProcessBuilder builder = + new ProcessBuilder( + "cmd.exe", + "/c", + toolsPath + File.separator + "import-schema.bat", + "-h", + ip, + "-p", + port, + "-u", + "root", + "-pw", + "root", + "-f", + "./", + "&", + "exit", + "%^errorlevel%"); + builder.environment().put("CLASSPATH", libPath); + testOutput(builder, output, 0); + } + + @Override + protected void testOnUnix() throws IOException { + final String[] output = { + "The file name must end with \"csv\"!", + }; + ProcessBuilder builder = + new ProcessBuilder( + "bash", + toolsPath + File.separator + "import-schema.sh", + "-h", + ip, + "-p", + port, + "-u", + "root", + "-pw", + "root", + "-f", + "./"); + builder.environment().put("CLASSPATH", libPath); + testOutput(builder, output, 0); + } +} diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-data.bat b/iotdb-client/cli/src/assembly/resources/tools/export-data.bat index c3acfff5f92ee..8fbbd54c7f139 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/export-data.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/export-data.bat @@ -22,7 +22,7 @@ title IoTDB Export echo ```````````````````````````````````````````````` -echo Starting IoTDB Client Export Script +echo Starting IoTDB Client Export Data Script echo ```````````````````````````````````````````````` if "%OS%" == "Windows_NT" setlocal @@ -40,7 +40,7 @@ set JAVA_OPTS=-ea^ -DIOTDB_HOME="%IOTDB_HOME%" @REM ***** CLASSPATH library setting ***** -set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" REM ----------------------------------------------------------------------------- diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-data.sh b/iotdb-client/cli/src/assembly/resources/tools/export-data.sh index 8aa7491b945b2..2150eea7017c2 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/export-data.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/export-data.sh @@ -19,7 +19,7 @@ # echo ------------------------------------------ -echo Starting IoTDB Client Export Script +echo Starting IoTDB Client Export Data Script echo ------------------------------------------ if [ -z "${IOTDB_INCLUDE}" ]; then @@ -49,9 +49,7 @@ if [ -z $JAVA ] ; then exit 1; fi -for f in ${IOTDB_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done +CLASSPATH=${IOTDB_HOME}/lib/* MAIN_CLASS=org.apache.iotdb.tool.ExportData diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat b/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat new file mode 100644 index 0000000000000..8ed2b81f5c757 --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat @@ -0,0 +1,62 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off + +title IoTDB Export + +echo ```````````````````````````````````````````````` +echo Starting IoTDB Client Export Schema Script +echo ```````````````````````````````````````````````` + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ExportSchema +if NOT DEFINED JAVA_HOME goto :err + +@REM ----------------------------------------------------------------------------- +@REM JVM Opts we'll use in legacy run or installation +set JAVA_OPTS=-ea^ + -DIOTDB_HOME="%IOTDB_HOME%" + +@REM ***** CLASSPATH library setting ***** +if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" + +REM ----------------------------------------------------------------------------- + +"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %* +set ret_code=%ERRORLEVEL% +goto finally + + +:err +echo JAVA_HOME environment variable must be set! +set ret_code=1 +pause + +@REM ----------------------------------------------------------------------------- +:finally + +ENDLOCAL + +EXIT /B %ret_code% diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh b/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh new file mode 100644 index 0000000000000..b65a7d3950ada --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo ------------------------------------------ +echo Starting IoTDB Client Export Schema Script +echo ------------------------------------------ + +if [ -z "${IOTDB_INCLUDE}" ]; then + #do nothing + : +elif [ -r "$IOTDB_INCLUDE" ]; then + . "$IOTDB_INCLUDE" +fi + +if [ -z "${IOTDB_HOME}" ]; then + export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +if [ -z $JAVA ] ; then + echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr + exit 1; +fi + +CLASSPATH=${IOTDB_HOME}/lib/* + +MAIN_CLASS=org.apache.iotdb.tool.ExportSchema + +"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" +exit $? \ No newline at end of file diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.bat b/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.bat index 350e806a76957..dd69a70798d23 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.bat @@ -40,7 +40,7 @@ set JAVA_OPTS=-ea^ -DIOTDB_HOME="%IOTDB_HOME%" @REM ***** CLASSPATH library setting ***** -set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" REM ----------------------------------------------------------------------------- diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.sh b/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.sh index a19eec09f4f51..4ce0875958330 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/export-tsfile.sh @@ -49,9 +49,7 @@ if [ -z $JAVA ] ; then exit 1; fi -for f in ${IOTDB_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done +CLASSPATH=${IOTDB_HOME}/lib/* MAIN_CLASS=org.apache.iotdb.tool.ExportTsFile diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-data.bat b/iotdb-client/cli/src/assembly/resources/tools/import-data.bat index cb34c5e897a6d..43b2624195bb9 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/import-data.bat +++ b/iotdb-client/cli/src/assembly/resources/tools/import-data.bat @@ -22,7 +22,7 @@ title IoTDB Import echo ```````````````````````````````````````````````` -echo Starting IoTDB Client Import Script +echo Starting IoTDB Client Import Data Script echo ```````````````````````````````````````````````` if "%OS%" == "Windows_NT" setlocal @@ -40,7 +40,7 @@ set JAVA_OPTS=-ea^ -DIOTDB_HOME="%IOTDB_HOME%" @REM ***** CLASSPATH library setting ***** -set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" REM ----------------------------------------------------------------------------- diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-data.sh b/iotdb-client/cli/src/assembly/resources/tools/import-data.sh index 97102b854c713..b9209dcb7a98b 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/import-data.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/import-data.sh @@ -19,7 +19,7 @@ # echo ------------------------------------------ -echo Starting IoTDB Client Import Script +echo Starting IoTDB Client Import Data Script echo ------------------------------------------ if [ -z "${IOTDB_INCLUDE}" ]; then @@ -49,9 +49,7 @@ if [ -z $JAVA ] ; then exit 1; fi -for f in ${IOTDB_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done +CLASSPATH=${IOTDB_HOME}/lib/* MAIN_CLASS=org.apache.iotdb.tool.ImportData diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat b/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat new file mode 100644 index 0000000000000..46a8d5abf6ea1 --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat @@ -0,0 +1,63 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off + +title IoTDB Import + +echo ```````````````````````````````````````````````` +echo Starting IoTDB Client Import Schema Script +echo ```````````````````````````````````````````````` + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportSchema +if NOT DEFINED JAVA_HOME goto :err + +@REM ----------------------------------------------------------------------------- +@REM JVM Opts we'll use in legacy run or installation +set JAVA_OPTS=-ea^ + -DIOTDB_HOME="%IOTDB_HOME%" + +@REM ***** CLASSPATH library setting ***** +if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*" + +REM ----------------------------------------------------------------------------- + +"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %* +set ret_code=%ERRORLEVEL% +goto finally + + +:err +echo JAVA_HOME environment variable must be set! +set ret_code=1 +pause + + +@REM ----------------------------------------------------------------------------- +:finally + +ENDLOCAL + +EXIT /B %ret_code% diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh b/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh new file mode 100644 index 0000000000000..ca91293e235c0 --- /dev/null +++ b/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo ------------------------------------------ +echo Starting IoTDB Client Import Schema Script +echo ------------------------------------------ + +if [ -z "${IOTDB_INCLUDE}" ]; then + #do nothing + : +elif [ -r "$IOTDB_INCLUDE" ]; then + . "$IOTDB_INCLUDE" +fi + +if [ -z "${IOTDB_HOME}" ]; then + export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +if [ -z $JAVA ] ; then + echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr + exit 1; +fi + +CLASSPATH=${IOTDB_HOME}/lib/* + +MAIN_CLASS=org.apache.iotdb.tool.ImportSchema + +"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" +exit $? \ No newline at end of file diff --git a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh b/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh index 8e2cbc8102c33..45ac10a618f48 100644 --- a/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh +++ b/iotdb-client/cli/src/assembly/resources/tools/load-tsfile.sh @@ -107,10 +107,7 @@ IOTDB_CLI_CONF=${IOTDB_HOME}/conf MAIN_CLASS=org.apache.iotdb.cli.Cli -CLASSPATH="" -for f in ${IOTDB_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done +CLASSPATH=${IOTDB_HOME}/lib/* if [ -n "$JAVA_HOME" ]; then for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java new file mode 100644 index 0000000000000..9e91d404d1211 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.session.Session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +public abstract class AbstractSchemaTool { + + protected static final String HOST_ARGS = "h"; + protected static final String HOST_NAME = "host"; + protected static final String HOST_DEFAULT_VALUE = "127.0.0.1"; + + protected static final String HELP_ARGS = "help"; + + protected static final String PORT_ARGS = "p"; + protected static final String PORT_NAME = "port"; + protected static final String PORT_DEFAULT_VALUE = "6667"; + + protected static final String PW_ARGS = "pw"; + protected static final String PW_NAME = "password"; + protected static final String PW_DEFAULT_VALUE = "root"; + + protected static final String USERNAME_ARGS = "u"; + protected static final String USERNAME_NAME = "username"; + protected static final String USERNAME_DEFAULT_VALUE = "root"; + + protected static final String TIMEOUT_ARGS = "timeout"; + protected static final String TIMEOUT_ARGS_NAME = "queryTimeout"; + + protected static final int MAX_HELP_CONSOLE_WIDTH = 92; + + protected static final int CODE_OK = 0; + protected static final int CODE_ERROR = 1; + + protected static String host; + protected static String port; + protected static String username; + protected static String password; + + protected static String aligned; + protected static Session session; + + protected static final List HEAD_COLUMNS = + Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression"); + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class); + + protected AbstractSchemaTool() {} + + protected static String checkRequiredArg( + String arg, String name, CommandLine commandLine, String defaultValue) + throws ArgsErrorException { + String str = commandLine.getOptionValue(arg); + if (str == null) { + if (StringUtils.isNotBlank(defaultValue)) { + return defaultValue; + } + String msg = String.format("Required values for option '%s' not provided", name); + LOGGER.info(msg); + LOGGER.info("Use -help for more information"); + throw new ArgsErrorException(msg); + } + return str; + } + + protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { + host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, HOST_DEFAULT_VALUE); + port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, PORT_DEFAULT_VALUE); + username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, USERNAME_DEFAULT_VALUE); + password = checkRequiredArg(PW_ARGS, PW_NAME, commandLine, PW_DEFAULT_VALUE); + } + + protected static Options createNewOptions() { + Options options = new Options(); + + Option opHost = + Option.builder(HOST_ARGS) + .longOpt(HOST_NAME) + .optionalArg(true) + .argName(HOST_NAME) + .hasArg() + .desc("Host Name (optional)") + .build(); + options.addOption(opHost); + + Option opPort = + Option.builder(PORT_ARGS) + .longOpt(PORT_NAME) + .optionalArg(true) + .argName(PORT_NAME) + .hasArg() + .desc("Port (optional)") + .build(); + options.addOption(opPort); + + Option opUsername = + Option.builder(USERNAME_ARGS) + .longOpt(USERNAME_NAME) + .optionalArg(true) + .argName(USERNAME_NAME) + .hasArg() + .desc("Username (optional)") + .build(); + options.addOption(opUsername); + + Option opPassword = + Option.builder(PW_ARGS) + .longOpt(PW_NAME) + .optionalArg(true) + .argName(PW_NAME) + .hasArg() + .desc("Password (optional)") + .build(); + options.addOption(opPassword); + return options; + } + + /** + * write data to CSV file. + * + * @param records the records of CSV file + * @param filePath the directory to save the file + */ + public static Boolean writeCsvFile(List> records, String filePath) { + try { + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath); + for (List CsvRecord : records) { + csvPrinterWrapper.printRecordln(CsvRecord); + } + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + return true; + } catch (IOException e) { + ioTPrinter.printException(e); + return false; + } + } + + static class CSVPrinterWrapper { + private final String filePath; + private final CSVFormat csvFormat; + private CSVPrinter csvPrinter; + + public CSVPrinterWrapper(String filePath) { + this.filePath = filePath; + this.csvFormat = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setEscape('\\') + .setQuoteMode(QuoteMode.NONE) + .build(); + } + + public void printRecord(final Iterable values) throws IOException { + if (csvPrinter == null) { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } + csvPrinter.printRecord(values); + } + + public void printRecordln(final Iterable values) throws IOException { + if (csvPrinter == null) { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } + Iterator var2 = values.iterator(); + + while (var2.hasNext()) { + Object value = var2.next(); + csvPrinter.print(value); + } + csvPrinter.println(); + } + + public void print(Object value) { + if (csvPrinter == null) { + try { + csvPrinter = csvFormat.print(new PrintWriter(filePath)); + } catch (IOException e) { + ioTPrinter.printException(e); + return; + } + } + try { + csvPrinter.print(value); + } catch (IOException e) { + ioTPrinter.printException(e); + } + } + + public void println() throws IOException { + csvPrinter.println(); + } + + public void close() throws IOException { + if (csvPrinter != null) { + csvPrinter.close(); + } + } + + public void flush() throws IOException { + if (csvPrinter != null) { + csvPrinter.flush(); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java new file mode 100644 index 0000000000000..6693336e1f921 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool; + +import org.apache.iotdb.cli.type.ExitType; +import org.apache.iotdb.cli.utils.CliContext; +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.cli.utils.JlineUtils; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.jline.reader.LineReader; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.List; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; + +/** Export Schema CSV file. */ +public class ExportSchema extends AbstractSchemaTool { + + private static final String TARGET_DIR_ARGS = "t"; + private static final String TARGET_DIR_ARGS_NAME = "target"; + private static final String TARGET_DIR_NAME = "targetDir"; + + private static final String TARGET_PATH_ARGS = "path"; + private static final String TARGET_PATH_ARGS_NAME = "path_pattern"; + private static final String TARGET_PATH_NAME = "exportPathPattern"; + private static String queryPath; + + private static final String TARGET_FILE_ARGS = "pf"; + private static final String TARGET_FILE_ARGS_NAME = "path_pattern_file"; + private static final String TARGET_FILE_NAME = "exportPathPatternFile"; + + private static final String LINES_PER_FILE_ARGS = "lpf"; + private static final String LINES_PER_FILE_ARGS_NAME = "lines_per_file"; + private static final String LINES_PER_FILE_NAME = "linesPerFile"; + private static int linesPerFile = 10000; + + private static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema"; + + private static final String DUMP_FILE_NAME_DEFAULT = "dump"; + private static String targetFile = DUMP_FILE_NAME_DEFAULT; + + private static String targetDirectory; + + private static long timeout = 60000; + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + private static final String BASE_VIEW_TYPE = "BASE"; + private static final String HEADER_VIEW_TYPE = "ViewType"; + private static final String HEADER_TIMESERIES = "Timeseries"; + + @SuppressWarnings({ + "squid:S3776", + "squid:S2093" + }) // Suppress high Cognitive Complexity warning, ignore try-with-resources + /* main function of export csv tool. */ + public static void main(String[] args) { + Options options = createOptions(); + HelpFormatter hf = new HelpFormatter(); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + hf.setOptionComparator(null); // avoid reordering + hf.setWidth(MAX_HELP_CONSOLE_WIDTH); + + if (args == null || args.length == 0) { + ioTPrinter.println("Too few params input, please check the following hint."); + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + try { + commandLine = parser.parse(options, args); + } catch (ParseException e) { + ioTPrinter.println(e.getMessage()); + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + if (commandLine.hasOption(HELP_ARGS)) { + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + int exitCode = CODE_OK; + try { + parseBasicParams(commandLine); + parseSpecialParams(commandLine); + session = new Session(host, Integer.parseInt(port), username, password); + session.open(false); + if (queryPath == null) { + String pathFile = commandLine.getOptionValue(TARGET_FILE_ARGS); + String path; + if (pathFile == null) { + LineReader lineReader = + JlineUtils.getLineReader( + new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), + username, + host, + port); + path = lineReader.readLine(EXPORT_SCHEMA_CLI_PREFIX + "> please input path pattern: "); + ioTPrinter.println(path); + String[] values = path.trim().split(";"); + for (int i = 0; i < values.length; i++) { + if (StringUtils.isBlank(values[i])) { + continue; + } else { + dumpResult(values[i], i); + } + } + } else if (!pathFile.endsWith(".txt")) { + ioTPrinter.println("The file name must end with \"txt\"!"); + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } else { + dumpFromPathFile(pathFile); + } + } else { + dumpResult(queryPath, 0); + } + } catch (IOException e) { + ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); + exitCode = CODE_ERROR; + } catch (ArgsErrorException e) { + ioTPrinter.println("Invalid args: " + e.getMessage()); + exitCode = CODE_ERROR; + } catch (IoTDBConnectionException e) { + ioTPrinter.println("Connect failed because " + e.getMessage()); + exitCode = CODE_ERROR; + } finally { + if (session != null) { + try { + session.close(); + } catch (IoTDBConnectionException e) { + exitCode = CODE_ERROR; + ioTPrinter.println( + "Encounter an error when closing session, error is: " + e.getMessage()); + } + } + } + System.exit(exitCode); + } + + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_ARGS_NAME, commandLine, null); + queryPath = commandLine.getOptionValue(TARGET_PATH_ARGS); + String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS); + if (timeoutString != null) { + timeout = Long.parseLong(timeoutString); + } + if (targetFile == null) { + targetFile = DUMP_FILE_NAME_DEFAULT; + } + if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { + targetDirectory += File.separator; + } + if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) { + linesPerFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS)); + } + } + + /** + * commandline option create. + * + * @return object Options + */ + private static Options createOptions() { + Options options = createNewOptions(); + + Option opTargetFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_ARGS_NAME) + .hasArg() + .argName(TARGET_DIR_NAME) + .desc("Target File Directory (required)") + .build(); + options.addOption(opTargetFile); + + Option targetPathPattern = + Option.builder(TARGET_PATH_ARGS) + .longOpt(TARGET_PATH_ARGS_NAME) + .hasArg() + .argName(TARGET_PATH_NAME) + .desc("Export Path Pattern (optional)") + .build(); + options.addOption(targetPathPattern); + + Option targetFileName = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_ARGS_NAME) + .hasArg() + .argName(TARGET_FILE_NAME) + .desc("Export File Name (optional)") + .build(); + options.addOption(targetFileName); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_ARGS_NAME) + .hasArg() + .argName(LINES_PER_FILE_NAME) + .desc("Lines per dump file.") + .build(); + options.addOption(opLinesPerFile); + + Option opTimeout = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_ARGS_NAME) + .hasArg() + .argName(TIMEOUT_ARGS) + .desc(timeout + " Timeout for session query") + .build(); + options.addOption(opTimeout); + + Option opHelp = + Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build(); + options.addOption(opHelp); + return options; + } + + /** + * This method will be called, if the query commands are written in a sql file. + * + * @param pathFile sql file path + * @throws IOException exception + */ + private static void dumpFromPathFile(String pathFile) throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(pathFile))) { + String path; + int index = 0; + while ((path = reader.readLine()) != null) { + dumpResult(path, index); + index++; + } + } + } + + /** + * Dump files from database to CSV file. + * + * @param pattern used to be export schema + * @param index used to create dump file name + */ + private static void dumpResult(String pattern, int index) { + File file = new File(targetDirectory); + if (!file.isDirectory()) { + file.mkdir(); + } + final String path = targetDirectory + targetFile + index; + try { + SessionDataSet sessionDataSet = + session.executeQueryStatement("show timeseries " + pattern, timeout); + writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(), linesPerFile); + sessionDataSet.closeOperationHandle(); + ioTPrinter.println("Export completely!"); + } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + public static void writeCsvFile( + SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) + throws IOException, IoTDBConnectionException, StatementExecutionException { + int viewTypeIndex = headers.indexOf(HEADER_VIEW_TYPE); + int timeseriesIndex = headers.indexOf(HEADER_TIMESERIES); + + int fileIndex = 0; + boolean hasNext = true; + while (hasNext) { + int i = 0; + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + while (i++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + if (i == 1) { + csvPrinterWrapper.printRecord(HEAD_COLUMNS); + } + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE) + || !fields.get(viewTypeIndex).getStringValue().equals(BASE_VIEW_TYPE)) { + continue; + } + HEAD_COLUMNS.forEach( + column -> { + Field field = fields.get(headers.indexOf(column)); + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + } else { + hasNext = false; + break; + } + } + fileIndex++; + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java new file mode 100644 index 0000000000000..c1b9315888cc0 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java @@ -0,0 +1,653 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; + +/** Import Schema CSV file. */ +public class ImportSchema extends AbstractSchemaTool { + + private static final String FILE_ARGS = "s"; + private static final String FILE_NAME = "source"; + private static final String FILE_ARGS_NAME = "sourceDir/sourceFile"; + + private static final String FAILED_FILE_ARGS = "fd"; + private static final String FAILED_FILE_NAME = "fail_dir"; + private static final String FAILED_FILE_ARGS_NAME = "failDir"; + + private static final String ALIGNED_ARGS = "aligned"; + private static Boolean aligned = false; + + private static final String BATCH_POINT_SIZE_ARGS = "batch"; + private static final String BATCH_POINT_SIZE_NAME = "batch_size"; + private static final String BATCH_POINT_SIZE_ARGS_NAME = "batchSize"; + private static int batchPointSize = 10_000; + + private static final String CSV_SUFFIXS = "csv"; + + private static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; + private static final String LINES_PER_FAILED_FILE_NAME = "lines_per_file"; + private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "linesPerFile"; + private static final String IMPORT_SCHEMA_CLI_PREFIX = "ImportSchema"; + private static int linesPerFailedFile = 10000; + + private static String targetPath; + private static String failedFileDirectory = null; + + private static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + /** + * create the commandline options. + * + * @return object Options + */ + private static Options createOptions() { + Options options = createNewOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .hasArg() + .argName(FILE_ARGS_NAME) + .desc( + "If input a file path, load a csv file, " + + "otherwise load all csv file under this directory (required)") + .build(); + options.addOption(opFile); + + Option opFailedFile = + Option.builder(FAILED_FILE_ARGS) + .required() + .longOpt(FAILED_FILE_NAME) + .hasArg() + .argName(FAILED_FILE_ARGS_NAME) + .desc( + "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)") + .build(); + options.addOption(opFailedFile); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_ARGS) + .desc("Whether import schema as aligned timeseries(optional)") + .build(); + options.addOption(opAligned); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .hasArg() + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .desc("10000 (only not aligned optional)") + .build(); + options.addOption(opBatchPointSize); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_NAME) + .hasArg() + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .desc("Lines per failed file") + .build(); + options.addOption(opFailedLinesPerFile); + + Option opHelp = + Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build(); + options.addOption(opHelp); + return options; + } + + /** + * parse optional params + * + * @param commandLine + */ + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { + targetPath = commandLine.getOptionValue(FILE_ARGS); + if (commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS) != null) { + batchPointSize = Integer.parseInt(commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS)); + } + if (commandLine.getOptionValue(FAILED_FILE_ARGS) != null) { + failedFileDirectory = commandLine.getOptionValue(FAILED_FILE_ARGS); + File file = new File(failedFileDirectory); + if (!file.isDirectory()) { + file.mkdir(); + failedFileDirectory = file.getAbsolutePath() + File.separator; + } + } + if (commandLine.getOptionValue(ALIGNED_ARGS) != null) { + aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS)); + } + if (commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS) != null) { + linesPerFailedFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS)); + } + } + + public static void main(String[] args) throws IoTDBConnectionException { + Options options = createOptions(); + HelpFormatter hf = new HelpFormatter(); + hf.setOptionComparator(null); + hf.setWidth(MAX_HELP_CONSOLE_WIDTH); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); + + if (args == null || args.length == 0) { + ioTPrinter.println("Too few params input, please check the following hint."); + hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + try { + commandLine = parser.parse(options, args); + } catch (org.apache.commons.cli.ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + if (commandLine.hasOption(HELP_ARGS)) { + hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + try { + parseBasicParams(commandLine); + String filename = commandLine.getOptionValue(FILE_ARGS); + if (filename == null) { + hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true); + System.exit(CODE_ERROR); + } + parseSpecialParams(commandLine); + } catch (ArgsErrorException e) { + ioTPrinter.println("Args error: " + e.getMessage()); + System.exit(CODE_ERROR); + } catch (Exception e) { + ioTPrinter.println("Encounter an error, because: " + e.getMessage()); + System.exit(CODE_ERROR); + } + System.exit(importFromTargetPath(host, Integer.parseInt(port), username, password, targetPath)); + } + + /** + * Specifying a CSV file or a directory including CSV files that you want to import. This method + * can be offered to console cli to implement importing CSV file by command. + * + * @param host + * @param port + * @param username + * @param password + * @param targetPath a CSV file or a directory including CSV files + * @return the status code + * @throws IoTDBConnectionException + */ + @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources + public static int importFromTargetPath( + String host, int port, String username, String password, String targetPath) { + try { + session = new Session(host, port, username, password, false); + session.open(false); + File file = new File(targetPath); + if (file.isFile()) { + importFromSingleFile(file); + } else if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files == null) { + return CODE_OK; + } + // 按文件名排序 + Arrays.sort(files, (f1, f2) -> f1.getName().compareTo(f2.getName())); + for (File subFile : files) { + if (subFile.isFile()) { + importFromSingleFile(subFile); + } + } + } else { + ioTPrinter.println("File not found!"); + return CODE_ERROR; + } + } catch (IoTDBConnectionException e) { + ioTPrinter.println("Encounter an error when connecting to server, because " + e.getMessage()); + return CODE_ERROR; + } finally { + if (session != null) { + try { + session.close(); + } catch (IoTDBConnectionException e) { + ; + } + } + } + return CODE_OK; + } + + /** + * import the CSV file and load headers and records. + * + * @param file the File object of the CSV file that you want to import. + */ + private static void importFromSingleFile(File file) { + if (file.getName().endsWith(CSV_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println(file.getName() + " : Empty file!"); + return; + } + if (!checkHeader(headerNames)) { + return; + } + String failedFilePath = null; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + writeScheme(file.getName(), headerNames, records, failedFilePath); + } catch (IOException | IllegalPathException e) { + ioTPrinter.println( + file.getName() + " : CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println(file.getName() + " : The file name must end with \"csv\"!"); + } + } + + /** + * if the data is aligned by time, the data will be written by this method. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param failedFilePath the directory to save the failed files + */ + @SuppressWarnings("squid:S3776") + private static void writeScheme( + String fileName, List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + List paths = new ArrayList<>(); + List dataTypes = new ArrayList<>(); + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + List pathsWithAlias = new ArrayList<>(); + List dataTypesWithAlias = new ArrayList<>(); + List encodingsWithAlias = new ArrayList<>(); + List compressorsWithAlias = new ArrayList<>(); + List measurementAlias = new ArrayList<>(); + + AtomicReference hasStarted = new AtomicReference<>(false); + AtomicInteger pointSize = new AtomicInteger(0); + ArrayList> failedRecords = new ArrayList<>(); + records.forEach( + recordObj -> { + boolean failed = false; + if (!aligned) { + if (Boolean.FALSE.equals(hasStarted.get())) { + hasStarted.set(true); + } else if (pointSize.get() >= batchPointSize) { + try { + writeAndEmptyDataSet( + paths, + dataTypes, + encodings, + compressors, + null, + null, + null, + measurementAlias, + 3); + writeAndEmptyDataSet( + pathsWithAlias, + dataTypesWithAlias, + encodingsWithAlias, + compressorsWithAlias, + null, + null, + null, + null, + 3); + paths.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAlias.clear(); + pointSize.set(0); + } catch (Exception e) { + failedRecords.add((List) (List) paths); + } + } + } else { + paths.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAlias.clear(); + } + String path = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(0))); + String alias = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(1))); + String dataTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(2))); + TSDataType dataType = typeInfer(dataTypeRaw); + String encodingTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(3))); + TSEncoding encodingType = encodingInfer(encodingTypeRaw); + String compressionTypeRaw = recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(4))); + CompressionType compressionType = compressInfer(compressionTypeRaw); + if (StringUtils.isBlank(path) || path.trim().startsWith(SYSTEM_DATABASE)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': illegal path %s", + recordObj.getRecordNumber(), headerNames, path)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(dataType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown dataType %n", + recordObj.getRecordNumber(), path, dataTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(encodingType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown encodingType %n", + recordObj.getRecordNumber(), path, encodingTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else if (ObjectUtils.isEmpty(compressionType)) { + ioTPrinter.println( + String.format( + "Line '%s', column '%s': '%s' unknown compressionType %n", + recordObj.getRecordNumber(), path, compressionTypeRaw)); + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; + } else { + if (StringUtils.isBlank(alias)) { + paths.add(path); + dataTypes.add(dataType); + encodings.add(encodingType); + compressors.add(compressionType); + } else { + pathsWithAlias.add(path); + dataTypesWithAlias.add(dataType); + encodingsWithAlias.add(encodingType); + compressorsWithAlias.add(compressionType); + measurementAlias.add(alias); + } + pointSize.getAndIncrement(); + } + if (!failed && aligned) { + String deviceId = path.substring(0, path.lastIndexOf(".")); + paths.add(0, path.substring(deviceId.length() + 1)); + writeAndEmptyDataSetAligned( + deviceId, paths, dataTypes, encodings, compressors, measurementAlias, 3); + } + }); + try { + if (CollectionUtils.isNotEmpty(paths)) { + writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null, null, null, null, 3); + } + } catch (Exception e) { + paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + try { + if (CollectionUtils.isNotEmpty(pathsWithAlias)) { + writeAndEmptyDataSet( + pathsWithAlias, + dataTypesWithAlias, + encodingsWithAlias, + compressorsWithAlias, + null, + null, + null, + measurementAlias, + 3); + } + } catch (Exception e) { + pathsWithAlias.forEach(t -> failedRecords.add(Collections.singletonList(t))); + } + pointSize.set(0); + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(failedFilePath, failedRecords); + } + if (Boolean.TRUE.equals(hasStarted.get())) { + if (!failedRecords.isEmpty()) { + ioTPrinter.println(fileName + " : Import completely fail!"); + } else { + ioTPrinter.println(fileName + " : Import completely successful!"); + } + } else { + ioTPrinter.println(fileName + " : No records!"); + } + } + + private static boolean checkHeader(List headerNames) { + if (CollectionUtils.isNotEmpty(headerNames) + && new HashSet<>(headerNames).size() == HEAD_COLUMNS.size()) { + List strangers = + headerNames.stream().filter(t -> !HEAD_COLUMNS.contains(t)).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(strangers)) { + ioTPrinter.println( + "The header of the CSV file to be imported is illegal. The correct format is \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"); + return false; + } + } + return true; + } + + private static void writeFailedLinesFile( + String failedFilePath, ArrayList> failedRecords) { + int fileIndex = 0; + int from = 0; + int failedRecordsSize = failedRecords.size(); + int restFailedRecords = failedRecordsSize; + while (from < failedRecordsSize) { + int step = Math.min(restFailedRecords, linesPerFailedFile); + writeCsvFile(failedRecords.subList(from, from + step), failedFilePath + "_" + fileIndex++); + from += step; + restFailedRecords -= step; + } + } + + private static void writeAndEmptyDataSet( + List paths, + List dataTypes, + List encodings, + List compressors, + List> propsList, + List> tagsList, + List> attributesList, + List measurementAliasList, + int retryTime) + throws StatementExecutionException { + try { + session.createMultiTimeseries( + paths, + dataTypes, + encodings, + compressors, + propsList, + tagsList, + attributesList, + measurementAliasList); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + try { + session.open(); + } catch (IoTDBConnectionException ex) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } + writeAndEmptyDataSet( + paths, + dataTypes, + encodings, + compressors, + propsList, + tagsList, + attributesList, + measurementAliasList, + --retryTime); + } + } catch (StatementExecutionException e) { + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + throw e; + } + } + + private static void writeAndEmptyDataSetAligned( + String deviceId, + List measurements, + List dataTypes, + List encodings, + List compressors, + List measurementAliasList, + int retryTime) { + try { + session.createAlignedTimeseries( + deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + try { + session.open(); + } catch (IoTDBConnectionException ex) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } + writeAndEmptyDataSetAligned( + deviceId, + measurements, + dataTypes, + encodings, + compressors, + measurementAliasList, + --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } finally { + deviceId = null; + measurements.clear(); + dataTypes.clear(); + encodings.clear(); + compressors.clear(); + measurementAliasList.clear(); + } + } + + /** + * read data from the CSV file + * + * @param path + * @return CSVParser csv parser + * @throws IOException when reading the csv file failed. + */ + private static CSVParser readCsvFile(String path) throws IOException { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setQuote('`') + .setEscape('\\') + .setIgnoreEmptyLines(true) + .build() + .parse(new InputStreamReader(new FileInputStream(path))); + } + + /** + * @param typeStr + * @return + */ + private static TSDataType typeInfer(String typeStr) { + try { + if (StringUtils.isNotBlank(typeStr)) { + return TSDataType.valueOf(typeStr); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } + + private static CompressionType compressInfer(String compressionType) { + try { + if (StringUtils.isNotBlank(compressionType)) { + return CompressionType.valueOf(compressionType); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } + + private static TSEncoding encodingInfer(String encodingType) { + try { + if (StringUtils.isNotBlank(encodingType)) { + return TSEncoding.valueOf(encodingType); + } + } catch (IllegalArgumentException e) { + ; + } + return null; + } +}