Skip to content

Import/Export schema script adapt table model #15192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CritasWang merged 2 commits into from
Apr 2, 2025
Merged

Import/Export schema script adapt table model #15192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9daa6b
export schema script adapt table model
Mar 25, 2025
a4fd92a
export schema script adapt table model
Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.tool.common.Constants;

import org.junit.AfterClass;
import org.junit.BeforeClass;
Expand Down Expand Up @@ -75,7 +76,7 @@ public void test() throws IOException {

@Override
protected void testOnWindows() throws IOException {
final String[] output = {"Export completely!"};
final String[] output = {Constants.EXPORT_COMPLETELY};
ProcessBuilder builder =
new ProcessBuilder(
"cmd.exe",
Expand Down Expand Up @@ -103,7 +104,7 @@ protected void testOnWindows() throws IOException {

prepareData();

final String[] output1 = {"Export completely!"};
final String[] output1 = {Constants.EXPORT_COMPLETELY};
ProcessBuilder builder1 =
new ProcessBuilder(
"cmd.exe",
Expand Down Expand Up @@ -131,7 +132,7 @@ protected void testOnWindows() throws IOException {

prepareData();

final String[] output2 = {"Export completely!"};
final String[] output2 = {Constants.EXPORT_COMPLETELY};
ProcessBuilder builder2 =
new ProcessBuilder(
"cmd.exe",
Expand Down Expand Up @@ -160,7 +161,7 @@ protected void testOnWindows() throws IOException {

@Override
protected void testOnUnix() throws IOException {
final String[] output = {"Export completely!"};
final String[] output = {Constants.EXPORT_COMPLETELY};
// -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**"
ProcessBuilder builder =
new ProcessBuilder(
Expand All @@ -185,7 +186,7 @@ protected void testOnUnix() throws IOException {

prepareData();

final String[] output1 = {"Export completely!"};
final String[] output1 = {Constants.EXPORT_COMPLETELY};
// -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**"
ProcessBuilder builder1 =
new ProcessBuilder(
Expand All @@ -210,7 +211,7 @@ protected void testOnUnix() throws IOException {

prepareData();

final String[] output2 = {"Export completely!"};
final String[] output2 = {Constants.EXPORT_COMPLETELY};
// -h 127.0.0.1 -p 6667 -u root -pw root -td ./ -q "select * from root.**"
ProcessBuilder builder2 =
new ProcessBuilder(
Expand Down
5 changes: 3 additions & 2 deletions integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.tool.common.Constants;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
Expand Down Expand Up @@ -81,7 +82,7 @@ public void test() throws IOException {
@Override
protected void testOnWindows() throws IOException {
prepareSchema();
final String[] output = {"Export completely!"};
final String[] output = {Constants.EXPORT_COMPLETELY};
ProcessBuilder builder =
new ProcessBuilder(
"cmd.exe",
Expand Down Expand Up @@ -109,7 +110,7 @@ protected void testOnWindows() throws IOException {
@Override
protected void testOnUnix() throws IOException {
prepareSchema();
final String[] output = {"Export completely!"};
final String[] output = {Constants.EXPORT_COMPLETELY};
ProcessBuilder builder =
new ProcessBuilder(
"bash",
Expand Down
8 changes: 6 additions & 2 deletions integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void test() throws IOException {
@Override
protected void testOnWindows() throws IOException {
final String[] output = {
"The file name must end with \"csv\"!",
"Import completely!",
};
ProcessBuilder builder =
new ProcessBuilder(
Expand All @@ -92,6 +92,8 @@ protected void testOnWindows() throws IOException {
"root",
"-pw",
"root",
"-db",
"test",
"-s",
"./",
"&",
Expand All @@ -104,7 +106,7 @@ protected void testOnWindows() throws IOException {
@Override
protected void testOnUnix() throws IOException {
final String[] output = {
"The file name must end with \"csv\"!",
"Import completely!",
};
ProcessBuilder builder =
new ProcessBuilder(
Expand All @@ -118,6 +120,8 @@ protected void testOnUnix() throws IOException {
"root",
"-pw",
"root",
"-db",
"test",
"-s",
"./");
builder.environment().put("IOTDB_HOME", homePath);
Expand Down
45 changes: 42 additions & 3 deletions iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import org.apache.tsfile.enums.TSDataType;

import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class Constants {
Expand Down Expand Up @@ -96,7 +98,7 @@ public class Constants {
public static final String DB_ARGS = "db";
public static final String DB_NAME = "database";
public static final String DB_DESC =
"The database to be exported,only takes effect when sql_dialect is table.(optional)";
"The database to be exported,only takes effect and required when sql_dialect is table .(optional)";

public static final String TABLE_ARGS = "table";
public static final String TABLE_DESC =
Expand Down Expand Up @@ -159,15 +161,21 @@ public class Constants {
public static final String COLON = ": ";
public static final String MINUS = "-";

public static final List<String> HEAD_COLUMNS =
Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression");

// export constants
public static final String EXPORT_CLI_PREFIX = "Export Data";
public static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema";

public static final String EXPORT_CLI_HEAD =
"Please obtain help information for the corresponding data type based on different parameters, for example:\n"
+ "./export_data.sh -help tsfile\n"
+ "./export_data.sh -help sql\n"
+ "./export_data.sh -help csv";

public static final String SCHEMA_CLI_CHECK_IN_HEAD =
"Too few params input, please check the following hint.";
public static final String START_TIME_ARGS = "start_time";
public static final String START_TIME_DESC = "The start time to be exported (optional)";

Expand All @@ -181,6 +189,11 @@ public class Constants {
public static final String TARGET_DIR_SUBSCRIPTION_DESC =
"Target file directory.default ./target (optional)";

public static final String TARGET_PATH_ARGS = "path";
public static final String TARGET_PATH_ARGS_NAME = "path_pattern";
public static final String TARGET_PATH_NAME = "exportPathPattern";
public static final String TARGET_PATH_DESC = "Export Path Pattern (optional)";

public static final String QUERY_COMMAND_ARGS = "q";
public static final String QUERY_COMMAND_NAME = "query";
public static final String QUERY_COMMAND_ARGS_NAME = "query_command";
Expand All @@ -200,13 +213,17 @@ public class Constants {

public static final String LINES_PER_FILE_ARGS = "lpf";
public static final String LINES_PER_FILE_NAME = "lines_per_file";
public static final String LINES_PER_FILE_DESC = "Lines per dump file.(optional)";
public static final String LINES_PER_FILE_DESC =
"Lines per dump file,only effective in tree model.(optional)";

public static final String DUMP_FILE_NAME_DEFAULT = "dump";

public static final String queryTableParamRequired =
"Either '-q' or '-table' is required when 'sql-dialect' is' table '";
public static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because ";
public static final String INSERT_SQL_MEET_ERROR_MSG = "Meet error when insert sql because ";
public static final String COLUMN_SQL_MEET_ERROR_MSG =
"Meet error when get table columns information because ";
public static final String TARGET_DATABASE_NOT_EXIST_MSG =
"The target database %s does not exist";
public static final String TARGET_TABLE_NOT_EXIST_MSG =
Expand Down Expand Up @@ -275,8 +292,19 @@ public class Constants {
public static final String HANDLER = "TsFileHandler";
public static final String CONSUMER_NAME_PREFIX = "consumer_";
public static final SimpleDateFormat DATE_FORMAT_VIEW = new SimpleDateFormat("yyyyMMddHHmmssSSS");
public static final String BASE_VIEW_TYPE = "BASE";
public static final String HEADER_VIEW_TYPE = "ViewType";
public static final String HEADER_TIMESERIES = "Timeseries";
public static final String EXPORT_COMPLETELY = "Export completely!";
public static final String EXPORT_SCHEMA_TABLES_SELECT =
"select * from information_schema.tables where database = '%s'";
public static final String EXPORT_SCHEMA_TABLES_SHOW = "show tables details from %s";
public static final String EXPORT_SCHEMA_COLUMNS_SELECT =
"select * from information_schema.columns where database like '%s' and table_name like '%s'";
public static final String EXPORT_SCHEMA_COLUMNS_DESC = "desc %s.%s details";

// import constants
public static final String IMPORT_SCHEMA_CLI_PREFIX = "ImportSchema";
public static final String IMPORT_CLI_PREFIX = "Import Data";

public static final String IMPORT_CLI_HEAD =
Expand All @@ -290,6 +318,12 @@ public class Constants {
public static final String FILE_DESC =
"The local directory path of the script file (folder) to be loaded. (required)";

public static final String FAILED_FILE_ARGS = "fd";
public static final String FAILED_FILE_NAME = "fail_dir";
public static final String FAILED_FILE_ARGS_NAME = "failDir";
public static final String FAILED_FILE_DESC =
"Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)";

public static final String ON_SUCCESS_ARGS = "os";
public static final String ON_SUCCESS_NAME = "on_success";
public static final String ON_SUCCESS_DESC =
Expand Down Expand Up @@ -325,6 +359,8 @@ public class Constants {
public static final String BATCH_POINT_SIZE_NAME = "batch_size";
public static final String BATCH_POINT_SIZE_ARGS_NAME = "batch_size";
public static final String BATCH_POINT_SIZE_DESC = "100000 (optional)";
public static final String BATCH_POINT_SIZE_LIMIT_DESC =
"10000 (only not aligned and sql_dialect tree optional)";

public static final String TIMESTAMP_PRECISION_ARGS = "tp";
public static final String TIMESTAMP_PRECISION_NAME = "timestamp_precision";
Expand All @@ -338,5 +374,8 @@ public class Constants {

public static final String LINES_PER_FAILED_FILE_ARGS = "lpf";
public static final String LINES_PER_FAILED_FILE_ARGS_NAME = "lines_per_failed_file";
public static final String LINES_PER_FAILED_FILE_DESC = "Lines per failed file.(optional)";
public static final String LINES_PER_FAILED_FILE_DESC =
"Lines per failed file,only takes effect and required when sql_dialect is table .(option)";
public static final String IMPORT_COMPLETELY = "Import completely!";
public static final int BATCH_POINT_SIZE = 10000;
}
Loading