From 87af16ed14f676d0137745ce487d8076f273b02d Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 6 Jun 2024 16:25:57 +0800 Subject: [PATCH 1/5] done --- .../test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 5 ++--- .../src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 4 +--- .../src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 4 +--- .../main/java/org/apache/iotdb/db/utils/DateTimeUtils.java | 4 +++- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index fe8c9c6c39e45..19b487193834b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -39,7 +39,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -import java.sql.Timestamp; import java.sql.Types; import java.time.LocalDate; import java.util.ArrayList; @@ -1173,13 +1172,13 @@ public void testNewDataType() { while (resultSet.next()) { long time = resultSet.getLong(1); Date date = resultSet.getDate(2); - Timestamp timestamp = resultSet.getTimestamp(3); + long timestamp = resultSet.getLong(3); byte[] blob = resultSet.getBytes(4); String text = resultSet.getString(5); assertEquals(2024 - 1900, date.getYear()); assertEquals(5 - 1, date.getMonth()); assertEquals(time % 31 + 1, date.getDate()); - assertEquals(time, timestamp.getTime()); + assertEquals(time, timestamp); assertArrayEquals(byteArray, blob); assertEquals(String.valueOf(time), text); } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index cca84cc8c330e..5bba6d0ea718d 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -587,6 +587,7 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { case INT32: return String.valueOf(BytesUtils.bytesToInt(values[index])); case INT64: + case TIMESTAMP: return String.valueOf(BytesUtils.bytesToLong(values[index])); case FLOAT: return String.valueOf(BytesUtils.bytesToFloat(values[index])); @@ -597,9 +598,6 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { return new String(values[index], StandardCharsets.UTF_8); case BLOB: return BytesUtils.parseBlobByteArrayToString(values[index]); - case TIMESTAMP: - return RpcUtils.formatDatetime( - RpcUtils.DEFAULT_TIME_FORMAT, "ms", BytesUtils.bytesToLong(values[index]), zoneId); case DATE: return DateUtils.formatDate(BytesUtils.bytesToInt(values[index])); default: diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index d901f36f01f15..c468be661df3e 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -548,6 +548,7 @@ public String getString(int index, TSDataType tsDataType) { case INT32: return String.valueOf(curTsBlock.getColumn(index).getInt(tsBlockIndex)); case INT64: + case TIMESTAMP: return String.valueOf(curTsBlock.getColumn(index).getLong(tsBlockIndex)); case FLOAT: return String.valueOf(curTsBlock.getColumn(index).getFloat(tsBlockIndex)); @@ -562,9 +563,6 @@ public String getString(int index, TSDataType tsDataType) { case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); - case TIMESTAMP: - return RpcUtils.formatDatetime( - timeFormat, "ms", curTsBlock.getColumn(index).getLong(tsBlockIndex), zoneId); case DATE: return DateUtils.formatDate(curTsBlock.getColumn(index).getInt(tsBlockIndex)); default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index 90e3c94305b13..1a3c394af7de2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -54,6 +54,8 @@ private DateTimeUtils() { // forbidding instantiation } + private static final String TIMESTAMP_PRECISION = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); private static Function CAST_TIMESTAMP_TO_MS; static { @@ -825,7 +827,7 @@ public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId z parser1.getInterpreter().setPredictionMode(PredictionMode.SLL); parser1.removeErrorListeners(); parser1.addErrorListener(SqlParseError.INSTANCE); - return astVisitor.parseDateExpression(parser1.dateExpression(), "ms"); + return astVisitor.parseDateExpression(parser1.dateExpression(), TIMESTAMP_PRECISION); } public static Integer parseDateExpressionToInt(String dateExpression) { From 5afb436dfe9f6b399de0afa36c5531f1bedea97f Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 6 Jun 2024 18:13:07 +0800 Subject: [PATCH 2/5] done --- .../org/apache/iotdb/cli/AbstractCli.java | 35 ++++++++++++++++++- .../apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 9 +++++ .../iotdb/commons/conf/CommonConfig.java | 2 +- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index 113b712e7f89d..66085e785928f 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -32,6 +32,9 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.ArrayUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; import java.io.BufferedReader; import java.io.IOException; @@ -598,6 +601,7 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri ctx.getPrinter().println("Msg: " + SUCCESS_MESSAGE); } } catch (Exception e) { + e.printStackTrace(); ctx.getPrinter().println("Msg: " + e); executeStatus = CODE_ERROR; } finally { @@ -663,7 +667,7 @@ private static List> cacheResult( RpcUtils.formatDatetime( timeFormat, timestampPrecision, resultSet.getLong(i), zoneId); } else { - tmp = resultSet.getString(i); + tmp = getStringByColumnIndex(ioTDBJDBCResultSet, i, zoneId); } if (tmp == null) { tmp = NULL; @@ -713,6 +717,35 @@ private static List> cacheResult( return lists; } + private static String getStringByColumnIndex( + IoTDBJDBCResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException { + TSDataType type = TSDataType.valueOf(resultSet.getColumnTypeByIndex(columnIndex)); + switch (type) { + case BOOLEAN: + return String.valueOf(resultSet.getBoolean(columnIndex)); + case INT32: + return String.valueOf(resultSet.getInt(columnIndex)); + case INT64: + return String.valueOf(resultSet.getLong(columnIndex)); + case FLOAT: + return String.valueOf(resultSet.getFloat(columnIndex)); + case DOUBLE: + return String.valueOf(resultSet.getDouble(columnIndex)); + case TEXT: + case STRING: + return resultSet.getString(columnIndex); + case BLOB: + return BytesUtils.parseBlobByteArrayToString(resultSet.getBytes(columnIndex)); + case DATE: + return DateUtils.formatDate(resultSet.getInt(columnIndex)); + case TIMESTAMP: + return RpcUtils.formatDatetime( + timeFormat, timestampPrecision, resultSet.getLong(columnIndex), zoneId); + default: + return null; + } + } + private static List> cacheTracingInfo(ResultSet resultSet, List maxSizeList) throws Exception { List> lists = new ArrayList<>(2); diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java index df558b188098f..bfc140aa7322d 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java @@ -27,6 +27,7 @@ import org.apache.iotdb.service.rpc.thrift.TSTracingInfo; import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.DateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1276,4 +1277,12 @@ public List getColumns() { public List getSgColumns() { return sgColumns; } + + public String getColumnTypeByIndex(int columnIndex) { + if (columnIndex == 1) { + // time column + return TSDataType.TIMESTAMP.name(); + } + return columnTypeList.get(columnIndex - 2); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 6e8c943400674..af05d4ca91329 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -156,7 +156,7 @@ public class CommonConfig { private long timePartitionInterval = 604_800_000; /** This variable set timestamp precision as millisecond, microsecond or nanosecond. */ - private String timestampPrecision = "ms"; + private String timestampPrecision = "us"; private boolean timestampPrecisionCheckEnabled = true; From afb11739957a70ffb6f797a061c9eda250d77a92 Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Thu, 6 Jun 2024 18:17:37 +0800 Subject: [PATCH 3/5] rollback change --- .../main/java/org/apache/iotdb/commons/conf/CommonConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index a217f33287055..1de7f33df3a83 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -159,7 +159,7 @@ public class CommonConfig { private long timePartitionInterval = 604_800_000; /** This variable set timestamp precision as millisecond, microsecond or nanosecond. */ - private String timestampPrecision = "us"; + private String timestampPrecision = "ms"; private boolean timestampPrecisionCheckEnabled = true; From 8eb1a09e4fffcd360e00c45e2ea494d7eb7edc6c Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Fri, 7 Jun 2024 00:44:47 +0800 Subject: [PATCH 4/5] fix IT --- .mvn/develocity.xml | 4 +--- .../db/it/aggregation/IoTDBAggregationIT.java | 15 ++++++--------- .../db/it/aggregation/maxby/IoTDBMaxByIT.java | 3 +-- .../db/it/aggregation/minby/IoTDBMinByIT.java | 3 +-- .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 2 +- 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml index 1e82305ef93c5..b505d1a366647 100644 --- a/.mvn/develocity.xml +++ b/.mvn/develocity.xml @@ -26,9 +26,7 @@ #{isFalse(env['GITHUB_ACTIONS'])} - - - + #{{'0.0.0.0'}} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java index bb2b1d539bff2..8eb1026121aac 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java @@ -348,8 +348,8 @@ public void countTest() { public void firstTest() { String[] retArray = new String[] { - "0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,1970-01-01T00:00:02.000Z", - "0,500,500,500.0,500,500,0x0500,1500-01-01,1970-01-01T00:00:00.500Z" + "0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,2000", + "0,500,500,500.0,500,500,0x0500,1500-01-01,500" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -453,9 +453,9 @@ public void firstTest() { public void lastTest() { String[] retArray = new String[] { - "0,8499,8499.0,8499,0x8499,8499-01-01,1970-01-01T00:00:08.499Z", - "0,1499,1499.0,1499,0x1499,1499-01-01,1970-01-01T00:00:01.499Z", - "0,2200,2200.0,2200,0x2200,2200-01-01,1970-01-01T00:00:02.200Z" + "0,8499,8499.0,8499,0x8499,8499-01-01,8499", + "0,1499,1499.0,1499,0x1499,1499-01-01,1499", + "0,2200,2200.0,2200,0x2200,2200-01-01,2200" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -727,10 +727,7 @@ public void firstLastValueTest() throws SQLException { @Test public void maxminValueTest() { String[] retArray = - new String[] { - "0,8499,500.0,999,1000,8499-01-01,1000-01-01,1970-01-01T00:00:08.499Z,1970-01-01T00:00:00.500Z", - "0,2499,500.0" - }; + new String[] {"0,8499,500.0,999,1000,8499-01-01,1000-01-01,8499,500", "0,2499,500.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java index 1ccdc8c1195e2..ccf435f836205 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = - new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; + String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java index 82867e782ccd4..77cb86e62d4d1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = - new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; + String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java index 61de72829ccc9..8f233c64a4047 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java @@ -205,7 +205,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); + assertEquals(String.valueOf(i), resultSet.getString(8)); } } } From a5d4cd2c4c33f8525edbdeeac003bd4e0c2261ed Mon Sep 17 00:00:00 2001 From: Chen YZ Date: Fri, 7 Jun 2024 11:15:37 +0800 Subject: [PATCH 5/5] fix IT --- .../org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java index 8f233c64a4047..2ddc99f6ead58 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java @@ -251,7 +251,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); + assertEquals(String.valueOf(i), resultSet.getString(8)); } } }