Skip to content

Set timestamp precision same as time column #12681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Cpaulyz merged 6 commits into from
Jun 7, 2024
Merged

Set timestamp precision same as time column #12681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
87af16e
done
Cpaulyz Jun 6, 2024
5afb436
done
Cpaulyz Jun 6, 2024
e8ee451
resolve conflict
Cpaulyz Jun 6, 2024
afb1173
rollback change
Cpaulyz Jun 6, 2024
8eb1a09
fix IT
Cpaulyz Jun 6, 2024
a5d4cd2
fix IT
Cpaulyz Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions .mvn/develocity.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@
<buildScan>
<backgroundBuildScanUpload>#{isFalse(env['GITHUB_ACTIONS'])}</backgroundBuildScanUpload>
<publishing>
<onlyIf>
<![CDATA[authenticated]]>
</onlyIf>
<onlyIf><![CDATA[authenticated]]></onlyIf>
</publishing>
<obfuscation>
<ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
Expand Down
5 changes: 2 additions & 3 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.LocalDate;
import java.util.ArrayList;
Expand Down Expand Up @@ -1173,13 +1172,13 @@ public void testNewDataType() {
while (resultSet.next()) {
long time = resultSet.getLong(1);
Date date = resultSet.getDate(2);
Timestamp timestamp = resultSet.getTimestamp(3);
long timestamp = resultSet.getLong(3);
byte[] blob = resultSet.getBytes(4);
String text = resultSet.getString(5);
assertEquals(2024 - 1900, date.getYear());
assertEquals(5 - 1, date.getMonth());
assertEquals(time % 31 + 1, date.getDate());
assertEquals(time, timestamp.getTime());
assertEquals(time, timestamp);
assertArrayEquals(byteArray, blob);
assertEquals(String.valueOf(time), text);
}
Expand Down
15 changes: 6 additions & 9 deletions integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,8 @@ public void countTest() {
public void firstTest() {
String[] retArray =
new String[] {
"0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,1970-01-01T00:00:02.000Z",
"0,500,500,500.0,500,500,0x0500,1500-01-01,1970-01-01T00:00:00.500Z"
"0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,2000",
"0,500,500,500.0,500,500,0x0500,1500-01-01,500"
};
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
Expand Down Expand Up @@ -453,9 +453,9 @@ public void firstTest() {
public void lastTest() {
String[] retArray =
new String[] {
"0,8499,8499.0,8499,0x8499,8499-01-01,1970-01-01T00:00:08.499Z",
"0,1499,1499.0,1499,0x1499,1499-01-01,1970-01-01T00:00:01.499Z",
"0,2200,2200.0,2200,0x2200,2200-01-01,1970-01-01T00:00:02.200Z"
"0,8499,8499.0,8499,0x8499,8499-01-01,8499",
"0,1499,1499.0,1499,0x1499,1499-01-01,1499",
"0,2200,2200.0,2200,0x2200,2200-01-01,2200"
};
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
Expand Down Expand Up @@ -727,10 +727,7 @@ public void firstLastValueTest() throws SQLException {
@Test
public void maxminValueTest() {
String[] retArray =
new String[] {
"0,8499,500.0,999,1000,8499-01-01,1000-01-01,1970-01-01T00:00:08.499Z,1970-01-01T00:00:00.500Z",
"0,2499,500.0"
};
new String[] {"0,8499,500.0,999,1000,8499-01-01,1000-01-01,8499,500", "0,2499,500.0"};
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {

Expand Down
3 changes: 1 addition & 2 deletions integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() {
"root.db.d1",
new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"},
new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"});
String[] retArray =
new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"};
String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"};
for (Map.Entry<String, String[]> expectedHeader : expectedHeaders.entrySet()) {
String y = expectedHeader.getKey();
resultSetEqualTest(
Expand Down
3 changes: 1 addition & 2 deletions integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() {
"root.db.d1",
new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"},
new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"});
String[] retArray =
new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"};
String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"};
for (Map.Entry<String, String[]> expectedHeader : expectedHeaders.entrySet()) {
String y = expectedHeader.getKey();
resultSetEqualTest(
Expand Down
4 changes: 2 additions & 2 deletions integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ public void testSelectorFunctions() {
} else if (j == 5) {
assertEquals("2024-01-0" + (i + 1), resultSet.getString(7));
} else if (j == 6) {
assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8));
assertEquals(String.valueOf(i), resultSet.getString(8));
}
}
}
Expand Down Expand Up @@ -251,7 +251,7 @@ public void testSelectorFunctions() {
} else if (j == 5) {
assertEquals("2024-01-0" + (i + 1), resultSet.getString(7));
} else if (j == 6) {
assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8));
assertEquals(String.valueOf(i), resultSet.getString(8));
}
}
}
Expand Down
35 changes: 34 additions & 1 deletion iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.DateUtils;

import java.io.BufferedReader;
import java.io.IOException;
Expand Down Expand Up @@ -598,6 +601,7 @@ private static int executeQuery(CliContext ctx, IoTDBConnection connection, Stri
ctx.getPrinter().println("Msg: " + SUCCESS_MESSAGE);
}
} catch (Exception e) {
e.printStackTrace();
ctx.getPrinter().println("Msg: " + e);
executeStatus = CODE_ERROR;
} finally {
Expand Down Expand Up @@ -663,7 +667,7 @@ private static List<List<String>> cacheResult(
RpcUtils.formatDatetime(
timeFormat, timestampPrecision, resultSet.getLong(i), zoneId);
} else {
tmp = resultSet.getString(i);
tmp = getStringByColumnIndex(ioTDBJDBCResultSet, i, zoneId);
}
if (tmp == null) {
tmp = NULL;
Expand Down Expand Up @@ -713,6 +717,35 @@ private static List<List<String>> cacheResult(
return lists;
}

private static String getStringByColumnIndex(
IoTDBJDBCResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException {
TSDataType type = TSDataType.valueOf(resultSet.getColumnTypeByIndex(columnIndex));
switch (type) {
case BOOLEAN:
return String.valueOf(resultSet.getBoolean(columnIndex));
case INT32:
return String.valueOf(resultSet.getInt(columnIndex));
case INT64:
return String.valueOf(resultSet.getLong(columnIndex));
case FLOAT:
return String.valueOf(resultSet.getFloat(columnIndex));
case DOUBLE:
return String.valueOf(resultSet.getDouble(columnIndex));
case TEXT:
case STRING:
return resultSet.getString(columnIndex);
case BLOB:
return BytesUtils.parseBlobByteArrayToString(resultSet.getBytes(columnIndex));
case DATE:
return DateUtils.formatDate(resultSet.getInt(columnIndex));
case TIMESTAMP:
return RpcUtils.formatDatetime(
timeFormat, timestampPrecision, resultSet.getLong(columnIndex), zoneId);
default:
return null;
}
}

private static List<List<String>> cacheTracingInfo(ResultSet resultSet, List<Integer> maxSizeList)
throws Exception {
List<List<String>> lists = new ArrayList<>(2);
Expand Down
8 changes: 8 additions & 0 deletions iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1338,4 +1338,12 @@ public List<String> getColumns() {
public List<String> getSgColumns() {
return sgColumns;
}

public String getColumnTypeByIndex(int columnIndex) {
if (columnIndex == 1) {
// time column
return TSDataType.TIMESTAMP.name();
}
return columnTypeList.get(columnIndex - 2);
}
}
4 changes: 1 addition & 3 deletions iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,7 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) {
case INT32:
return String.valueOf(BytesUtils.bytesToInt(values[index]));
case INT64:
case TIMESTAMP:
return String.valueOf(BytesUtils.bytesToLong(values[index]));
case FLOAT:
return String.valueOf(BytesUtils.bytesToFloat(values[index]));
Expand All @@ -597,9 +598,6 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) {
return new String(values[index], StandardCharsets.UTF_8);
case BLOB:
return BytesUtils.parseBlobByteArrayToString(values[index]);
case TIMESTAMP:
return RpcUtils.formatDatetime(
RpcUtils.DEFAULT_TIME_FORMAT, "ms", BytesUtils.bytesToLong(values[index]), zoneId);
case DATE:
return DateUtils.formatDate(BytesUtils.bytesToInt(values[index]));
default:
Expand Down
4 changes: 1 addition & 3 deletions iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,7 @@ public String getString(int index, TSDataType tsDataType) {
case INT32:
return String.valueOf(curTsBlock.getColumn(index).getInt(tsBlockIndex));
case INT64:
case TIMESTAMP:
return String.valueOf(curTsBlock.getColumn(index).getLong(tsBlockIndex));
case FLOAT:
return String.valueOf(curTsBlock.getColumn(index).getFloat(tsBlockIndex));
Expand All @@ -576,9 +577,6 @@ public String getString(int index, TSDataType tsDataType) {
case BLOB:
return BytesUtils.parseBlobByteArrayToString(
curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues());
case TIMESTAMP:
return RpcUtils.formatDatetime(
timeFormat, "ms", curTsBlock.getColumn(index).getLong(tsBlockIndex), zoneId);
case DATE:
return DateUtils.formatDate(curTsBlock.getColumn(index).getInt(tsBlockIndex));
default:
Expand Down
4 changes: 3 additions & 1 deletion iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ private DateTimeUtils() {
// forbidding instantiation
}

private static final String TIMESTAMP_PRECISION =
CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
private static Function<Long, Long> CAST_TIMESTAMP_TO_MS;

static {
Expand Down Expand Up @@ -825,7 +827,7 @@ public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId z
parser1.getInterpreter().setPredictionMode(PredictionMode.SLL);
parser1.removeErrorListeners();
parser1.addErrorListener(SqlParseError.INSTANCE);
return astVisitor.parseDateExpression(parser1.dateExpression(), "ms");
return astVisitor.parseDateExpression(parser1.dateExpression(), TIMESTAMP_PRECISION);
}

public static Integer parseDateExpressionToInt(String dateExpression) {
Expand Down