From 6b192950d0b2fe0ba4c3282ca1e82edcf8c11602 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 14:22:20 +0800 Subject: [PATCH 1/6] Pipe module --- .../org/apache/iotdb/pipe/api/access/Row.java | 46 ++++--- .../org/apache/iotdb/pipe/api/type/Type.java | 15 ++- .../apache/iotdb/udf/api/utils/RowImpl.java | 8 ++ .../request/PipeTransferTabletRawReq.java | 70 ++++++---- .../protocol/opcua/OpcUaConnector.java | 14 ++ .../common/row/PipeDataTypeTransformer.java | 14 +- .../db/pipe/event/common/row/PipeRow.java | 53 +++++--- .../tablet/TabletInsertionDataContainer.java | 99 +++++++++++---- .../aggregate/AggregateProcessor.java | 43 ++++++- .../aggregate/TimeSeriesRuntimeState.java | 94 ++++++++++++++ ...CustomizedReadableIntermediateResults.java | 49 +++++-- .../IntermediateResultOperator.java | 41 +++--- .../AbstractSameTypeNumericOperator.java | 46 +++++-- .../FractionPoweredSumOperator.java | 56 +++++--- .../integertype/CountOperator.java | 54 +++++--- .../datastructure/TimeSeriesWindow.java | 111 ++++++++++++++++ .../AbstractSimpleTimeWindowingProcessor.java | 52 ++++++-- .../processor/AbstractWindowingProcessor.java | 45 ++++--- .../statement/crud/InsertTabletStatement.java | 4 + .../PipeDataNodeThriftRequestTest.java | 120 +++++++++++------- .../event/PipeDataTypeTransformerTest.java | 13 +- .../event/PipeTabletInsertionEventTest.java | 40 +++++- 22 files changed, 843 insertions(+), 244 deletions(-) diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java index b3c1342347da1..f22ac5e026dc6 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java @@ -25,6 +25,7 @@ import org.apache.tsfile.read.common.Path; +import java.time.LocalDate; import java.util.List; public interface Row { @@ -44,17 +45,28 @@ public interface Row { * @param columnIndex index of the specified column * @return the int value at the specified column in this row */ - int getInt(int columnIndex); + int getInt(final int columnIndex); + + /** + * Returns the localDate value at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code Type.DATE}. + * + * @param columnIndex index of the specified column + * @return the int value at the specified column in this row + */ + LocalDate getDate(final int columnIndex); /** * Returns the long value at the specified column in this row. * - *

Users need to ensure that the data type of the specified column is {@code Type.INT64}. + *

Users need to ensure that the data type of the specified column is {@code Type.INT64} or + * {@code Type.TIMESTAMP}. * * @param columnIndex index of the specified column * @return the long value at the specified column in this row */ - long getLong(int columnIndex); + long getLong(final int columnIndex); /** * Returns the float value at the specified column in this row. @@ -64,7 +76,7 @@ public interface Row { * @param columnIndex index of the specified column * @return the float value at the specified column in this row */ - float getFloat(int columnIndex); + float getFloat(final int columnIndex); /** * Returns the double value at the specified column in this row. @@ -74,7 +86,7 @@ public interface Row { * @param columnIndex index of the specified column * @return the double value at the specified column in this row */ - double getDouble(int columnIndex); + double getDouble(final int columnIndex); /** * Returns the boolean value at the specified column in this row. @@ -84,35 +96,37 @@ public interface Row { * @param columnIndex index of the specified column * @return the boolean value at the specified column in this row */ - boolean getBoolean(int columnIndex); + boolean getBoolean(final int columnIndex); /** * Returns the Binary value at the specified column in this row. * - *

Users need to ensure that the data type of the specified column is {@code Type.TEXT}. + *

Users need to ensure that the data type of the specified column is {@code Type.TEXT}, {@code + * Type.BLOB} or {@code Type.STRING}. * * @param columnIndex index of the specified column * @return the Binary value at the specified column in this row */ - Binary getBinary(int columnIndex); + Binary getBinary(final int columnIndex); /** * Returns the String value at the specified column in this row. * - *

Users need to ensure that the data type of the specified column is {@code Type.TEXT}. + *

Users need to ensure that the data type of the specified column is {@code Type.TEXT} or + * {@code Type.STRING}. * * @param columnIndex index of the specified column * @return the String value at the specified column in this row */ - String getString(int columnIndex); + String getString(final int columnIndex); /** - * Returns the Object value at the specified column in this row. + * Returns the raw Object stored at the specified column in this row. * * @param columnIndex index of the specified column * @return the Object value at the specified column in this row */ - Object getObject(int columnIndex); + Object getObject(final int columnIndex); /** * Returns the actual data type of the value at the specified column in this row. @@ -120,7 +134,7 @@ public interface Row { * @param columnIndex index of the specified column * @return the actual data type of the value at the specified column in this row */ - Type getDataType(int columnIndex); + Type getDataType(final int columnIndex); /** * Returns {@code true} if the value of the specified column is null. @@ -128,7 +142,7 @@ public interface Row { * @param columnIndex index of the specified column * @return {@code true} if the value of the specified column is null */ - boolean isNull(int columnIndex); + boolean isNull(final int columnIndex); /** * Returns the number of columns (excluding the timestamp column). @@ -144,7 +158,7 @@ public interface Row { * @return the actual column index of the given column name * @throws PipeParameterNotValidException if the given column name is not existed in the Row */ - int getColumnIndex(Path columnName) throws PipeParameterNotValidException; + int getColumnIndex(final Path columnName) throws PipeParameterNotValidException; /** * Returns the actual column (measurement) name of the given column index. @@ -152,7 +166,7 @@ public interface Row { * @param columnIndex the column index * @return the actual column (measurement) name of the given column index */ - String getColumnName(int columnIndex); + String getColumnName(final int columnIndex); /** * Returns the column data types in the Row. diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java index 85a4390eae46e..9093133631a0e 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java @@ -36,7 +36,20 @@ public enum Type { DOUBLE((byte) 4), /* TEXT */ - TEXT((byte) 5); + TEXT((byte) 5), + + /* TsDataType.Vector and TsDataType.UNKNOWN are inner types of TsFile-module, which should not be supported in UDF APIs. To be consistent with TsDataType, the next value starts with 8 */ + /* TIMESTAMP */ + TIMESTAMP((byte) 8), + + /* DATE */ + DATE((byte) 9), + + /* BLOB */ + BLOB((byte) 10), + + /* STRING */ + STRING((byte) 11); private final byte dataType; diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java index e0ca3a53fc658..4a323b3a14b26 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java @@ -121,6 +121,14 @@ private static Type transformToUDFDataType(TSDataType tsDataType) { return Type.DOUBLE; case 5: return Type.TEXT; + case 8: + return Type.TIMESTAMP; + case 9: + return Type.DATE; + case 10: + return Type.BLOB; + case 11: + return Type.STRING; default: throw new IllegalArgumentException("Invalid input: " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java index e2512fac3ff7a..f6d677389e7dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -43,6 +43,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Arrays; import java.util.Comparator; import java.util.Objects; @@ -70,7 +71,7 @@ public InsertTabletStatement constructStatement() { try { final TSInsertTabletReq request = new TSInsertTabletReq(); - for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { + for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) { request.addToMeasurements(measurementSchema.getMeasurementId()); request.addToTypes(measurementSchema.getType().ordinal()); } @@ -84,13 +85,13 @@ public InsertTabletStatement constructStatement() { PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements())); return StatementGenerator.createStatement(request); - } catch (MetadataException e) { + } catch (final MetadataException e) { LOGGER.warn("Generate Statement from tablet {} error.", tablet, e); return null; } } - private static boolean checkSorted(Tablet tablet) { + private static boolean checkSorted(final Tablet tablet) { for (int i = 1; i < tablet.rowSize; i++) { if (tablet.timestamps[i] < tablet.timestamps[i - 1]) { return false; @@ -99,13 +100,13 @@ private static boolean checkSorted(Tablet tablet) { return true; } - private static void sortTablet(Tablet tablet) { + private static void sortTablet(final Tablet tablet) { /* * following part of code sort the batch data by time, * so we can insert continuous data in value list to get a better performance */ // sort to get index, and use index to sort value list - Integer[] index = new Integer[tablet.rowSize]; + final Integer[] index = new Integer[tablet.rowSize]; for (int i = 0; i < tablet.rowSize; i++) { index[i] = i; } @@ -113,7 +114,7 @@ private static void sortTablet(Tablet tablet) { Arrays.sort(tablet.timestamps, 0, tablet.rowSize); int columnIndex = 0; for (int i = 0; i < tablet.getSchemas().size(); i++) { - IMeasurementSchema schema = tablet.getSchemas().get(i); + final IMeasurementSchema schema = tablet.getSchemas().get(i); if (schema != null) { tablet.values[columnIndex] = sortList(tablet.values[columnIndex], schema.getType(), index); if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { @@ -133,46 +134,57 @@ private static void sortTablet(Tablet tablet) { * @return sorted list * @throws UnSupportedDataTypeException if dataType is illegal */ - private static Object sortList(Object valueList, TSDataType dataType, Integer[] index) { + private static Object sortList( + final Object valueList, final TSDataType dataType, final Integer[] index) { switch (dataType) { case BOOLEAN: - boolean[] boolValues = (boolean[]) valueList; - boolean[] sortedValues = new boolean[boolValues.length]; + final boolean[] boolValues = (boolean[]) valueList; + final boolean[] sortedValues = new boolean[boolValues.length]; for (int i = 0; i < index.length; i++) { sortedValues[i] = boolValues[index[i]]; } return sortedValues; case INT32: - int[] intValues = (int[]) valueList; - int[] sortedIntValues = new int[intValues.length]; + final int[] intValues = (int[]) valueList; + final int[] sortedIntValues = new int[intValues.length]; for (int i = 0; i < index.length; i++) { sortedIntValues[i] = intValues[index[i]]; } return sortedIntValues; + case DATE: + final LocalDate[] dateValues = (LocalDate[]) valueList; + final LocalDate[] sortedDateValues = new LocalDate[dateValues.length]; + for (int i = 0; i < index.length; i++) { + sortedDateValues[i] = dateValues[index[i]]; + } + return sortedDateValues; case INT64: - long[] longValues = (long[]) valueList; - long[] sortedLongValues = new long[longValues.length]; + case TIMESTAMP: + final long[] longValues = (long[]) valueList; + final long[] sortedLongValues = new long[longValues.length]; for (int i = 0; i < index.length; i++) { sortedLongValues[i] = longValues[index[i]]; } return sortedLongValues; case FLOAT: - float[] floatValues = (float[]) valueList; - float[] sortedFloatValues = new float[floatValues.length]; + final float[] floatValues = (float[]) valueList; + final float[] sortedFloatValues = new float[floatValues.length]; for (int i = 0; i < index.length; i++) { sortedFloatValues[i] = floatValues[index[i]]; } return sortedFloatValues; case DOUBLE: - double[] doubleValues = (double[]) valueList; - double[] sortedDoubleValues = new double[doubleValues.length]; + final double[] doubleValues = (double[]) valueList; + final double[] sortedDoubleValues = new double[doubleValues.length]; for (int i = 0; i < index.length; i++) { sortedDoubleValues[i] = doubleValues[index[i]]; } return sortedDoubleValues; case TEXT: - Binary[] binaryValues = (Binary[]) valueList; - Binary[] sortedBinaryValues = new Binary[binaryValues.length]; + case BLOB: + case STRING: + final Binary[] binaryValues = (Binary[]) valueList; + final Binary[] sortedBinaryValues = new Binary[binaryValues.length]; for (int i = 0; i < index.length; i++) { sortedBinaryValues[i] = binaryValues[index[i]]; } @@ -190,8 +202,8 @@ private static Object sortList(Object valueList, TSDataType dataType, Integer[] * @param index index * @return sorted bitMap */ - private static BitMap sortBitMap(BitMap bitMap, Integer[] index) { - BitMap sortedBitMap = new BitMap(bitMap.getSize()); + private static BitMap sortBitMap(final BitMap bitMap, final Integer[] index) { + final BitMap sortedBitMap = new BitMap(bitMap.getSize()); for (int i = 0; i < index.length; i++) { if (bitMap.isMarked(index[i])) { sortedBitMap.mark(i); @@ -202,7 +214,8 @@ private static BitMap sortBitMap(BitMap bitMap, Integer[] index) { /////////////////////////////// WriteBack & Batch /////////////////////////////// - public static PipeTransferTabletRawReq toTPipeTransferRawReq(Tablet tablet, boolean isAligned) { + public static PipeTransferTabletRawReq toTPipeTransferRawReq( + final Tablet tablet, final boolean isAligned) { final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); tabletReq.tablet = tablet; @@ -213,8 +226,8 @@ public static PipeTransferTabletRawReq toTPipeTransferRawReq(Tablet tablet, bool /////////////////////////////// Thrift /////////////////////////////// - public static PipeTransferTabletRawReq toTPipeTransferReq(Tablet tablet, boolean isAligned) - throws IOException { + public static PipeTransferTabletRawReq toTPipeTransferReq( + final Tablet tablet, final boolean isAligned) throws IOException { final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); tabletReq.tablet = tablet; @@ -233,7 +246,7 @@ public static PipeTransferTabletRawReq toTPipeTransferReq(Tablet tablet, boolean return tabletReq; } - public static PipeTransferTabletRawReq fromTPipeTransferReq(TPipeTransferReq transferReq) { + public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) { final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); tabletReq.tablet = Tablet.deserialize(transferReq.body); @@ -248,7 +261,8 @@ public static PipeTransferTabletRawReq fromTPipeTransferReq(TPipeTransferReq tra /////////////////////////////// Air Gap /////////////////////////////// - public static byte[] toTPipeTransferBytes(Tablet tablet, boolean isAligned) throws IOException { + public static byte[] toTPipeTransferBytes(final Tablet tablet, final boolean isAligned) + throws IOException { try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream); @@ -262,14 +276,14 @@ public static byte[] toTPipeTransferBytes(Tablet tablet, boolean isAligned) thro /////////////////////////////// Object /////////////////////////////// @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj; + final PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj; return Objects.equals(tablet, that.tablet) && isAligned == that.isAligned && version == that.version diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java index fa1c372db2854..158d8e84bb907 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java @@ -45,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDate; import java.util.Arrays; import java.util.Map; import java.util.UUID; @@ -260,7 +261,13 @@ private void transferTablet(final OpcUaServer server, final Tablet tablet) throw LocalizedText.english( Integer.toString(((int[]) tablet.values[columnIndex])[rowIndex]))); break; + case DATE: + eventNode.setMessage( + LocalizedText.english( + (((LocalDate[]) tablet.values[columnIndex])[rowIndex]).toString())); + break; case INT64: + case TIMESTAMP: eventNode.setMessage( LocalizedText.english( Long.toString(((long[]) tablet.values[columnIndex])[rowIndex]))); @@ -276,6 +283,8 @@ private void transferTablet(final OpcUaServer server, final Tablet tablet) throw Double.toString(((double[]) tablet.values[columnIndex])[rowIndex]))); break; case TEXT: + case BLOB: + case STRING: eventNode.setMessage( LocalizedText.english( ((Binary[]) tablet.values[columnIndex])[rowIndex].toString())); @@ -300,13 +309,18 @@ private NodeId convertToOpcDataType(final TSDataType type) { return Identifiers.Boolean; case INT32: return Identifiers.Int32; + case DATE: + return Identifiers.DateTime; case INT64: + case TIMESTAMP: return Identifiers.Int64; case FLOAT: return Identifiers.Float; case DOUBLE: return Identifiers.Double; case TEXT: + case BLOB: + case STRING: return Identifiers.String; case VECTOR: case UNKNOWN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java index 41ffd7f169109..caa98cdad6ace 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java @@ -29,7 +29,7 @@ /** Transform between {@link TSDataType} and {@link org.apache.iotdb.pipe.api.type.Type}. */ public class PipeDataTypeTransformer { - public static List transformToPipeDataTypeList(List tsDataTypeList) { + public static List transformToPipeDataTypeList(final List tsDataTypeList) { return tsDataTypeList == null ? null : tsDataTypeList.stream() @@ -37,11 +37,11 @@ public static List transformToPipeDataTypeList(List tsDataType .collect(Collectors.toList()); } - public static Type transformToPipeDataType(TSDataType tsDataType) { + public static Type transformToPipeDataType(final TSDataType tsDataType) { return tsDataType == null ? null : getPipeDataType(tsDataType.getType()); } - private static Type getPipeDataType(byte type) { + private static Type getPipeDataType(final byte type) { switch (type) { case 0: return Type.BOOLEAN; @@ -55,6 +55,14 @@ private static Type getPipeDataType(byte type) { return Type.DOUBLE; case 5: return Type.TEXT; + case 8: + return Type.TIMESTAMP; + case 9: + return Type.DATE; + case 10: + return Type.BLOB; + case 11: + return Type.STRING; default: throw new IllegalArgumentException("Invalid input: " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java index 0296935e7bb1c..6200f66536ee3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java @@ -29,6 +29,7 @@ import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.schema.MeasurementSchema; +import java.time.LocalDate; import java.util.Arrays; import java.util.List; @@ -48,15 +49,15 @@ public class PipeRow implements Row { protected final String[] columnNameStringList; public PipeRow( - int rowIndex, - String deviceId, - boolean isAligned, - MeasurementSchema[] measurementSchemaList, - long[] timestampColumn, - TSDataType[] valueColumnTypes, - Object[] valueColumns, - BitMap[] bitMaps, - String[] columnNameStringList) { + final int rowIndex, + final String deviceId, + final boolean isAligned, + final MeasurementSchema[] measurementSchemaList, + final long[] timestampColumn, + final TSDataType[] valueColumnTypes, + final Object[] valueColumns, + final BitMap[] bitMaps, + final String[] columnNameStringList) { this.rowIndex = rowIndex; this.deviceId = deviceId; this.isAligned = isAligned; @@ -74,49 +75,57 @@ public long getTime() { } @Override - public int getInt(int columnIndex) { + public int getInt(final int columnIndex) { return ((int[]) valueColumns[columnIndex])[rowIndex]; } @Override - public long getLong(int columnIndex) { + public LocalDate getDate(final int columnIndex) { + return ((LocalDate[]) valueColumns[columnIndex])[rowIndex]; + } + + @Override + public long getLong(final int columnIndex) { return ((long[]) valueColumns[columnIndex])[rowIndex]; } @Override - public float getFloat(int columnIndex) { + public float getFloat(final int columnIndex) { return ((float[]) valueColumns[columnIndex])[rowIndex]; } @Override - public double getDouble(int columnIndex) { + public double getDouble(final int columnIndex) { return ((double[]) valueColumns[columnIndex])[rowIndex]; } @Override - public boolean getBoolean(int columnIndex) { + public boolean getBoolean(final int columnIndex) { return ((boolean[]) valueColumns[columnIndex])[rowIndex]; } @Override - public org.apache.iotdb.pipe.api.type.Binary getBinary(int columnIndex) { + public org.apache.iotdb.pipe.api.type.Binary getBinary(final int columnIndex) { return PipeBinaryTransformer.transformToPipeBinary( ((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex]); } @Override - public String getString(int columnIndex) { + public String getString(final int columnIndex) { final org.apache.tsfile.utils.Binary binary = ((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex]; return binary == null ? null : binary.getStringValue(TSFileConfig.STRING_CHARSET); } @Override - public Object getObject(int columnIndex) { + public Object getObject(final int columnIndex) { switch (getDataType(columnIndex)) { case INT32: return getInt(columnIndex); + case DATE: + return getDate(columnIndex); case INT64: + case TIMESTAMP: return getLong(columnIndex); case FLOAT: return getFloat(columnIndex); @@ -125,6 +134,8 @@ public Object getObject(int columnIndex) { case BOOLEAN: return getBoolean(columnIndex); case TEXT: + case BLOB: + case STRING: return getBinary(columnIndex); default: throw new UnsupportedOperationException( @@ -135,12 +146,12 @@ public Object getObject(int columnIndex) { } @Override - public Type getDataType(int columnIndex) { + public Type getDataType(final int columnIndex) { return PipeDataTypeTransformer.transformToPipeDataType(valueColumnTypes[columnIndex]); } @Override - public boolean isNull(int columnIndex) { + public boolean isNull(final int columnIndex) { return bitMaps[columnIndex].isMarked(rowIndex); } @@ -150,7 +161,7 @@ public int size() { } @Override - public int getColumnIndex(Path columnName) throws PipeParameterNotValidException { + public int getColumnIndex(final Path columnName) throws PipeParameterNotValidException { for (int i = 0; i < columnNameStringList.length; i++) { if (columnNameStringList[i].equals(columnName.getFullPath())) { return i; @@ -161,7 +172,7 @@ public int getColumnIndex(Path columnName) throws PipeParameterNotValidException } @Override - public String getColumnName(int columnIndex) { + public String getColumnName(final int columnIndex) { return columnNameStringList[columnIndex]; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java index 83fa0497f18f1..26ae666ed2394 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java @@ -35,6 +35,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -42,6 +43,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -141,7 +143,7 @@ private void parse(final InsertRowNode insertRowNode, final PipePattern pattern) this.isAligned = insertRowNode.isAligned(); final long[] originTimestampColumn = new long[] {insertRowNode.getTime()}; - List rowIndexList = generateRowIndexList(originTimestampColumn); + final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); generateColumnIndexMapper( @@ -170,7 +172,7 @@ private void parse(final InsertRowNode insertRowNode, final PipePattern pattern) this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i]; - BitMap bitMap = new BitMap(this.timestampColumn.length); + final BitMap bitMap = new BitMap(this.timestampColumn.length); if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) { this.valueColumns[filteredColumnIndex] = null; bitMap.markAll(); @@ -209,7 +211,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa final long[] originTimestampColumn = insertTabletNode.getTimes(); final int originRowSize = originTimestampColumn.length; - List rowIndexList = generateRowIndexList(originTimestampColumn); + final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); generateColumnIndexMapper( @@ -253,7 +255,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i]; - BitMap bitMap = new BitMap(this.timestampColumn.length); + final BitMap bitMap = new BitMap(this.timestampColumn.length); if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) { this.valueColumns[filteredColumnIndex] = null; bitMap.markAll(); @@ -293,7 +295,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte final long[] originTimestampColumn = Arrays.copyOf( tablet.timestamps, tablet.rowSize); // tablet.timestamps.length == tablet.maxRowNumber - List rowIndexList = generateRowIndexList(originTimestampColumn); + final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); final List originMeasurementSchemaList = tablet.getSchemas(); @@ -344,7 +346,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList.get(i); this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i]; - BitMap bitMap = new BitMap(this.timestampColumn.length); + final BitMap bitMap = new BitMap(this.timestampColumn.length); if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) { this.valueColumns[filteredColumnIndex] = null; bitMap.markAll(); @@ -414,7 +416,7 @@ private List generateRowIndexList(final long[] originTimestampColumn) { return generateFullRowIndexList(rowCount); } - List rowIndexList = new ArrayList<>(); + final List rowIndexList = new ArrayList<>(); // We assume that `originTimestampColumn` is ordered. if (originTimestampColumn[originTimestampColumn.length - 1] < sourceEvent.getStartTime() || originTimestampColumn[0] > sourceEvent.getEndTime()) { @@ -431,7 +433,7 @@ private List generateRowIndexList(final long[] originTimestampColumn) { return rowIndexList; } - private static List generateFullRowIndexList(int rowCount) { + private static List generateFullRowIndexList(final int rowCount) { if (rowCount <= CACHED_FULL_ROW_INDEX_LIST_ROW_COUNT_UPPER) { return cachedFullRowIndexList.get(rowCount); } @@ -439,20 +441,20 @@ private static List generateFullRowIndexList(int rowCount) { } private static Object filterValueColumnsByRowIndexList( - @NonNull TSDataType type, - @NonNull Object originValueColumn, - @NonNull List rowIndexList, - boolean isSingleOriginValueColumn, - @NonNull BitMap originNullValueColumnBitmap, - @NonNull BitMap nullValueColumnBitmap /* output parameters */) { + @NonNull final TSDataType type, + @NonNull final Object originValueColumn, + @NonNull final List rowIndexList, + final boolean isSingleOriginValueColumn, + @NonNull final BitMap originNullValueColumnBitmap, + @NonNull final BitMap nullValueColumnBitmap /* output parameters */) { switch (type) { case INT32: { - int[] intValueColumns = + final int[] intValueColumns = isSingleOriginValueColumn ? new int[] {(int) originValueColumn} : (int[]) originValueColumn; - int[] valueColumns = new int[rowIndexList.size()]; + final int[] valueColumns = new int[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { valueColumns[i] = 0; @@ -463,13 +465,52 @@ private static Object filterValueColumnsByRowIndexList( } return valueColumns; } + case DATE: + { + // Always store 'LocalDate[]' to help convert to tablet + final LocalDate[] valueColumns = new LocalDate[rowIndexList.size()]; + if (isSingleOriginValueColumn && originValueColumn instanceof LocalDate + || !isSingleOriginValueColumn && originValueColumn instanceof LocalDate[]) { + // For tablet + final LocalDate[] dateValueColumns = + isSingleOriginValueColumn + ? new LocalDate[] {(LocalDate) originValueColumn} + : (LocalDate[]) originValueColumn; + + for (int i = 0; i < rowIndexList.size(); ++i) { + if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + valueColumns[i] = LocalDate.MIN; + nullValueColumnBitmap.mark(i); + } else { + valueColumns[i] = dateValueColumns[rowIndexList.get(i)]; + } + } + } else { + // For insertRowNode / insertTabletNode + final int[] intValueColumns = + isSingleOriginValueColumn + ? new int[] {(int) originValueColumn} + : (int[]) originValueColumn; + for (int i = 0; i < rowIndexList.size(); ++i) { + if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + valueColumns[i] = LocalDate.MIN; + nullValueColumnBitmap.mark(i); + } else { + valueColumns[i] = + DateUtils.parseIntToLocalDate(intValueColumns[rowIndexList.get(i)]); + } + } + } + return valueColumns; + } case INT64: + case TIMESTAMP: { - long[] longValueColumns = + final long[] longValueColumns = isSingleOriginValueColumn ? new long[] {(long) originValueColumn} : (long[]) originValueColumn; - long[] valueColumns = new long[rowIndexList.size()]; + final long[] valueColumns = new long[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { valueColumns[i] = 0L; @@ -482,11 +523,11 @@ private static Object filterValueColumnsByRowIndexList( } case FLOAT: { - float[] floatValueColumns = + final float[] floatValueColumns = isSingleOriginValueColumn ? new float[] {(float) originValueColumn} : (float[]) originValueColumn; - float[] valueColumns = new float[rowIndexList.size()]; + final float[] valueColumns = new float[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { valueColumns[i] = 0F; @@ -499,11 +540,11 @@ private static Object filterValueColumnsByRowIndexList( } case DOUBLE: { - double[] doubleValueColumns = + final double[] doubleValueColumns = isSingleOriginValueColumn ? new double[] {(double) originValueColumn} : (double[]) originValueColumn; - double[] valueColumns = new double[rowIndexList.size()]; + final double[] valueColumns = new double[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { valueColumns[i] = 0D; @@ -516,11 +557,11 @@ private static Object filterValueColumnsByRowIndexList( } case BOOLEAN: { - boolean[] booleanValueColumns = + final boolean[] booleanValueColumns = isSingleOriginValueColumn ? new boolean[] {(boolean) originValueColumn} : (boolean[]) originValueColumn; - boolean[] valueColumns = new boolean[rowIndexList.size()]; + final boolean[] valueColumns = new boolean[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { valueColumns[i] = false; @@ -532,12 +573,14 @@ private static Object filterValueColumnsByRowIndexList( return valueColumns; } case TEXT: + case BLOB: + case STRING: { - Binary[] binaryValueColumns = + final Binary[] binaryValueColumns = isSingleOriginValueColumn ? new Binary[] {(Binary) originValueColumn} : (Binary[]) originValueColumn; - Binary[] valueColumns = new Binary[rowIndexList.size()]; + final Binary[] valueColumns = new Binary[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { if (Objects.isNull(binaryValueColumns[rowIndexList.get(i)]) || Objects.isNull(binaryValueColumns[rowIndexList.get(i)].getValues()) @@ -558,7 +601,7 @@ private static Object filterValueColumnsByRowIndexList( //////////////////////////// process //////////////////////////// - public List processRowByRow(BiConsumer consumer) { + public List processRowByRow(final BiConsumer consumer) { if (valueColumns.length == 0 || timestampColumn.length == 0) { return Collections.emptyList(); } @@ -581,7 +624,7 @@ public List processRowByRow(BiConsumer return rowCollector.convertToTabletInsertionEvents(shouldReport); } - public List processTablet(BiConsumer consumer) { + public List processTablet(final BiConsumer consumer) { final PipeRowCollector rowCollector = new PipeRowCollector(pipeTaskMeta, sourceEvent); consumer.accept(convertToTablet(), rowCollector); return rowCollector.convertToTabletInsertionEvents(shouldReport); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index dde1fab5a2802..0aeae73da75b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -56,14 +56,17 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -450,7 +453,13 @@ private Map> processRow( state.updateWindows( timestamp, row.getInt(index), outputMinReportIntervalMilliseconds); break; + case DATE: + result = + state.updateWindows( + timestamp, row.getDate(index), outputMinReportIntervalMilliseconds); + break; case INT64: + case TIMESTAMP: result = state.updateWindows( timestamp, row.getLong(index), outputMinReportIntervalMilliseconds); @@ -466,10 +475,16 @@ private Map> processRow( timestamp, row.getDouble(index), outputMinReportIntervalMilliseconds); break; case TEXT: + case STRING: result = state.updateWindows( timestamp, row.getString(index), outputMinReportIntervalMilliseconds); break; + case BLOB: + result = + state.updateWindows( + timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds); + break; default: throw new UnsupportedOperationException( String.format("The type %s is not supported", row.getDataType(index))); @@ -620,7 +635,11 @@ public void collectWindowOutputs( case INT32: valueColumns[columnIndex] = new int[distinctOutputs.size()]; break; + case DATE: + valueColumns[columnIndex] = new LocalDate[distinctOutputs.size()]; + break; case INT64: + case TIMESTAMP: valueColumns[columnIndex] = new long[distinctOutputs.size()]; break; case FLOAT: @@ -630,7 +649,9 @@ public void collectWindowOutputs( valueColumns[columnIndex] = new double[distinctOutputs.size()]; break; case TEXT: - valueColumns[columnIndex] = new String[distinctOutputs.size()]; + case BLOB: + case STRING: + valueColumns[columnIndex] = new Binary[distinctOutputs.size()]; break; default: throw new UnsupportedOperationException( @@ -649,7 +670,12 @@ public void collectWindowOutputs( ((int[]) valueColumns[columnIndex])[rowIndex] = (int) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; + case DATE: + ((LocalDate[]) valueColumns[columnIndex])[rowIndex] = + (LocalDate) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); + break; case INT64: + case TIMESTAMP: ((long[]) valueColumns[columnIndex])[rowIndex] = (long) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; @@ -662,8 +688,19 @@ public void collectWindowOutputs( (double) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; case TEXT: - ((String[]) valueColumns[columnIndex])[rowIndex] = - (String) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); + case STRING: + ((Binary[]) valueColumns[columnIndex])[rowIndex] = + aggregatedResults.get(columnNameStringList[columnIndex]).getRight() + instanceof Binary + ? (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight() + : new Binary( + (String) + aggregatedResults.get(columnNameStringList[columnIndex]).getRight(), + TSFileConfig.STRING_CHARSET); + break; + case BLOB: + ((Binary[]) valueColumns[columnIndex])[rowIndex] = + (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; default: throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java index 64864e087250e..2cb9a41469032 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; @@ -33,6 +34,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -165,6 +167,52 @@ public Pair, Pair> updateWindows( return output; } + public Pair, Pair> updateWindows( + final long timestamp, final LocalDate value, final long outputMinReportIntervalMilliseconds) + throws IOException { + Pair, Pair> output = null; + + if (timestamp <= lastReportTimeStamp) { + return null; + } + final Set addedWindows = + windowingProcessor.mayAddWindow(currentOpeningWindows, timestamp, value); + if (Objects.nonNull(addedWindows)) { + addedWindows.forEach( + window -> + window.initWindow( + intermediateResultName2OperatorSupplierMap, + aggregatorOutputName2OperatorMap, + systemParameters)); + } + final Iterator windowIterator = currentOpeningWindows.iterator(); + while (windowIterator.hasNext()) { + final TimeSeriesWindow window = windowIterator.next(); + final Pair stateWindowOutputPair = + window.updateIntermediateResult(timestamp, value); + if (Objects.isNull(stateWindowOutputPair)) { + continue; + } + if (stateWindowOutputPair.getLeft().isEmit() + && Objects.nonNull(stateWindowOutputPair.getRight())) { + outputList.add(stateWindowOutputPair.getRight()); + lastReportTimeStamp = + Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + } + if (stateWindowOutputPair.getLeft().isPurge()) { + windowIterator.remove(); + } + } + if (!outputList.isEmpty()) { + output = + new Pair<>( + new ArrayList<>(outputList), + getTimestampWindowBufferPair(outputMinReportIntervalMilliseconds)); + outputList.clear(); + } + return output; + } + public Pair, Pair> updateWindows( final long timestamp, final long value, final long outputMinReportIntervalMilliseconds) throws IOException { @@ -349,6 +397,52 @@ public Pair, Pair> updateWindows( return output; } + public Pair, Pair> updateWindows( + final long timestamp, final Binary value, final long outputMinReportIntervalMilliseconds) + throws IOException { + Pair, Pair> output = null; + + if (timestamp <= lastReportTimeStamp) { + return null; + } + final Set addedWindows = + windowingProcessor.mayAddWindow(currentOpeningWindows, timestamp, value); + if (Objects.nonNull(addedWindows)) { + addedWindows.forEach( + window -> + window.initWindow( + intermediateResultName2OperatorSupplierMap, + aggregatorOutputName2OperatorMap, + systemParameters)); + } + final Iterator windowIterator = currentOpeningWindows.iterator(); + while (windowIterator.hasNext()) { + final TimeSeriesWindow window = windowIterator.next(); + final Pair stateWindowOutputPair = + window.updateIntermediateResult(timestamp, value); + if (Objects.isNull(stateWindowOutputPair)) { + continue; + } + if (stateWindowOutputPair.getLeft().isEmit() + && Objects.nonNull(stateWindowOutputPair.getRight())) { + outputList.add(stateWindowOutputPair.getRight()); + lastReportTimeStamp = + Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime()); + } + if (stateWindowOutputPair.getLeft().isPurge()) { + windowIterator.remove(); + } + } + if (!outputList.isEmpty()) { + output = + new Pair<>( + new ArrayList<>(outputList), + getTimestampWindowBufferPair(outputMinReportIntervalMilliseconds)); + outputList.clear(); + } + return output; + } + public List forceOutput() { return currentOpeningWindows.stream() .map(TimeSeriesWindow::forceOutput) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java index 9af6bcae73631..f9c9c5f828809 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java @@ -19,20 +19,28 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.pipe.api.type.Binary; +import org.apache.iotdb.rpc.RpcUtils; + import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.Map; public class CustomizedReadableIntermediateResults { private final Map> intermediateResults; public CustomizedReadableIntermediateResults( - Map> intermediateResults) { + final Map> intermediateResults) { this.intermediateResults = intermediateResults; } - public boolean getBoolean(String key) { + public boolean getBoolean(final String key) { final Pair typeResultPair = intermediateResults.get(key); if (typeResultPair.getLeft() == TSDataType.BOOLEAN) { return (boolean) typeResultPair.getRight(); @@ -41,7 +49,7 @@ public boolean getBoolean(String key) { String.format("The type %s cannot be casted to boolean.", typeResultPair.getLeft())); } - public int getInt(String key) { + public int getInt(final String key) { final Pair typeResultPair = intermediateResults.get(key); final TSDataType type = typeResultPair.getLeft(); @@ -49,7 +57,10 @@ public int getInt(String key) { switch (type) { case INT32: return (int) value; + case DATE: + return DateUtils.parseDateExpressionToInt((LocalDate) value); case INT64: + case TIMESTAMP: return (int) (long) value; case FLOAT: return (int) (float) value; @@ -61,7 +72,7 @@ public int getInt(String key) { } } - public long getLong(String key) { + public long getLong(final String key) { final Pair typeResultPair = intermediateResults.get(key); final TSDataType type = typeResultPair.getLeft(); @@ -69,7 +80,10 @@ public long getLong(String key) { switch (type) { case INT32: return (int) value; + case DATE: + return DateUtils.parseDateExpressionToInt((LocalDate) value); case INT64: + case TIMESTAMP: return (long) value; case FLOAT: return (long) (float) value; @@ -81,7 +95,7 @@ public long getLong(String key) { } } - public float getFloat(String key) { + public float getFloat(final String key) { final Pair typeResultPair = intermediateResults.get(key); final TSDataType type = typeResultPair.getLeft(); @@ -89,7 +103,10 @@ public float getFloat(String key) { switch (type) { case INT32: return (int) value; + case DATE: + return DateUtils.parseDateExpressionToInt((LocalDate) value); case INT64: + case TIMESTAMP: return (long) value; case FLOAT: return (float) value; @@ -101,7 +118,7 @@ public float getFloat(String key) { } } - public double getDouble(String key) { + public double getDouble(final String key) { final Pair typeResultPair = intermediateResults.get(key); final TSDataType type = typeResultPair.getLeft(); @@ -109,7 +126,10 @@ public double getDouble(String key) { switch (type) { case INT32: return (int) value; + case DATE: + return DateUtils.parseDateExpressionToInt((LocalDate) value); case INT64: + case TIMESTAMP: return (long) value; case FLOAT: return (float) value; @@ -123,7 +143,7 @@ public double getDouble(String key) { // Note: This method will cast any decimal types to string without throwing // any exceptions. - public String getString(String key) { + public String getString(final String key) { final Pair typeResultPair = intermediateResults.get(key); final TSDataType type = typeResultPair.getLeft(); @@ -133,14 +153,25 @@ public String getString(String key) { return Boolean.toString((boolean) value); case INT32: return Integer.toString((int) value); + case DATE: + return ((LocalDate) value).toString(); case INT64: return Long.toString((long) value); + case TIMESTAMP: + return RpcUtils.formatDatetime( + RpcUtils.DEFAULT_TIME_FORMAT, + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(), + (long) value, + ZoneId.systemDefault()); case FLOAT: return Float.toString((float) value); case DOUBLE: return Double.toString((double) value); case TEXT: + case STRING: return (String) value; + case BLOB: + return BytesUtils.parseBlobByteArrayToString(((Binary) value).getValues()); default: throw new UnsupportedOperationException( String.format("The type %s cannot be casted to string.", typeResultPair.getLeft())); @@ -148,11 +179,11 @@ public String getString(String key) { } // The caller may cast the object by itself. - public Object getObject(String key) { + public Object getObject(final String key) { return intermediateResults.get(key).getRight(); } - public TSDataType getType(String key) { + public TSDataType getType(final String key) { return intermediateResults.get(key).getLeft(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java index d68191885775a..95e1a9cb0d1e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java @@ -19,12 +19,15 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult; +import org.apache.iotdb.pipe.api.type.Binary; + import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Map; /** @@ -53,7 +56,7 @@ public interface IntermediateResultOperator { * * @param systemParams the system parameters */ - void configureSystemParameters(Map systemParams); + void configureSystemParameters(final Map systemParams); /** * The operator may init its value and typically set the output intermediate value type by the @@ -66,17 +69,21 @@ public interface IntermediateResultOperator { * @param initialInput the initial data * @return if the input type is supported */ - boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp); + + boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp); + + boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp); - boolean initAndGetIsSupport(int initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp); - boolean initAndGetIsSupport(long initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp); - boolean initAndGetIsSupport(float initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp); - boolean initAndGetIsSupport(double initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp); - boolean initAndGetIsSupport(String initialInput, long initialTimestamp); + boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp); /** * Use the input to update the intermediate result. The input is all raw types instead of Object @@ -84,17 +91,21 @@ public interface IntermediateResultOperator { * * @param input the inputs */ - void updateValue(boolean input, long timestamp); + void updateValue(final boolean input, final long timestamp); + + void updateValue(final int input, final long timestamp); + + void updateValue(final LocalDate input, final long timestamp); - void updateValue(int input, long timestamp); + void updateValue(final long input, final long timestamp); - void updateValue(long input, long timestamp); + void updateValue(final float input, final long timestamp); - void updateValue(float input, long timestamp); + void updateValue(final double input, final long timestamp); - void updateValue(double input, long timestamp); + void updateValue(final String input, final long timestamp); - void updateValue(String input, long timestamp); + void updateValue(final Binary input, final long timestamp); /** * Get the result and its type to calculate the aggregated value. If the type is List, Map or @@ -106,8 +117,8 @@ public interface IntermediateResultOperator { * Serialize its intermediate result to the outputStream to allow shutdown restart. The operator * may as well serialize its own status. */ - void serialize(DataOutputStream outputStream) throws IOException; + void serialize(final DataOutputStream outputStream) throws IOException; /** Deserialize the object from byteBuffer to allow shutdown restart */ - void deserialize(ByteBuffer byteBuffer) throws IOException; + void deserialize(final ByteBuffer byteBuffer) throws IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java index 8069c17c03728..eab8b64256047 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric; import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; @@ -28,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Map; /** @@ -43,56 +45,78 @@ public abstract class AbstractSameTypeNumericOperator implements IntermediateRes protected double doubleValue; @Override - public void configureSystemParameters(Map systemParams) { + public void configureSystemParameters(final Map systemParams) { // Do nothing } @Override - public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) { return false; } @Override - public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) { outPutDataType = TSDataType.INT32; return true; } @Override - public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) { + return false; + } + + @Override + public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) { outPutDataType = TSDataType.INT64; return true; } @Override - public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) { outPutDataType = TSDataType.FLOAT; return true; } @Override - public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) { outPutDataType = TSDataType.DOUBLE; return true; } @Override - public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) { + return false; + } + + @Override + public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) { return false; } @Override - public void updateValue(boolean input, long timestamp) { + public void updateValue(final boolean input, final long timestamp) { throw new UnsupportedOperationException( "AbstractSameTypeNumericOperator does not support boolean input"); } @Override - public void updateValue(String input, long timestamp) { + public void updateValue(final LocalDate input, final long timestamp) { + throw new UnsupportedOperationException( + "AbstractSameTypeNumericOperator does not support date input"); + } + + @Override + public void updateValue(final String input, final long timestamp) { throw new UnsupportedOperationException( "AbstractSameTypeNumericOperator does not support string input"); } + @Override + public void updateValue(final Binary input, final long timestamp) { + throw new UnsupportedOperationException( + "AbstractSameTypeNumericOperator does not support binary input"); + } + @Override public Pair getResult() { switch (outPutDataType) { @@ -110,7 +134,7 @@ public Pair getResult() { } @Override - public void serialize(DataOutputStream outputStream) throws IOException { + public void serialize(final DataOutputStream outputStream) throws IOException { outPutDataType.serializeTo(outputStream); switch (outPutDataType) { case INT32: @@ -131,7 +155,7 @@ public void serialize(DataOutputStream outputStream) throws IOException { } @Override - public void deserialize(ByteBuffer byteBuffer) throws IOException { + public void deserialize(final ByteBuffer byteBuffer) throws IOException { outPutDataType = TSDataType.deserializeFrom(byteBuffer); switch (outPutDataType) { case INT32: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java index e80f05e455ada..a49d90c6ede7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.doubletype; import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; @@ -28,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Map; public class FractionPoweredSumOperator implements IntermediateResultOperator { @@ -41,7 +43,7 @@ public class FractionPoweredSumOperator implements IntermediateResultOperator { * * @param power the power to use */ - public FractionPoweredSumOperator(double power) { + public FractionPoweredSumOperator(final double power) { this.power = power; } @@ -51,88 +53,110 @@ public String getName() { } @Override - public void configureSystemParameters(Map systemParams) { + public void configureSystemParameters(final Map systemParams) { // Do nothing } @Override - public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) { return false; } @Override - public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) { sum = Math.pow(initialInput, power); return true; } @Override - public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) { + return false; + } + + @Override + public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) { sum = Math.pow(initialInput, power); return true; } @Override - public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) { sum = Math.pow(initialInput, power); return true; } @Override - public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) { sum = Math.pow(initialInput, power); return true; } @Override - public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) { + return false; + } + + @Override + public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) { return false; } @Override - public void updateValue(boolean input, long timestamp) { + public void updateValue(final boolean input, final long timestamp) { throw new UnsupportedOperationException( "FractionPoweredSumOperator does not support boolean input"); } @Override - public void updateValue(int input, long timestamp) { + public void updateValue(final int input, final long timestamp) { sum += Math.pow(input, power); } @Override - public void updateValue(long input, long timestamp) { + public void updateValue(final LocalDate input, final long timestamp) { + throw new UnsupportedOperationException( + "FractionPoweredSumOperator does not support date input"); + } + + @Override + public void updateValue(final long input, final long timestamp) { sum += Math.pow(input, power); } @Override - public void updateValue(float input, long timestamp) { + public void updateValue(final float input, final long timestamp) { sum += Math.pow(input, power); } @Override - public void updateValue(double input, long timestamp) { + public void updateValue(final double input, final long timestamp) { sum += Math.pow(input, power); } @Override - public void updateValue(String input, long timestamp) { + public void updateValue(final String input, final long timestamp) { throw new UnsupportedOperationException( "FractionPoweredSumOperator does not support string input"); } + @Override + public void updateValue(final Binary initialInput, final long initialTimestamp) { + throw new UnsupportedOperationException( + "FractionPoweredSumOperator does not support binary input"); + } + @Override public Pair getResult() { return new Pair<>(TSDataType.DOUBLE, sum); } @Override - public void serialize(DataOutputStream outputStream) throws IOException { + public void serialize(final DataOutputStream outputStream) throws IOException { ReadWriteIOUtils.write(sum, outputStream); } @Override - public void deserialize(ByteBuffer byteBuffer) throws IOException { + public void deserialize(final ByteBuffer byteBuffer) throws IOException { sum = ReadWriteIOUtils.readDouble(byteBuffer); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java index 4e5397fe6735f..a42dfa179425a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.integertype; import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; @@ -28,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Map; public class CountOperator implements IntermediateResultOperator { @@ -39,73 +41,95 @@ public String getName() { } @Override - public void configureSystemParameters(Map systemParams) { + public void configureSystemParameters(final Map systemParams) { // Do nothing } @Override - public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) { + public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) { count = 1; return true; } @Override - public void updateValue(boolean input, long timestamp) { + public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) { + count = 1; + return true; + } + + @Override + public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) { + count = 1; + return true; + } + + @Override + public void updateValue(final boolean input, final long timestamp) { + ++count; + } + + @Override + public void updateValue(final int input, final long timestamp) { + ++count; + } + + @Override + public void updateValue(LocalDate input, long timestamp) { ++count; } @Override - public void updateValue(int input, long timestamp) { + public void updateValue(final long input, final long timestamp) { ++count; } @Override - public void updateValue(long input, long timestamp) { + public void updateValue(final float input, final long timestamp) { ++count; } @Override - public void updateValue(float input, long timestamp) { + public void updateValue(final double input, final long timestamp) { ++count; } @Override - public void updateValue(double input, long timestamp) { + public void updateValue(final String input, final long timestamp) { ++count; } @Override - public void updateValue(String input, long timestamp) { + public void updateValue(Binary input, long timestamp) { ++count; } @@ -115,12 +139,12 @@ public Pair getResult() { } @Override - public void serialize(DataOutputStream outputStream) throws IOException { + public void serialize(final DataOutputStream outputStream) throws IOException { ReadWriteIOUtils.write(count, outputStream); } @Override - public void deserialize(ByteBuffer byteBuffer) throws IOException { + public void deserialize(final ByteBuffer byteBuffer) throws IOException { count = ReadWriteIOUtils.readInt(byteBuffer); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java index 511d9a61fb00e..6cd046fb594fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.CustomizedReadableIntermediateResults; import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator; import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; @@ -33,6 +34,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -213,6 +215,61 @@ public Pair updateIntermediateResult( return state.isEmit() ? stateOutputPair : null; } + public Pair updateIntermediateResult( + final long timestamp, final LocalDate value) { + final Pair stateOutputPair = + processor.updateAndMaySetWindowState(this, timestamp, value); + final WindowState state = stateOutputPair.getLeft(); + + if (state.isEmitWithoutCompute()) { + stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.DATE)); + } + + if (state.isCalculate()) { + final Iterator>> iterator = + intermediateResultName2tsTypeAndOperatorMap.entrySet().iterator(); + Map.Entry> entry; + + while (iterator.hasNext()) { + entry = iterator.next(); + final IntermediateResultOperator operator = entry.getValue().getRight(); + if (entry.getValue().getLeft() == TSDataType.UNKNOWN) { + if (!operator.initAndGetIsSupport(value, timestamp)) { + // Remove unsupported aggregated results + aggregatedOutputName2OperatorMap + .entrySet() + .removeIf( + entry1 -> + entry1 + .getValue() + .getDeclaredIntermediateValueNames() + .contains(operator.getName())); + // If no aggregated values can be calculated, purge the window + if (aggregatedOutputName2OperatorMap.isEmpty()) { + return new Pair<>(WindowState.PURGE, null); + } + // Remove unsupported intermediate values + iterator.remove(); + continue; + } + entry.getValue().setLeft(TSDataType.DATE); + } else if (entry.getValue().getLeft() != TSDataType.DATE) { + LOGGER.warn( + "Different data type encountered in one window, will purge. Previous type: {}, now type: {}", + entry.getValue().getLeft(), + TSDataType.DATE); + return new Pair<>(WindowState.PURGE, null); + } else { + operator.updateValue(value, timestamp); + } + } + } + if (state.isEmitWithCompute()) { + stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.DATE)); + } + return state.isEmit() ? stateOutputPair : null; + } + public Pair updateIntermediateResult( final long timestamp, final long value) { final Pair stateOutputPair = @@ -431,6 +488,60 @@ public Pair updateIntermediateResult( return state.isEmit() ? stateOutputPair : null; } + public Pair updateIntermediateResult( + final long timestamp, final Binary value) { + final Pair stateOutputPair = + processor.updateAndMaySetWindowState(this, timestamp, value); + final WindowState state = stateOutputPair.getLeft(); + + if (state.isEmitWithoutCompute()) { + stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.BLOB)); + } + if (state.isCalculate()) { + final Iterator>> iterator = + intermediateResultName2tsTypeAndOperatorMap.entrySet().iterator(); + Map.Entry> entry; + + while (iterator.hasNext()) { + entry = iterator.next(); + final IntermediateResultOperator operator = entry.getValue().getRight(); + if (entry.getValue().getLeft() == TSDataType.UNKNOWN) { + if (!operator.initAndGetIsSupport(value, timestamp)) { + // Remove unsupported aggregated results + aggregatedOutputName2OperatorMap + .entrySet() + .removeIf( + entry1 -> + entry1 + .getValue() + .getDeclaredIntermediateValueNames() + .contains(operator.getName())); + // If no aggregated values can be calculated, purge the window + if (aggregatedOutputName2OperatorMap.isEmpty()) { + return new Pair<>(WindowState.PURGE, null); + } + // Remove unsupported intermediate values + iterator.remove(); + continue; + } + entry.getValue().setLeft(TSDataType.BLOB); + } else if (entry.getValue().getLeft() != TSDataType.BLOB) { + LOGGER.warn( + "Different data type encountered in one window, will purge. Previous type: {}, now type: {}", + entry.getValue().getLeft(), + TSDataType.BLOB); + return new Pair<>(WindowState.PURGE, null); + } else { + operator.updateValue(value, timestamp); + } + } + } + if (state.isEmitWithCompute()) { + stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.BLOB)); + } + return state.isEmit() ? stateOutputPair : null; + } + public WindowOutput forceOutput() { return processor .forceOutput(this) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java index b74cbb09181fe..c7acd0441c84f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java @@ -22,82 +22,106 @@ import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.utils.Pair; +import java.time.LocalDate; import java.util.List; import java.util.Set; public abstract class AbstractSimpleTimeWindowingProcessor extends AbstractWindowingProcessor { public Set mayAddWindow( - List windowList, long timeStamp, boolean value) { + final List windowList, final long timeStamp, final boolean value) { return mayAddWindow(windowList, timeStamp); } public Set mayAddWindow( - List windowList, long timeStamp, int value) { + final List windowList, final long timeStamp, final int value) { return mayAddWindow(windowList, timeStamp); } public Set mayAddWindow( - List windowList, long timeStamp, long value) { + final List windowList, final long timeStamp, final LocalDate value) { return mayAddWindow(windowList, timeStamp); } public Set mayAddWindow( - List windowList, long timeStamp, float value) { + final List windowList, final long timeStamp, final long value) { return mayAddWindow(windowList, timeStamp); } public Set mayAddWindow( - List windowList, long timeStamp, double value) { + final List windowList, final long timeStamp, final float value) { return mayAddWindow(windowList, timeStamp); } public Set mayAddWindow( - List windowList, long timeStamp, String value) { + final List windowList, final long timeStamp, final double value) { + return mayAddWindow(windowList, timeStamp); + } + + public Set mayAddWindow( + final List windowList, final long timeStamp, final String value) { + return mayAddWindow(windowList, timeStamp); + } + + public Set mayAddWindow( + final List windowList, final long timeStamp, final Binary value) { return mayAddWindow(windowList, timeStamp); } public abstract Set mayAddWindow( - List windowList, long timeStamp); + final List windowList, final long timeStamp); + + @Override + public Pair updateAndMaySetWindowState( + final TimeSeriesWindow window, final long timeStamp, final boolean value) { + return updateAndMaySetWindowState(window, timeStamp); + } + + @Override + public Pair updateAndMaySetWindowState( + final TimeSeriesWindow window, final long timeStamp, final int value) { + return updateAndMaySetWindowState(window, timeStamp); + } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, boolean value) { + final TimeSeriesWindow window, final long timeStamp, final LocalDate value) { return updateAndMaySetWindowState(window, timeStamp); } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, int value) { + final TimeSeriesWindow window, final long timeStamp, final long value) { return updateAndMaySetWindowState(window, timeStamp); } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, long value) { + final TimeSeriesWindow window, final long timeStamp, final float value) { return updateAndMaySetWindowState(window, timeStamp); } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, float value) { + final TimeSeriesWindow window, final long timeStamp, final double value) { return updateAndMaySetWindowState(window, timeStamp); } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, double value) { + final TimeSeriesWindow window, final long timeStamp, final String value) { return updateAndMaySetWindowState(window, timeStamp); } @Override public Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, String value) { + final TimeSeriesWindow window, final long timeStamp, final Binary value) { return updateAndMaySetWindowState(window, timeStamp); } public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp); + final TimeSeriesWindow window, final long timeStamp); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java index 7f8d1ba70a5af..384db345b8592 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java @@ -24,11 +24,13 @@ import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; +import org.apache.iotdb.pipe.api.type.Binary; import org.apache.tsfile.utils.Pair; import java.io.DataOutputStream; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.List; import java.util.Set; @@ -47,22 +49,28 @@ public abstract class AbstractWindowingProcessor extends AbstractFormalProcessor * @return The added windows */ public abstract Set mayAddWindow( - List windowList, long timeStamp, boolean value); + final List windowList, final long timeStamp, final boolean value); public abstract Set mayAddWindow( - List windowList, long timeStamp, int value); + final List windowList, final long timeStamp, final int value); public abstract Set mayAddWindow( - List windowList, long timeStamp, long value); + final List windowList, final long timeStamp, final LocalDate value); public abstract Set mayAddWindow( - List windowList, long timeStamp, float value); + final List windowList, final long timeStamp, final long value); public abstract Set mayAddWindow( - List windowList, long timeStamp, double value); + final List windowList, final long timeStamp, final float value); public abstract Set mayAddWindow( - List windowList, long timeStamp, String value); + final List windowList, final long timeStamp, final double value); + + public abstract Set mayAddWindow( + final List windowList, final long timeStamp, final String value); + + public abstract Set mayAddWindow( + final List windowList, final long timeStamp, final Binary value); /** * The {@link AbstractWindowingProcessor} may decide whether a {@link TimeSeriesWindow} shall be @@ -76,24 +84,30 @@ public abstract Set mayAddWindow( * restarted. */ public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, boolean value); + final TimeSeriesWindow window, final long timeStamp, final boolean value); + + public abstract Pair updateAndMaySetWindowState( + final TimeSeriesWindow window, final long timeStamp, final int value); + + public abstract Pair updateAndMaySetWindowState( + final TimeSeriesWindow window, final long timeStamp, final LocalDate value); public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, int value); + final TimeSeriesWindow window, final long timeStamp, final long value); public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, long value); + final TimeSeriesWindow window, final long timeStamp, final float value); public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, float value); + final TimeSeriesWindow window, final long timeStamp, final double value); public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, double value); + final TimeSeriesWindow window, final long timeStamp, final String value); public abstract Pair updateAndMaySetWindowState( - TimeSeriesWindow window, long timeStamp, String value); + final TimeSeriesWindow window, final long timeStamp, final Binary value); - public abstract WindowOutput forceOutput(TimeSeriesWindow window); + public abstract WindowOutput forceOutput(final TimeSeriesWindow window); /** * Serialize the customized attributes in a window to an output stream @@ -102,7 +116,7 @@ public abstract Pair updateAndMaySetWindowState( * @param outputStream the outputStream */ public void serializeCustomizedAttributes( - TimeSeriesWindow window, DataOutputStream outputStream) { + final TimeSeriesWindow window, final DataOutputStream outputStream) { // Do nothing by default } @@ -111,7 +125,8 @@ public void serializeCustomizedAttributes( * * @param byteBuffer the customized attributes in one window */ - public void deserializeCustomizedAttributes(TimeSeriesWindow window, ByteBuffer byteBuffer) { + public void deserializeCustomizedAttributes( + final TimeSeriesWindow window, final ByteBuffer byteBuffer) { // Do nothing by default } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index a5dd5fbf511f2..b38997ef9c0c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -280,10 +280,12 @@ public Object getFirstValueOfIndex(int index) { Object value; switch (dataTypes[index]) { case INT32: + case DATE: int[] intValues = (int[]) columns[index]; value = intValues[0]; break; case INT64: + case TIMESTAMP: long[] longValues = (long[]) columns[index]; value = longValues[0]; break; @@ -300,6 +302,8 @@ public Object getFirstValueOfIndex(int index) { value = boolValues[0]; break; case TEXT: + case BLOB: + case STRING: Binary[] binaryValues = (Binary[]) columns[index]; value = binaryValues[0]; break; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java index 0d8e5718c9eb2..8e69473b95c6e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java @@ -38,9 +38,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.rpc.RpcUtils; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; @@ -51,6 +53,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -61,9 +64,9 @@ public class PipeDataNodeThriftRequestTest { @Test public void testPipeTransferDataNodeHandshakeReq() throws IOException { - PipeTransferDataNodeHandshakeV1Req req = + final PipeTransferDataNodeHandshakeV1Req req = PipeTransferDataNodeHandshakeV1Req.toTPipeTransferReq(TIME_PRECISION); - PipeTransferDataNodeHandshakeV1Req deserializeReq = + final PipeTransferDataNodeHandshakeV1Req deserializeReq = PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -75,7 +78,7 @@ public void testPipeTransferDataNodeHandshakeReq() throws IOException { @Test public void testPipeTransferInsertNodeReq() { - PipeTransferTabletInsertNodeReq req = + final PipeTransferTabletInsertNodeReq req = PipeTransferTabletInsertNodeReq.toTPipeTransferReq( new InsertRowNode( new PlanNodeId(""), @@ -86,7 +89,7 @@ public void testPipeTransferInsertNodeReq() { 1, new Object[] {1}, false)); - PipeTransferTabletInsertNodeReq deserializeReq = + final PipeTransferTabletInsertNodeReq deserializeReq = PipeTransferTabletInsertNodeReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -95,8 +98,8 @@ public void testPipeTransferInsertNodeReq() { Assert.assertEquals(req.getInsertNode(), deserializeReq.getInsertNode()); - Statement statement = req.constructStatement(); - List paths = new ArrayList<>(); + final Statement statement = req.constructStatement(); + final List paths = new ArrayList<>(); paths.add(new PartialPath(new String[] {"root", "sg", "d", "s"})); Assert.assertEquals(statement.getPaths(), paths); } @@ -104,9 +107,9 @@ public void testPipeTransferInsertNodeReq() { @Test public void testPipeTransferTabletBinaryReq() { // Not do real test here since "serializeToWal" needs private inner class of walBuffer - PipeTransferTabletBinaryReq req = + final PipeTransferTabletBinaryReq req = PipeTransferTabletBinaryReq.toTPipeTransferReq(ByteBuffer.wrap(new byte[] {'a', 'b'})); - PipeTransferTabletBinaryReq deserializeReq = + final PipeTransferTabletBinaryReq deserializeReq = PipeTransferTabletBinaryReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -116,7 +119,7 @@ public void testPipeTransferTabletBinaryReq() { @Test public void testPipeTransferSchemaPlanReq() { - PipeTransferPlanNodeReq req = + final PipeTransferPlanNodeReq req = PipeTransferPlanNodeReq.toTPipeTransferReq( new CreateAlignedTimeSeriesNode( new PlanNodeId(""), @@ -129,7 +132,8 @@ public void testPipeTransferSchemaPlanReq() { null, null)); - PipeTransferPlanNodeReq deserializeReq = PipeTransferPlanNodeReq.fromTPipeTransferReq(req); + final PipeTransferPlanNodeReq deserializeReq = + PipeTransferPlanNodeReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); Assert.assertEquals(req.getType(), deserializeReq.getType()); @@ -141,29 +145,40 @@ public void testPipeTransferSchemaPlanReq() { @Test public void testPipeTransferTabletReq() { try { - List schemaList = new ArrayList<>(); + final List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT32)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.FLOAT)); schemaList.add(new MeasurementSchema("s4", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s5", TSDataType.BOOLEAN)); schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT)); - Tablet t = new Tablet("root.sg.d", schemaList, 1024); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s9", TSDataType.BLOB)); + schemaList.add(new MeasurementSchema("s10", TSDataType.STRING)); + final Tablet t = new Tablet("root.sg.d", schemaList, 1024); t.rowSize = 2; t.addTimestamp(0, 2000); t.addTimestamp(1, 1000); t.addValue("s1", 0, 2); t.addValue("s6", 0, "2"); + t.addValue("s8", 0, LocalDate.of(2024, 2, 2)); + t.addValue("s9", 0, new Binary("2", TSFileConfig.STRING_CHARSET)); + t.addValue("s10", 0, "2"); t.addValue("s1", 1, 1); t.addValue("s6", 1, "1"); - PipeTransferTabletRawReq req = PipeTransferTabletRawReq.toTPipeTransferReq(t, false); - PipeTransferTabletRawReq deserializeReq = PipeTransferTabletRawReq.fromTPipeTransferReq(req); + t.addValue("s8", 1, LocalDate.of(2024, 1, 1)); + t.addValue("s9", 1, new Binary("1", TSFileConfig.STRING_CHARSET)); + t.addValue("s10", 1, "1"); + final PipeTransferTabletRawReq req = PipeTransferTabletRawReq.toTPipeTransferReq(t, false); + final PipeTransferTabletRawReq deserializeReq = + PipeTransferTabletRawReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); Assert.assertEquals(req.getType(), deserializeReq.getType()); Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody()); - Statement statement = + final Statement statement = req.constructStatement(); // will call PipeTransferTabletRawReq.sortTablet() here List paths = new ArrayList<>(); paths.add(new PartialPath(new String[] {"root", "sg", "d", "s1"})); @@ -172,8 +187,12 @@ public void testPipeTransferTabletReq() { paths.add(new PartialPath(new String[] {"root", "sg", "d", "s4"})); paths.add(new PartialPath(new String[] {"root", "sg", "d", "s5"})); paths.add(new PartialPath(new String[] {"root", "sg", "d", "s6"})); + paths.add(new PartialPath(new String[] {"root", "sg", "d", "s7"})); + paths.add(new PartialPath(new String[] {"root", "sg", "d", "s8"})); + paths.add(new PartialPath(new String[] {"root", "sg", "d", "s9"})); + paths.add(new PartialPath(new String[] {"root", "sg", "d", "s10"})); Assert.assertEquals(statement.getPaths(), paths); - } catch (IOException e) { + } catch (final IOException e) { Assert.fail(); } } @@ -184,7 +203,7 @@ public void testPipeTransferTabletBatchReq() throws IOException { final List insertNodeBuffers = new ArrayList<>(); final List tabletBuffers = new ArrayList<>(); - InsertRowNode node = + final InsertRowNode node = new InsertRowNode( new PlanNodeId(""), new PartialPath(new String[] {"root", "sg", "d"}), @@ -203,21 +222,32 @@ public void testPipeTransferTabletBatchReq() throws IOException { binaryBuffers.add(ByteBuffer.wrap(new byte[] {'a', 'b'})); // Raw buffer - List schemaList = new ArrayList<>(); + final List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT32)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s3", TSDataType.FLOAT)); schemaList.add(new MeasurementSchema("s4", TSDataType.DOUBLE)); schemaList.add(new MeasurementSchema("s5", TSDataType.BOOLEAN)); schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT)); - Tablet t = new Tablet("root.sg.d", schemaList, 1024); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s9", TSDataType.BLOB)); + schemaList.add(new MeasurementSchema("s10", TSDataType.STRING)); + + final Tablet t = new Tablet("root.sg.d", schemaList, 1024); t.rowSize = 2; t.addTimestamp(0, 2000); t.addTimestamp(1, 1000); t.addValue("s1", 0, 2); t.addValue("s6", 0, "2"); + t.addValue("s8", 0, LocalDate.of(2024, 2, 2)); + t.addValue("s9", 0, new Binary("2", TSFileConfig.STRING_CHARSET)); + t.addValue("s10", 0, "2"); t.addValue("s1", 1, 1); t.addValue("s6", 1, "1"); + t.addValue("s8", 1, LocalDate.of(2024, 1, 1)); + t.addValue("s9", 1, new Binary("1", TSFileConfig.STRING_CHARSET)); + t.addValue("s10", 1, "1"); try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { @@ -227,11 +257,11 @@ public void testPipeTransferTabletBatchReq() throws IOException { ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size())); } - PipeTransferTabletBatchReq req = + final PipeTransferTabletBatchReq req = PipeTransferTabletBatchReq.toTPipeTransferReq( binaryBuffers, insertNodeBuffers, tabletBuffers); - PipeTransferTabletBatchReq deserializedReq = + final PipeTransferTabletBatchReq deserializedReq = PipeTransferTabletBatchReq.fromTPipeTransferReq(req); Assert.assertArrayEquals( @@ -243,12 +273,12 @@ public void testPipeTransferTabletBatchReq() throws IOException { @Test public void testPipeTransferFilePieceReq() throws IOException { - byte[] body = "testPipeTransferFilePieceReq".getBytes(); - String fileName = "1.tsfile"; + final byte[] body = "testPipeTransferFilePieceReq".getBytes(); + final String fileName = "1.tsfile"; - PipeTransferTsFilePieceReq req = + final PipeTransferTsFilePieceReq req = PipeTransferTsFilePieceReq.toTPipeTransferReq(fileName, 0, body); - PipeTransferTsFilePieceReq deserializeReq = + final PipeTransferTsFilePieceReq deserializeReq = PipeTransferTsFilePieceReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -262,12 +292,12 @@ public void testPipeTransferFilePieceReq() throws IOException { @Test public void testPipeTransferFilePieceWithModReq() throws IOException { - byte[] body = "testPipeTransferFilePieceWithModReq".getBytes(); - String fileName = "1.tsfile.mod"; + final byte[] body = "testPipeTransferFilePieceWithModReq".getBytes(); + final String fileName = "1.tsfile.mod"; - PipeTransferTsFilePieceWithModReq req = + final PipeTransferTsFilePieceWithModReq req = PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, 0, body); - PipeTransferTsFilePieceWithModReq deserializeReq = + final PipeTransferTsFilePieceWithModReq deserializeReq = PipeTransferTsFilePieceWithModReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -281,12 +311,12 @@ public void testPipeTransferFilePieceWithModReq() throws IOException { @Test public void testPipeTransferSchemaSnapshotPieceReq() throws IOException { - byte[] body = "testPipeTransferSchemaSnapshotPieceReq".getBytes(); - String fileName = "1.temp"; + final byte[] body = "testPipeTransferSchemaSnapshotPieceReq".getBytes(); + final String fileName = "1.temp"; - PipeTransferSchemaSnapshotPieceReq req = + final PipeTransferSchemaSnapshotPieceReq req = PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(fileName, 0, body); - PipeTransferSchemaSnapshotPieceReq deserializeReq = + final PipeTransferSchemaSnapshotPieceReq deserializeReq = PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -300,10 +330,12 @@ public void testPipeTransferSchemaSnapshotPieceReq() throws IOException { @Test public void testPipeTransferTsFileSealReq() throws IOException { - String fileName = "1.tsfile"; + final String fileName = "1.tsfile"; - PipeTransferTsFileSealReq req = PipeTransferTsFileSealReq.toTPipeTransferReq(fileName, 100); - PipeTransferTsFileSealReq deserializeReq = PipeTransferTsFileSealReq.fromTPipeTransferReq(req); + final PipeTransferTsFileSealReq req = + PipeTransferTsFileSealReq.toTPipeTransferReq(fileName, 100); + final PipeTransferTsFileSealReq deserializeReq = + PipeTransferTsFileSealReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); Assert.assertEquals(req.getType(), deserializeReq.getType()); @@ -315,16 +347,16 @@ public void testPipeTransferTsFileSealReq() throws IOException { @Test public void testPipeTransferSchemaSnapshotSealReq() throws IOException { - String mTreeSnapshotName = "mtree.snapshot"; - String tLogName = "tlog.txt"; - String databaseName = "root.db"; + final String mTreeSnapshotName = "mtree.snapshot"; + final String tLogName = "tlog.txt"; + final String databaseName = "root.db"; // CREATE_TIME_SERIES - String typeString = "19"; + final String typeString = "19"; - PipeTransferSchemaSnapshotSealReq req = + final PipeTransferSchemaSnapshotSealReq req = PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( "root.**", mTreeSnapshotName, 100, tLogName, 10, databaseName, typeString); - PipeTransferSchemaSnapshotSealReq deserializeReq = + final PipeTransferSchemaSnapshotSealReq deserializeReq = PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); @@ -338,9 +370,9 @@ public void testPipeTransferSchemaSnapshotSealReq() throws IOException { @Test public void testPipeTransferFilePieceResp() throws IOException { - PipeTransferFilePieceResp resp = + final PipeTransferFilePieceResp resp = PipeTransferFilePieceResp.toTPipeTransferResp(RpcUtils.SUCCESS_STATUS, 100); - PipeTransferFilePieceResp deserializeResp = + final PipeTransferFilePieceResp deserializeResp = PipeTransferFilePieceResp.fromTPipeTransferResp(resp); Assert.assertEquals(resp.getStatus(), deserializeResp.getStatus()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java index ee05f26fb82be..5ba92b46a4e7c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java @@ -50,6 +50,10 @@ public void createDataTypeList() { tsDataTypeList.add(TSDataType.DOUBLE); tsDataTypeList.add(TSDataType.BOOLEAN); tsDataTypeList.add(TSDataType.TEXT); + tsDataTypeList.add(TSDataType.TIMESTAMP); + tsDataTypeList.add(TSDataType.DATE); + tsDataTypeList.add(TSDataType.BLOB); + tsDataTypeList.add(TSDataType.STRING); tsDataTypeList.add(null); nullTypeList = null; @@ -60,14 +64,19 @@ public void createDataTypeList() { typeList.add(Type.DOUBLE); typeList.add(Type.BOOLEAN); typeList.add(Type.TEXT); + typeList.add(Type.INT64); + typeList.add(Type.INT32); + typeList.add(Type.TEXT); + typeList.add(Type.TEXT); typeList.add(null); } @Test public void testDataTypeTransformer() { - List nullResultList = + final List nullResultList = PipeDataTypeTransformer.transformToPipeDataTypeList(nullTsDataTypeList); - List resultList = PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypeList); + final List resultList = + PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypeList); Assert.assertEquals(nullResultList, nullTypeList); Assert.assertEquals(resultList, typeList); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index 8c4d9ce691746..44fa83cca6235 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -38,6 +38,7 @@ import org.junit.Before; import org.junit.Test; +import java.time.LocalDate; import java.util.Arrays; public class PipeTabletInsertionEventTest { @@ -49,7 +50,8 @@ public class PipeTabletInsertionEventTest { final String deviceId = "root.sg.d1"; final long[] times = new long[] {110L, 111L, 112L, 113L, 114L}; - final String[] measurementIds = new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}; + final String[] measurementIds = + new String[] {"s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10"}; final TSDataType[] dataTypes = new TSDataType[] { TSDataType.INT32, @@ -57,10 +59,14 @@ public class PipeTabletInsertionEventTest { TSDataType.FLOAT, TSDataType.DOUBLE, TSDataType.BOOLEAN, - TSDataType.TEXT + TSDataType.TEXT, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.BLOB, + TSDataType.STRING, }; - final MeasurementSchema[] schemas = new MeasurementSchema[6]; + final MeasurementSchema[] schemas = new MeasurementSchema[10]; final String pattern = "root.sg.d1"; @@ -90,6 +96,10 @@ private void createInsertRowNode() throws IllegalPathException { values[3] = 1D; values[4] = false; values[5] = BytesUtils.valueOf("text"); + values[6] = 20000L; + values[7] = LocalDate.of(2024, 1, 1); + values[8] = BytesUtils.valueOf("blob"); + values[9] = BytesUtils.valueOf("string"); insertRowNode = new InsertRowNode( @@ -125,6 +135,10 @@ private void createInsertTabletNode() throws IllegalPathException { values[3] = new double[times.length]; values[4] = new boolean[times.length]; values[5] = new Binary[times.length]; + values[6] = new long[times.length]; + values[7] = new LocalDate[times.length]; + values[8] = new Binary[times.length]; + values[9] = new Binary[times.length]; for (int r = 0; r < times.length; r++) { ((int[]) values[0])[r] = 100; @@ -133,6 +147,10 @@ private void createInsertTabletNode() throws IllegalPathException { ((double[]) values[3])[r] = 1D; ((boolean[]) values[4])[r] = false; ((Binary[]) values[5])[r] = BytesUtils.valueOf("text"); + ((long[]) values[6])[r] = 20000L; + ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1); + ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob"); + ((Binary[]) values[9])[r] = BytesUtils.valueOf("string"); } this.insertTabletNode = @@ -177,6 +195,10 @@ private void createTablet() { values[3] = new double[1]; values[4] = new boolean[1]; values[5] = new Binary[1]; + values[6] = new long[times.length]; + values[7] = new LocalDate[times.length]; + values[8] = new Binary[times.length]; + values[9] = new Binary[times.length]; for (int r = 0; r < 1; r++) { ((int[]) values[0])[r] = 100; @@ -185,6 +207,10 @@ private void createTablet() { ((double[]) values[3])[r] = 1D; ((boolean[]) values[4])[r] = false; ((Binary[]) values[5])[r] = BytesUtils.valueOf("text"); + ((long[]) values[6])[r] = 20000L; + ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1); + ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob"); + ((Binary[]) values[9])[r] = BytesUtils.valueOf("string"); } tabletForInsertRowNode = new Tablet(deviceId, Arrays.asList(schemas), 1); @@ -205,6 +231,10 @@ private void createTablet() { values[3] = new double[times.length]; values[4] = new boolean[times.length]; values[5] = new Binary[times.length]; + values[6] = new long[times.length]; + values[7] = new LocalDate[times.length]; + values[8] = new Binary[times.length]; + values[9] = new Binary[times.length]; for (int r = 0; r < times.length; r++) { ((int[]) values[0])[r] = 100; @@ -213,6 +243,10 @@ private void createTablet() { ((double[]) values[3])[r] = 1D; ((boolean[]) values[4])[r] = false; ((Binary[]) values[5])[r] = BytesUtils.valueOf("text"); + ((long[]) values[6])[r] = 20000L; + ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1); + ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob"); + ((Binary[]) values[9])[r] = BytesUtils.valueOf("string"); } tabletForInsertTabletNode = new Tablet(deviceId, Arrays.asList(schemas), times.length); From e65335a8557bcf0688fabbf7b1ca1eb58f5d6e68 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 15:32:16 +0800 Subject: [PATCH 2/6] load module --- .../iotdb/it/utils/TsFileGenerator.java | 103 +++-- ...adTsFileIT.java => IoTDBLoadTsFileIT.java} | 394 +++++++++++------- .../execution/load/AlignedChunkData.java | 124 +++--- .../execution/load/NonAlignedChunkData.java | 61 +-- 4 files changed, 421 insertions(+), 261 deletions(-) rename integration-test/src/test/java/org/apache/iotdb/db/it/{IOTDBLoadTsFileIT.java => IoTDBLoadTsFileIT.java} (65%) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java index 31f6fecdd0738..d30c0c36d5048 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java @@ -37,6 +37,7 @@ import java.io.File; import java.io.IOException; +import java.time.LocalDate; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,7 +53,7 @@ public class TsFileGenerator implements AutoCloseable { private final Map> device2MeasurementSchema; private Random random; - public TsFileGenerator(File tsFile) throws IOException { + public TsFileGenerator(final File tsFile) throws IOException { this.tsFile = tsFile; this.writer = new TsFileWriter(tsFile); this.device2TimeSet = new HashMap<>(); @@ -64,11 +65,12 @@ public void resetRandom() { random = new Random(); } - public void resetRandom(long seed) { + public void resetRandom(final long seed) { random = new Random(seed); } - public void registerTimeseries(String path, List measurementSchemaList) { + public void registerTimeseries( + final String path, final List measurementSchemaList) { if (device2MeasurementSchema.containsKey(path)) { LOGGER.error("Register same device {}.", path); return; @@ -78,7 +80,8 @@ public void registerTimeseries(String path, List measurementS device2MeasurementSchema.put(path, measurementSchemaList); } - public void registerAlignedTimeseries(String path, List measurementSchemaList) + public void registerAlignedTimeseries( + final String path, final List measurementSchemaList) throws WriteProcessException { if (device2MeasurementSchema.containsKey(path)) { LOGGER.error("Register same device {}.", path); @@ -89,14 +92,15 @@ public void registerAlignedTimeseries(String path, List measu device2MeasurementSchema.put(path, measurementSchemaList); } - public void generateData(String device, int number, long timeGap, boolean isAligned) + public void generateData( + final String device, final int number, final long timeGap, final boolean isAligned) throws IOException, WriteProcessException { - List schemas = device2MeasurementSchema.get(device); - TreeSet timeSet = device2TimeSet.get(device); - Tablet tablet = new Tablet(device, schemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - long sensorNum = schemas.size(); + final List schemas = device2MeasurementSchema.get(device); + final TreeSet timeSet = device2TimeSet.get(device); + final Tablet tablet = new Tablet(device, schemas); + final long[] timestamps = tablet.timestamps; + final Object[] values = tablet.values; + final long sensorNum = schemas.size(); long startTime = timeSet.isEmpty() ? 0L : timeSet.last(); for (long r = 0; r < number; r++) { @@ -131,18 +135,22 @@ public void generateData(String device, int number, long timeGap, boolean isAlig } public void generateData( - String device, int number, long timeGap, boolean isAligned, long startTimestamp) + final String device, + final int number, + final long timeGap, + final boolean isAligned, + final long startTimestamp) throws IOException, WriteProcessException { - List schemas = device2MeasurementSchema.get(device); - TreeSet timeSet = device2TimeSet.get(device); - Tablet tablet = new Tablet(device, schemas); - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - long sensorNum = schemas.size(); + final List schemas = device2MeasurementSchema.get(device); + final TreeSet timeSet = device2TimeSet.get(device); + final Tablet tablet = new Tablet(device, schemas); + final long[] timestamps = tablet.timestamps; + final Object[] values = tablet.values; + final long sensorNum = schemas.size(); long startTime = startTimestamp; for (long r = 0; r < number; r++) { - int row = tablet.rowSize++; + final int row = tablet.rowSize++; startTime += timeGap; timestamps[row] = startTime; timeSet.add(startTime); @@ -172,12 +180,16 @@ public void generateData( LOGGER.info("Write {} points into device {}", number, device); } - private void generateDataPoint(Object obj, int row, MeasurementSchema schema) { + private void generateDataPoint(final Object obj, final int row, final MeasurementSchema schema) { switch (schema.getType()) { case INT32: generateINT32(obj, row); break; + case DATE: + generateDATE(obj, row); + break; case INT64: + case TIMESTAMP: generateINT64(obj, row); break; case FLOAT: @@ -190,6 +202,8 @@ private void generateDataPoint(Object obj, int row, MeasurementSchema schema) { generateBOOLEAN(obj, row); break; case TEXT: + case BLOB: + case STRING: generateTEXT(obj, row); break; default: @@ -197,53 +211,60 @@ private void generateDataPoint(Object obj, int row, MeasurementSchema schema) { } } - private void generateINT32(Object obj, int row) { - int[] ints = (int[]) obj; + private void generateINT32(final Object obj, final int row) { + final int[] ints = (int[]) obj; ints[row] = random.nextInt(); } - private void generateINT64(Object obj, int row) { - long[] longs = (long[]) obj; + private void generateDATE(final Object obj, final int row) { + final LocalDate[] dates = (LocalDate[]) obj; + dates[row] = + LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28)); + } + + private void generateINT64(final Object obj, final int row) { + final long[] longs = (long[]) obj; longs[row] = random.nextLong(); } - private void generateFLOAT(Object obj, int row) { - float[] floats = (float[]) obj; + private void generateFLOAT(final Object obj, final int row) { + final float[] floats = (float[]) obj; floats[row] = random.nextFloat(); } - private void generateDOUBLE(Object obj, int row) { - double[] doubles = (double[]) obj; + private void generateDOUBLE(final Object obj, final int row) { + final double[] doubles = (double[]) obj; doubles[row] = random.nextDouble(); } - private void generateBOOLEAN(Object obj, int row) { - boolean[] booleans = (boolean[]) obj; + private void generateBOOLEAN(final Object obj, final int row) { + final boolean[] booleans = (boolean[]) obj; booleans[row] = random.nextBoolean(); } - private void generateTEXT(Object obj, int row) { - Binary[] binaries = (Binary[]) obj; + private void generateTEXT(final Object obj, final int row) { + final Binary[] binaries = (Binary[]) obj; binaries[row] = new Binary(String.format("test point %d", random.nextInt()), TSFileConfig.STRING_CHARSET); } - public void generateDeletion(String device, int number) throws IOException, IllegalPathException { - try (ModificationFile modificationFile = + public void generateDeletion(final String device, final int number) + throws IOException, IllegalPathException { + try (final ModificationFile modificationFile = new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) { writer.flushAllChunkGroups(); - TreeSet timeSet = device2TimeSet.get(device); + final TreeSet timeSet = device2TimeSet.get(device); if (timeSet.isEmpty()) { return; } - long fileOffset = tsFile.length(); - long maxTime = timeSet.last() - 1; + final long fileOffset = tsFile.length(); + final long maxTime = timeSet.last() - 1; for (int i = 0; i < number; i++) { - int endTime = random.nextInt((int) (maxTime)) + 1; - int startTime = random.nextInt(endTime); - for (MeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) { - Deletion deletion = + final int endTime = random.nextInt((int) (maxTime)) + 1; + final int startTime = random.nextInt(endTime); + for (final MeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) { + final Deletion deletion = new Deletion( new PartialPath( device diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java similarity index 65% rename from integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 0a3a44ee629ab..c1b1fdf91091f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -64,8 +64,8 @@ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IOTDBLoadTsFileIT { - private static final Logger LOGGER = LoggerFactory.getLogger(IOTDBLoadTsFileIT.class); +public class IoTDBLoadTsFileIT { + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBLoadTsFileIT.class); private static final long PARTITION_INTERVAL = 10 * 1000L; private static final int connectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(300); private static final long loadTsFileAnalyzeSchemaMemorySizeInBytes = 10 * 1024L; @@ -95,8 +95,8 @@ public void tearDown() throws Exception { } private void registerSchema() throws SQLException { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_0); statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_1); @@ -113,19 +113,24 @@ private void registerSchema() throws SQLException { SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13))); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17))); statement.execute(convert2SQL(SchemaConfig.DEVICE_2, SchemaConfig.MEASUREMENT_20)); statement.execute(convert2SQL(SchemaConfig.DEVICE_3, SchemaConfig.MEASUREMENT_30)); statement.execute( - convert2AlignedSQL(SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40))); + convert2AlignedSQL( + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40))); } } - private String convert2SQL(String device, MeasurementSchema schema) { - String sql = + private String convert2SQL(final String device, final MeasurementSchema schema) { + final String sql = String.format( "create timeseries %s %s", new Path(device, schema.getMeasurementId(), true).getFullPath(), @@ -134,10 +139,10 @@ private String convert2SQL(String device, MeasurementSchema schema) { return sql; } - private String convert2AlignedSQL(String device, List schemas) { + private String convert2AlignedSQL(final String device, final List schemas) { String sql = String.format("create aligned timeseries %s(", device); for (int i = 0; i < schemas.size(); i++) { - MeasurementSchema schema = schemas.get(i); + final MeasurementSchema schema = schemas.get(i); sql += (String.format("%s %s", schema.getMeasurementId(), schema.getType().name())); sql += (i == schemas.size() - 1 ? ")" : ","); } @@ -146,17 +151,17 @@ private String convert2AlignedSQL(String device, List schemas } private void deleteSG() throws SQLException { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_0)); statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_1)); - } catch (IoTDBSQLException ignored) { + } catch (final IoTDBSQLException ignored) { } } private boolean deleteDir() { - for (File file : tmpDir.listFiles()) { + for (final File file : tmpDir.listFiles()) { if (!file.delete()) { return false; } @@ -168,49 +173,59 @@ private boolean deleteDir() { public void testLoad() throws Exception { registerSchema(); - long writtenPoint1 = 0; + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true); writtenPoint1 = generator.getTotalNumber(); } - long writtenPoint2 = 0; + final long writtenPoint2; // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint2 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); @@ -223,9 +238,9 @@ public void testLoad() throws Exception { } } - // try delete after loading. Expect no deadlock - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + // Try to delete after loading. Expect no deadlock + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute( String.format( "delete timeseries %s.%s", @@ -235,29 +250,38 @@ public void testLoad() throws Exception { @Test public void testLoadWithExtendTemplate() throws Exception { - long writtenPoint1 = 0; + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint1 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute("create database root.sg.test_0"); statement.execute( @@ -266,17 +290,17 @@ public void testLoadWithExtendTemplate() throws Exception { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); } else { Assert.fail("This ResultSet is empty."); } } - Set nodes = + final Set nodes = new HashSet<>( Arrays.asList( "lat", @@ -284,14 +308,18 @@ public void testLoadWithExtendTemplate() throws Exception { SchemaConfig.MEASUREMENT_00.getMeasurementId(), SchemaConfig.MEASUREMENT_01.getMeasurementId(), SchemaConfig.MEASUREMENT_02.getMeasurementId(), - SchemaConfig.MEASUREMENT_03.getMeasurementId())); - try (ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) { + SchemaConfig.MEASUREMENT_03.getMeasurementId(), + SchemaConfig.MEASUREMENT_04.getMeasurementId(), + SchemaConfig.MEASUREMENT_05.getMeasurementId(), + SchemaConfig.MEASUREMENT_06.getMeasurementId(), + SchemaConfig.MEASUREMENT_07.getMeasurementId())); + try (final ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) { while (resultSet.next()) { String device = resultSet.getString(ColumnHeaderConstant.CHILD_NODES); Assert.assertTrue(nodes.remove(device)); } Assert.assertTrue(nodes.isEmpty()); - } catch (Exception e) { + } catch (final Exception e) { e.printStackTrace(); Assert.fail("Parse result set error."); } @@ -300,67 +328,77 @@ public void testLoadWithExtendTemplate() throws Exception { @Test public void testLoadWithAutoRegister() throws Exception { - long writtenPoint1 = 0; + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint1 = generator.getTotalNumber(); } - long writtenPoint2 = 0; + final long writtenPoint2; // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint2 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); - long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); + final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); Assert.assertEquals(writtenPoint2, sg2Count); } else { Assert.fail("This ResultSet is empty."); } } - Map isAligned = new HashMap<>(); + final Map isAligned = new HashMap<>(); isAligned.put(SchemaConfig.DEVICE_0, "false"); isAligned.put(SchemaConfig.DEVICE_1, "true"); isAligned.put(SchemaConfig.DEVICE_2, "false"); isAligned.put(SchemaConfig.DEVICE_3, "false"); isAligned.put(SchemaConfig.DEVICE_4, "true"); - try (ResultSet resultSet = statement.executeQuery("show devices")) { + try (final ResultSet resultSet = statement.executeQuery("show devices")) { int size = 0; while (resultSet.next()) { size += 1; @@ -369,7 +407,7 @@ public void testLoadWithAutoRegister() throws Exception { isAligned.get(device), resultSet.getString(ColumnHeaderConstant.IS_ALIGNED)); } Assert.assertEquals(isAligned.size(), size); - } catch (Exception e) { + } catch (final Exception e) { e.printStackTrace(); Assert.fail("Parse result set error."); } @@ -381,33 +419,43 @@ public void testAuth() throws Exception { createUser("test", "test123"); // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "test1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "test1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); } // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "test2-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "test2-0-0-0.tsfile"))) { generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true); @@ -443,33 +491,41 @@ public void testAuth() throws Exception { @Test public void testLoadWithOnSuccess() throws Exception { - File file1 = new File(tmpDir, "1-0-0-0.tsfile"); - File file2 = new File(tmpDir, "2-0-0-0.tsfile"); + final File file1 = new File(tmpDir, "1-0-0-0.tsfile"); + final File file2 = new File(tmpDir, "2-0-0-0.tsfile"); long writtenPoint1 = 0; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(file1)) { + try (final TsFileGenerator generator = new TsFileGenerator(file1)) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint1 = generator.getTotalNumber(); } - long writtenPoint2 = 0; + final long writtenPoint2; // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(file2)) { + try (final TsFileGenerator generator = new TsFileGenerator(file2)) { generator.registerTimeseries( SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( @@ -482,16 +538,16 @@ public void testLoadWithOnSuccess() throws Exception { writtenPoint2 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute( String.format("load \"%s\" sglevel=2 onSuccess=none", file1.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); } else { Assert.fail("This ResultSet is empty."); @@ -500,13 +556,13 @@ public void testLoadWithOnSuccess() throws Exception { Assert.assertTrue(file1.exists()); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute( String.format("load \"%s\" sglevel=2 onSuccess=delete", file2.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); @@ -525,19 +581,19 @@ public void testLoadWithOnSuccess() throws Exception { public void testLoadWithLastCache() throws Exception { registerSchema(); - String device = SchemaConfig.DEVICE_0; - String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId(); + final String device = SchemaConfig.DEVICE_0; + final String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId(); - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute( String.format("insert into %s(timestamp, %s) values(100, 100)", device, measurement)); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery(String.format("select last %s from %s", measurement, device))) { if (resultSet.next()) { - String lastValue = resultSet.getString(ColumnHeaderConstant.VALUE); + final String lastValue = resultSet.getString(ColumnHeaderConstant.VALUE); Assert.assertEquals("100", lastValue); } else { Assert.fail("This ResultSet is empty."); @@ -545,50 +601,58 @@ public void testLoadWithLastCache() throws Exception { } } - File file1 = new File(tmpDir, "1-0-0-0.tsfile"); - File file2 = new File(tmpDir, "2-0-0-0.tsfile"); + final File file1 = new File(tmpDir, "1-0-0-0.tsfile"); + final File file2 = new File(tmpDir, "2-0-0-0.tsfile"); // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(file1)) { + try (final TsFileGenerator generator = new TsFileGenerator(file1)) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); } // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(file2)) { + try (final TsFileGenerator generator = new TsFileGenerator(file2)) { generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery(String.format("select last %s from %s", measurement, device))) { if (resultSet.next()) { - String lastTime = resultSet.getString(ColumnHeaderConstant.TIME); + final String lastTime = resultSet.getString(ColumnHeaderConstant.TIME); Assert.assertEquals(String.valueOf(PARTITION_INTERVAL), lastTime); } else { Assert.fail("This ResultSet is empty."); @@ -599,56 +663,64 @@ public void testLoadWithLastCache() throws Exception { @Test public void testLoadWithOnNonStandardTsFileName() throws Exception { - File file1 = new File(tmpDir, "1-0-0-0.tsfile"); - File file2 = new File(tmpDir, "a-1.tsfile"); - long writtenPoint1 = 0; + final File file1 = new File(tmpDir, "1-0-0-0.tsfile"); + final File file2 = new File(tmpDir, "a-1.tsfile"); + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(file1)) { + try (final TsFileGenerator generator = new TsFileGenerator(file1)) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint1 = generator.getTotalNumber(); } - long writtenPoint2 = 0; + final long writtenPoint2; // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(file2)) { + try (final TsFileGenerator generator = new TsFileGenerator(file2)) { generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true); writtenPoint2 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); - long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); + final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); Assert.assertEquals(writtenPoint2, sg2Count); } else { Assert.fail("This ResultSet is empty."); @@ -659,23 +731,32 @@ public void testLoadWithOnNonStandardTsFileName() throws Exception { @Test public void testLoadWithMods() throws Exception { - long writtenPoint1 = 0; + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true); generator.generateDeletion(SchemaConfig.DEVICE_0, 10); @@ -683,16 +764,17 @@ public void testLoadWithMods() throws Exception { writtenPoint1 = generator.getTotalNumber(); } - long writtenPoint2 = 0; + final long writtenPoint2; // device 2, device 3, device4, sg 1 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) { generator.resetRandom(1000); generator.registerTimeseries( - SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20)); + SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20)); generator.registerTimeseries( - SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30)); + SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30)); generator.registerAlignedTimeseries( - SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)); + SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)); generator.generateData(SchemaConfig.DEVICE_2, 100, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_3, 100, PARTITION_INTERVAL / 10_000, false); generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true); @@ -705,17 +787,17 @@ public void testLoadWithMods() throws Exception { writtenPoint2 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); - long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); + final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)"); Assert.assertEquals(writtenPoint2, sg2Count); } else { Assert.fail("This ResultSet is empty."); @@ -726,14 +808,14 @@ public void testLoadWithMods() throws Exception { @Test public void testLoadWithEmptyTsFile() throws Exception { - try (TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {} + try (final TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {} - try (Connection connection = EnvFactory.getEnv().getConnection(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = statement.executeQuery("show timeseries")) { + try (final ResultSet resultSet = statement.executeQuery("show timeseries")) { Assert.assertFalse(resultSet.next()); } } @@ -741,21 +823,30 @@ public void testLoadWithEmptyTsFile() throws Exception { @Test public void testLoadTsFileWithWrongTimestampPrecision() throws Exception { - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Arrays.asList( SchemaConfig.MEASUREMENT_00, SchemaConfig.MEASUREMENT_01, SchemaConfig.MEASUREMENT_02, - SchemaConfig.MEASUREMENT_03)); + SchemaConfig.MEASUREMENT_03, + SchemaConfig.MEASUREMENT_04, + SchemaConfig.MEASUREMENT_05, + SchemaConfig.MEASUREMENT_06, + SchemaConfig.MEASUREMENT_07)); generator.registerAlignedTimeseries( SchemaConfig.DEVICE_1, Arrays.asList( SchemaConfig.MEASUREMENT_10, SchemaConfig.MEASUREMENT_11, SchemaConfig.MEASUREMENT_12, - SchemaConfig.MEASUREMENT_13)); + SchemaConfig.MEASUREMENT_13, + SchemaConfig.MEASUREMENT_14, + SchemaConfig.MEASUREMENT_15, + SchemaConfig.MEASUREMENT_16, + SchemaConfig.MEASUREMENT_17)); // generate ns timestamp generator.generateData( SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false, 1694689856546000000L); @@ -763,11 +854,11 @@ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception { SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true, 1694689856546000000L); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath())); - } catch (IoTDBSQLException e) { + } catch (final IoTDBSQLException e) { Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); } } @@ -776,24 +867,25 @@ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception { public void testLoadLocally() throws Exception { registerSchema(); - long writtenPoint1 = 0; + final long writtenPoint1; // device 0, device 1, sg 0 - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { generator.registerTimeseries( SchemaConfig.DEVICE_0, Collections.singletonList(SchemaConfig.MEASUREMENT_00)); generator.generateData(SchemaConfig.DEVICE_0, 1, PARTITION_INTERVAL / 10_000, false); writtenPoint1 = generator.getTotalNumber(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (ResultSet resultSet = + try (final ResultSet resultSet = statement.executeQuery("select count(*) from root.** group by level=1,2")) { if (resultSet.next()) { - long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); + final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)"); Assert.assertEquals(writtenPoint1, sg1Count); } else { Assert.fail("This ResultSet is empty."); @@ -816,6 +908,14 @@ private static class SchemaConfig { new MeasurementSchema("sensor_02", TSDataType.DOUBLE, TSEncoding.GORILLA); private static final MeasurementSchema MEASUREMENT_03 = new MeasurementSchema("sensor_03", TSDataType.TEXT, TSEncoding.PLAIN); + private static final MeasurementSchema MEASUREMENT_04 = + new MeasurementSchema("sensor_04", TSDataType.TIMESTAMP, TSEncoding.RLE); + private static final MeasurementSchema MEASUREMENT_05 = + new MeasurementSchema("sensor_05", TSDataType.DATE, TSEncoding.RLE); + private static final MeasurementSchema MEASUREMENT_06 = + new MeasurementSchema("sensor_06", TSDataType.BLOB, TSEncoding.PLAIN); + private static final MeasurementSchema MEASUREMENT_07 = + new MeasurementSchema("sensor_07", TSDataType.STRING, TSEncoding.PLAIN); // device 1, aligned, sg 0 private static final String DEVICE_1 = "root.sg.test_0.a_1"; @@ -827,6 +927,14 @@ private static class SchemaConfig { new MeasurementSchema("sensor_12", TSDataType.DOUBLE, TSEncoding.GORILLA); private static final MeasurementSchema MEASUREMENT_13 = new MeasurementSchema("sensor_13", TSDataType.TEXT, TSEncoding.PLAIN); + private static final MeasurementSchema MEASUREMENT_14 = + new MeasurementSchema("sensor_14", TSDataType.TIMESTAMP, TSEncoding.RLE); + private static final MeasurementSchema MEASUREMENT_15 = + new MeasurementSchema("sensor_15", TSDataType.DATE, TSEncoding.RLE); + private static final MeasurementSchema MEASUREMENT_16 = + new MeasurementSchema("sensor_16", TSDataType.BLOB, TSEncoding.PLAIN); + private static final MeasurementSchema MEASUREMENT_17 = + new MeasurementSchema("sensor_17", TSDataType.STRING, TSEncoding.PLAIN); // device 2, non aligned, sg 1 private static final String DEVICE_2 = "root.sg.test_1.d_2"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java index 46bd37e630b9e..00d8a605a7548 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java @@ -75,7 +75,9 @@ public class AlignedChunkData implements ChunkData { private List chunkList; public AlignedChunkData( - String device, ChunkHeader chunkHeader, TTimePartitionSlot timePartitionSlot) { + final String device, + final ChunkHeader chunkHeader, + final TTimePartitionSlot timePartitionSlot) { this.dataSize = 0; this.device = device; this.chunkHeaderList = new ArrayList<>(); @@ -94,7 +96,7 @@ public AlignedChunkData( private void addAttrDataSize() { // Should be init before serialize, corresponding serializeAttr dataSize += 2 * Byte.BYTES; // isModification and isAligned dataSize += Long.BYTES; // timePartitionSlot - int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length; + final int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length; dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength); dataSize += deviceLength; // device dataSize += Integer.BYTES; // chunkHeaderListSize @@ -127,9 +129,9 @@ public boolean isAligned() { } @Override - public void writeToFileWriter(TsFileIOWriter writer) throws IOException { + public void writeToFileWriter(final TsFileIOWriter writer) throws IOException { if (chunkList != null) { - for (Chunk chunk : chunkList) { + for (final Chunk chunk : chunkList) { writer.writeChunk(chunk); } } else { @@ -137,7 +139,7 @@ public void writeToFileWriter(TsFileIOWriter writer) throws IOException { } } - public void addValueChunk(ChunkHeader chunkHeader) { + public void addValueChunk(final ChunkHeader chunkHeader) { this.chunkHeaderList.add(chunkHeader); this.pageNumbers.add(0); dataSize += chunkHeader.getSerializedSize(); @@ -147,7 +149,7 @@ public void addValueChunk(ChunkHeader chunkHeader) { } @Override - public void serialize(DataOutputStream stream) throws IOException { + public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(isModification(), stream); ReadWriteIOUtils.write(isAligned(), stream); serializeAttr(stream); @@ -155,31 +157,32 @@ public void serialize(DataOutputStream stream) throws IOException { close(); } - private void serializeAttr(DataOutputStream stream) throws IOException { + private void serializeAttr(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); ReadWriteIOUtils.write(device, stream); ReadWriteIOUtils.write(dataSize, stream); ReadWriteIOUtils.write(needDecodeChunk, stream); ReadWriteIOUtils.write(chunkHeaderList.size(), stream); - for (ChunkHeader chunkHeader : chunkHeaderList) { + for (final ChunkHeader chunkHeader : chunkHeaderList) { chunkHeader.serializeTo(stream); // Chunk header already serialize chunk type } if (needDecodeChunk) { - for (Integer pageNumber : pageNumbers) { + for (final Integer pageNumber : pageNumbers) { ReadWriteIOUtils.write(pageNumber, stream); } } } @Override - public void writeEntireChunk(ByteBuffer chunkData, IChunkMetadata chunkMetadata) + public void writeEntireChunk(final ByteBuffer chunkData, final IChunkMetadata chunkMetadata) throws IOException { dataSize += ReadWriteIOUtils.write(chunkData, stream); dataSize += chunkMetadata.getStatistics().serialize(stream); } @Override - public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws IOException { + public void writeEntirePage(final PageHeader pageHeader, final ByteBuffer pageData) + throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); // serialize needDecode==false dataSize += ReadWriteIOUtils.write(false, stream); @@ -190,17 +193,17 @@ public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws I } @Override - public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) + public void writeDecodePage(final long[] times, final Object[] values, final int satisfiedLength) throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); satisfiedLengthQueue.offer(satisfiedLength); - long startTime = timePartitionSlot.getStartTime(); - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + final long startTime = timePartitionSlot.getStartTime(); + final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); - for (long time : times) { + for (final long time : times) { if (time >= endTime) { break; } @@ -210,12 +213,13 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) } } - public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataType dataType) + public void writeDecodeValuePage( + final long[] times, final TsPrimitiveType[] values, final TSDataType dataType) throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); - long startTime = timePartitionSlot.getStartTime(); - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); - int satisfiedLength = satisfiedLengthQueue.poll(); + final long startTime = timePartitionSlot.getStartTime(); + final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); @@ -232,9 +236,11 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT dataSize += ReadWriteIOUtils.write(false, stream); switch (dataType) { case INT32: + case DATE: dataSize += ReadWriteIOUtils.write(values[i].getInt(), stream); break; case INT64: + case TIMESTAMP: dataSize += ReadWriteIOUtils.write(values[i].getLong(), stream); break; case FLOAT: @@ -247,6 +253,8 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT dataSize += ReadWriteIOUtils.write(values[i].getBoolean(), stream); break; case TEXT: + case BLOB: + case STRING: dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream); break; default: @@ -258,7 +266,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT } } - private void deserializeTsFileData(InputStream stream) throws IOException, PageException { + private void deserializeTsFileData(final InputStream stream) throws IOException, PageException { if (needDecodeChunk) { buildChunkWriter(stream); } else { @@ -266,21 +274,21 @@ private void deserializeTsFileData(InputStream stream) throws IOException, PageE } } - private void deserializeEntireChunk(InputStream stream) throws IOException { + private void deserializeEntireChunk(final InputStream stream) throws IOException { chunkList = new ArrayList<>(); - for (ChunkHeader chunkHeader : chunkHeaderList) { - ByteBuffer chunkData = + for (final ChunkHeader chunkHeader : chunkHeaderList) { + final ByteBuffer chunkData = ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)); - Statistics statistics = + final Statistics statistics = Statistics.deserialize(stream, chunkHeader.getDataType()); chunkList.add(new Chunk(chunkHeader, chunkData, null, statistics)); } } - private void buildChunkWriter(InputStream stream) throws IOException, PageException { - List measurementSchemaList = new ArrayList<>(); + private void buildChunkWriter(final InputStream stream) throws IOException, PageException { + final List measurementSchemaList = new ArrayList<>(); IMeasurementSchema timeSchema = null; - for (ChunkHeader chunkHeader : chunkHeaderList) { + for (final ChunkHeader chunkHeader : chunkHeaderList) { if (TSDataType.VECTOR.equals(chunkHeader.getDataType())) { timeSchema = new MeasurementSchema( @@ -299,7 +307,7 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept } chunkWriter = new AlignedChunkWriterImpl(timeSchema, measurementSchemaList); timeBatch = new ArrayList<>(); - int chunkHeaderSize = chunkHeaderList.size(); + final int chunkHeaderSize = chunkHeaderList.size(); for (int i = 0; i < chunkHeaderSize; i++) { buildChunk(stream, chunkHeaderList.get(i), pageNumbers.get(i), i - 1, i == 0); } @@ -308,52 +316,60 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept @SuppressWarnings({"squid:S6541", "squid:S3776"}) private void buildChunk( - InputStream stream, - ChunkHeader chunkHeader, - int pageNumber, - int valueChunkIndex, - boolean isTimeChunk) + final InputStream stream, + final ChunkHeader chunkHeader, + final int pageNumber, + final int valueChunkIndex, + final boolean isTimeChunk) throws IOException, PageException { boolean needDecode; int decodePageIndex = 0; for (int j = 0; j < pageNumber; j++) { needDecode = ReadWriteIOUtils.readBool(stream); if (needDecode) { - int length = ReadWriteIOUtils.readInt(stream); + final int length = ReadWriteIOUtils.readInt(stream); long[] timePageBatch = new long[length]; if (!isTimeChunk) { timePageBatch = timeBatch.get(decodePageIndex); } for (int i = 0; i < length; i++) { if (isTimeChunk) { - long time = ReadWriteIOUtils.readLong(stream); + final long time = ReadWriteIOUtils.readLong(stream); timePageBatch[i] = time; chunkWriter.writeTime(time); } else { - boolean isNull = ReadWriteIOUtils.readBool(stream); + final boolean isNull = ReadWriteIOUtils.readBool(stream); switch (chunkHeader.getDataType()) { case INT32: - int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream); + case DATE: + final int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream); chunkWriter.write(timePageBatch[i], int32Value, isNull, valueChunkIndex); break; case INT64: - long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream); + case TIMESTAMP: + final long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream); chunkWriter.write(timePageBatch[i], int64Value, isNull, valueChunkIndex); break; case FLOAT: - float floatValue = isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream); + final float floatValue = + isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream); chunkWriter.write(timePageBatch[i], floatValue, isNull, valueChunkIndex); break; case DOUBLE: - double doubleValue = isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream); + final double doubleValue = + isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream); chunkWriter.write(timePageBatch[i], doubleValue, isNull, valueChunkIndex); break; case BOOLEAN: - boolean boolValue = isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream); + final boolean boolValue = + isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream); chunkWriter.write(timePageBatch[i], boolValue, isNull, valueChunkIndex); break; case TEXT: - Binary binaryValue = isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); + case BLOB: + case STRING: + final Binary binaryValue = + isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); chunkWriter.write(timePageBatch[i], binaryValue, isNull, valueChunkIndex); break; default: @@ -373,7 +389,8 @@ private void buildChunk( chunkWriter.sealCurrentValuePage(valueChunkIndex); } } else { - PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); + final PageHeader pageHeader = + PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); if (isTimeChunk) { chunkWriter.writePageHeaderAndDataIntoTimeBuff( ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), @@ -388,26 +405,27 @@ private void buildChunk( } } - public static AlignedChunkData deserialize(InputStream stream) throws IOException, PageException { - TTimePartitionSlot timePartitionSlot = + public static AlignedChunkData deserialize(final InputStream stream) + throws IOException, PageException { + final TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); - String device = ReadWriteIOUtils.readString(stream); - long dataSize = ReadWriteIOUtils.readLong(stream); - boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); - int chunkHeaderListSize = ReadWriteIOUtils.readInt(stream); - List chunkHeaderList = new ArrayList<>(); + final String device = ReadWriteIOUtils.readString(stream); + final long dataSize = ReadWriteIOUtils.readLong(stream); + final boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); + final int chunkHeaderListSize = ReadWriteIOUtils.readInt(stream); + final List chunkHeaderList = new ArrayList<>(); for (int i = 0; i < chunkHeaderListSize; i++) { - byte chunkType = ReadWriteIOUtils.readByte(stream); + final byte chunkType = ReadWriteIOUtils.readByte(stream); chunkHeaderList.add(ChunkHeader.deserializeFrom(stream, chunkType)); } - List pageNumbers = new ArrayList<>(); + final List pageNumbers = new ArrayList<>(); if (needDecodeChunk) { for (int i = 0; i < chunkHeaderListSize; i++) { pageNumbers.add(ReadWriteIOUtils.readInt(stream)); } } - AlignedChunkData chunkData = + final AlignedChunkData chunkData = new AlignedChunkData(device, chunkHeaderList.get(0), timePartitionSlot); chunkData.needDecodeChunk = needDecodeChunk; chunkData.chunkHeaderList = chunkHeaderList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java index 77467d55e98b1..371e97726c5e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java @@ -60,7 +60,9 @@ public class NonAlignedChunkData implements ChunkData { private Chunk chunk; public NonAlignedChunkData( - String device, ChunkHeader chunkHeader, TTimePartitionSlot timePartitionSlot) { + final String device, + final ChunkHeader chunkHeader, + final TTimePartitionSlot timePartitionSlot) { this.dataSize = 0; this.device = device; this.chunkHeader = chunkHeader; @@ -76,7 +78,7 @@ public NonAlignedChunkData( private void addAttrDataSize() { // should be init before serialize, corresponding serializeAttr dataSize += 2 * Byte.BYTES; // isModification and isAligned dataSize += Long.BYTES; // timePartitionSlot - int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length; + final int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length; dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength); dataSize += deviceLength; // device dataSize += chunkHeader.getSerializedSize(); // timeChunkHeader @@ -108,7 +110,7 @@ public boolean isAligned() { } @Override - public void writeToFileWriter(TsFileIOWriter writer) throws IOException { + public void writeToFileWriter(final TsFileIOWriter writer) throws IOException { if (chunk != null) { writer.writeChunk(chunk); } else { @@ -117,7 +119,7 @@ public void writeToFileWriter(TsFileIOWriter writer) throws IOException { } @Override - public void serialize(DataOutputStream stream) throws IOException { + public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(isModification(), stream); ReadWriteIOUtils.write(isAligned(), stream); serializeAttr(stream); @@ -125,7 +127,7 @@ public void serialize(DataOutputStream stream) throws IOException { close(); } - private void serializeAttr(DataOutputStream stream) throws IOException { + private void serializeAttr(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream); ReadWriteIOUtils.write(device, stream); ReadWriteIOUtils.write(dataSize, stream); @@ -137,14 +139,15 @@ private void serializeAttr(DataOutputStream stream) throws IOException { } @Override - public void writeEntireChunk(ByteBuffer chunkData, IChunkMetadata chunkMetadata) + public void writeEntireChunk(final ByteBuffer chunkData, final IChunkMetadata chunkMetadata) throws IOException { dataSize += ReadWriteIOUtils.write(chunkData, stream); dataSize += chunkMetadata.getStatistics().serialize(stream); } @Override - public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws IOException { + public void writeEntirePage(final PageHeader pageHeader, final ByteBuffer pageData) + throws IOException { pageNumber += 1; dataSize += ReadWriteIOUtils.write(false, stream); dataSize += pageHeader.serializeTo(stream); @@ -152,11 +155,11 @@ public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws I } @Override - public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) + public void writeDecodePage(final long[] times, final Object[] values, final int satisfiedLength) throws IOException { pageNumber += 1; - long startTime = timePartitionSlot.getStartTime(); - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + final long startTime = timePartitionSlot.getStartTime(); + final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); @@ -168,9 +171,11 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) dataSize += ReadWriteIOUtils.write(times[i], stream); switch (chunkHeader.getDataType()) { case INT32: + case DATE: dataSize += ReadWriteIOUtils.write((int) values[i], stream); break; case INT64: + case TIMESTAMP: dataSize += ReadWriteIOUtils.write((long) values[i], stream); break; case FLOAT: @@ -183,6 +188,8 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) dataSize += ReadWriteIOUtils.write((boolean) values[i], stream); break; case TEXT: + case BLOB: + case STRING: dataSize += ReadWriteIOUtils.write((Binary) values[i], stream); break; default: @@ -193,7 +200,7 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength) } } - private void deserializeTsFileData(InputStream stream) throws IOException, PageException { + private void deserializeTsFileData(final InputStream stream) throws IOException, PageException { if (needDecodeChunk) { buildChunkWriter(stream); } else { @@ -201,15 +208,15 @@ private void deserializeTsFileData(InputStream stream) throws IOException, PageE } } - private void deserializeEntireChunk(InputStream stream) throws IOException { - ByteBuffer chunkData = + private void deserializeEntireChunk(final InputStream stream) throws IOException { + final ByteBuffer chunkData = ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)); - Statistics statistics = + final Statistics statistics = Statistics.deserialize(stream, chunkHeader.getDataType()); chunk = new Chunk(chunkHeader, chunkData, null, statistics); } - private void buildChunkWriter(InputStream stream) throws IOException, PageException { + private void buildChunkWriter(final InputStream stream) throws IOException, PageException { chunkWriter = new ChunkWriterImpl( new MeasurementSchema( @@ -226,9 +233,11 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept long time = ReadWriteIOUtils.readLong(stream); switch (chunkHeader.getDataType()) { case INT32: + case DATE: chunkWriter.write(time, ReadWriteIOUtils.readInt(stream)); break; case INT64: + case TIMESTAMP: chunkWriter.write(time, ReadWriteIOUtils.readLong(stream)); break; case FLOAT: @@ -241,6 +250,8 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept chunkWriter.write(time, ReadWriteIOUtils.readBool(stream)); break; case TEXT: + case BLOB: + case STRING: chunkWriter.write(time, ReadWriteIOUtils.readBinary(stream)); break; default: @@ -251,7 +262,8 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept chunkWriter.sealCurrentPage(); } else { - PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); + final PageHeader pageHeader = + PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true); chunkWriter.writePageHeaderAndDataIntoBuff( ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)), pageHeader); @@ -259,21 +271,22 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept } } - public static NonAlignedChunkData deserialize(InputStream stream) + public static NonAlignedChunkData deserialize(final InputStream stream) throws IOException, PageException { - TTimePartitionSlot timePartitionSlot = + final TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream)); - String device = ReadWriteIOUtils.readString(stream); - long dataSize = ReadWriteIOUtils.readLong(stream); - boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); - byte chunkType = ReadWriteIOUtils.readByte(stream); - ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); + final String device = ReadWriteIOUtils.readString(stream); + final long dataSize = ReadWriteIOUtils.readLong(stream); + final boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream); + final byte chunkType = ReadWriteIOUtils.readByte(stream); + final ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType); int pageNumber = 0; if (needDecodeChunk) { pageNumber = ReadWriteIOUtils.readInt(stream); } - NonAlignedChunkData chunkData = new NonAlignedChunkData(device, chunkHeader, timePartitionSlot); + final NonAlignedChunkData chunkData = + new NonAlignedChunkData(device, chunkHeader, timePartitionSlot); chunkData.needDecodeChunk = needDecodeChunk; chunkData.pageNumber = pageNumber; chunkData.deserializeTsFileData(stream); From a659eaf03859d362edf1fe5356471f95cc6fdc5e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 15:52:08 +0800 Subject: [PATCH 3/6] fix --- .../changing/ChangingValueFilter.java | 22 ++++++++++++------- .../sdt/SwingingDoorTrendingFilter.java | 20 +++++++++++------ 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java index cc83fbeb21ec8..7dc8c87c09b2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java @@ -21,6 +21,7 @@ import org.apache.iotdb.pipe.api.type.Binary; +import java.time.LocalDate; import java.util.Objects; public class ChangingValueFilter { @@ -36,26 +37,28 @@ public class ChangingValueFilter { private T lastStoredValue; public ChangingValueFilter( - ChangingValueSamplingProcessor processor, long firstTimestamp, T firstValue) { + final ChangingValueSamplingProcessor processor, + final long firstTimestamp, + final T firstValue) { this.processor = processor; init(firstTimestamp, firstValue); } - private void init(long firstTimestamp, T firstValue) { + private void init(final long firstTimestamp, final T firstValue) { lastStoredTimestamp = firstTimestamp; lastStoredValue = firstValue; } - public boolean filter(long timestamp, T value) { + public boolean filter(final long timestamp, final T value) { try { return tryFilter(timestamp, value); - } catch (Exception e) { + } catch (final Exception e) { init(timestamp, value); return true; } } - private boolean tryFilter(long timestamp, T value) { + private boolean tryFilter(final long timestamp, final T value) { final long timeDiff = Math.abs(timestamp - lastStoredTimestamp); if (timeDiff <= processor.getCompressionMinTimeInterval()) { @@ -67,8 +70,11 @@ private boolean tryFilter(long timestamp, T value) { return true; } - // For boolean and string type, we only compare the value - if (value instanceof Boolean || value instanceof String || value instanceof Binary) { + // For non-numerical types, we only compare the value + if (value instanceof Boolean + || value instanceof String + || value instanceof Binary + || value instanceof LocalDate) { if (Objects.equals(lastStoredValue, value)) { return false; } @@ -88,7 +94,7 @@ private boolean tryFilter(long timestamp, T value) { return false; } - private void reset(long timestamp, T value) { + private void reset(final long timestamp, final T value) { lastStoredTimestamp = timestamp; lastStoredValue = value; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java index 8d9e73deb0701..850cbd3ed0729 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java @@ -21,6 +21,7 @@ import org.apache.iotdb.pipe.api.type.Binary; +import java.time.LocalDate; import java.util.Objects; public class SwingingDoorTrendingFilter { @@ -56,12 +57,14 @@ public class SwingingDoorTrendingFilter { private T lastStoredValue; public SwingingDoorTrendingFilter( - SwingingDoorTrendingSamplingProcessor processor, long firstTimestamp, T firstValue) { + final SwingingDoorTrendingSamplingProcessor processor, + final long firstTimestamp, + final T firstValue) { this.processor = processor; init(firstTimestamp, firstValue); } - private void init(long firstTimestamp, T firstValue) { + private void init(final long firstTimestamp, final T firstValue) { upperDoor = Double.MIN_VALUE; lowerDoor = Double.MAX_VALUE; @@ -72,16 +75,16 @@ private void init(long firstTimestamp, T firstValue) { lastStoredValue = firstValue; } - public boolean filter(long timestamp, T value) { + public boolean filter(final long timestamp, final T value) { try { return tryFilter(timestamp, value); - } catch (Exception e) { + } catch (final Exception e) { init(timestamp, value); return true; } } - private boolean tryFilter(long timestamp, T value) { + private boolean tryFilter(final long timestamp, final T value) { final long timeDiff = timestamp - lastStoredTimestamp; final long absTimeDiff = Math.abs(timeDiff); @@ -95,7 +98,10 @@ private boolean tryFilter(long timestamp, T value) { } // For boolean and string type, we only compare the value - if (value instanceof Boolean || value instanceof String || value instanceof Binary) { + if (value instanceof Boolean + || value instanceof String + || value instanceof Binary + || value instanceof LocalDate) { if (Objects.equals(lastStoredValue, value)) { return false; } @@ -138,7 +144,7 @@ private boolean tryFilter(long timestamp, T value) { return false; } - private void reset(long timestamp, T value) { + private void reset(final long timestamp, final T value) { upperDoor = Double.MIN_VALUE; lowerDoor = Double.MAX_VALUE; From 74c9364dbc3f4a992f300fc7b902dd148751b554 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:12:11 +0800 Subject: [PATCH 4/6] fix ut --- .../queryengine/execution/load/NonAlignedChunkData.java | 4 ++-- .../iotdb/db/pipe/event/PipeDataTypeTransformerTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java index 371e97726c5e5..2ae0acd2180bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java @@ -228,9 +228,9 @@ private void buildChunkWriter(final InputStream stream) throws IOException, Page for (int j = 0; j < pageNumber; j++) { needDecode = ReadWriteIOUtils.readBool(stream); if (needDecode) { - int length = ReadWriteIOUtils.readInt(stream); + final int length = ReadWriteIOUtils.readInt(stream); for (int i = 0; i < length; i++) { - long time = ReadWriteIOUtils.readLong(stream); + final long time = ReadWriteIOUtils.readLong(stream); switch (chunkHeader.getDataType()) { case INT32: case DATE: diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java index 5ba92b46a4e7c..a2249a35e50e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java @@ -64,10 +64,10 @@ public void createDataTypeList() { typeList.add(Type.DOUBLE); typeList.add(Type.BOOLEAN); typeList.add(Type.TEXT); - typeList.add(Type.INT64); - typeList.add(Type.INT32); - typeList.add(Type.TEXT); - typeList.add(Type.TEXT); + typeList.add(Type.TIMESTAMP); + typeList.add(Type.DATE); + typeList.add(Type.BLOB); + typeList.add(Type.STRING); typeList.add(null); } From 612b4a8ba4a675ef68f82c50d166e1d4b0fa1d09 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 18:38:39 +0800 Subject: [PATCH 5/6] Added ITs --- .../it/autocreate/IoTDBPipeDataSinkIT.java | 105 ++++++++++++++++++ .../pipe/it/single/IoTDBPipeAggregateIT.java | 14 +++ 2 files changed, 119 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java index 50a9b0d2ec5b2..8cd8d66d38791 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java @@ -148,4 +148,109 @@ public void testLegacyConnector() throws Exception { Collections.singleton("0,1.0,")); } } + + @Test + public void testReceiverAutoCreateByLog() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "forced-log"); + } + }); + } + + @Test + public void testReceiverAutoCreateByFile() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "batch"); + } + }); + } + + @Test + public void testReceiverAutoCreateWithPattern() throws Exception { + testReceiverAutoCreate( + new HashMap() { + { + put("source.realtime.mode", "batch"); + put("source.path", "root.ln.wf01.wt0*.*"); + } + }); + } + + private void testReceiverAutoCreate(final Map extractorAttributes) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + connectorAttributes.put("sink", "iotdb-thrift-sink"); + connectorAttributes.put("sink.batch.enable", "false"); + connectorAttributes.put("sink.ip", receiverIp); + connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + // Do not fail if the failure has nothing to do with pipe + // Because the failures will randomly generate due to resource limitation + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create timeSeries root.ln.wf01.wt01.boolean boolean", + "create timeSeries root.ln.wf01.wt01.int32 int32", + "create timeSeries root.ln.wf01.wt01.int64 int64", + "create timeSeries root.ln.wf01.wt01.float float", + "create timeSeries root.ln.wf01.wt01.double double", + "create timeSeries root.ln.wf01.wt01.time_stamp timestamp", + "create timeSeries root.ln.wf01.wt01.date date", + "create timeSeries root.ln.wf01.wt01.text text", + "create timeSeries root.ln.wf01.wt01.string string", + "create timeSeries root.ln.wf01.wt01.blob blob", + "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)", + "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')", + "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)", + // For pattern parse + "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show timeSeries root.ln.wf01.wt01.*", + "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,", + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,", + "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,")))); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "show devices root.ln.wf01.wt02", + "Device,IsAligned,Template,TTL,", + Collections.singleton("root.ln.wf01.wt02,true,null,INF,")); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java index 544509b82894a..cbd9c08f4e5c2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java @@ -78,6 +78,20 @@ public void testAggregator() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + // Test unsupported types + if (!TestUtils.tryExecuteNonQueriesWithRetry( + env, + Arrays.asList( + "create timeSeries root.ln.wf01.wt01.boolean boolean", + "create timeSeries root.ln.wf01.wt01.date date", + "create timeSeries root.ln.wf01.wt01.text text", + "create timeSeries root.ln.wf01.wt01.string string", + "create timeSeries root.ln.wf01.wt01.blob blob", + "insert into root.ln.wf01.wt01(time, boolean, date, text, string, blob) values (20000, false, '2000-12-13', 'abc', 'def', X'f103')", + "flush"))) { + return; + } + if (!TestUtils.tryExecuteNonQueriesWithRetry( env, Arrays.asList( From 78ff20775428cdb7eb52eb13623852e601bacb98 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 5 Jun 2024 22:10:45 +0800 Subject: [PATCH 6/6] Added subscription tests --- .../it/local/IoTDBSubscriptionBasicIT.java | 95 ++++++++++++++++++- .../payload/SubscriptionSessionDataSet.java | 9 +- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 13a5e8abc2617..c2b67018858ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -36,10 +36,15 @@ import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler; import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; @@ -53,6 +58,7 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.time.Duration; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -83,12 +89,66 @@ public void tearDown() throws Exception { } @Test - public void testBasicSubscribeTablets() throws Exception { + public void testSubscribeBooleanData() throws Exception { + testBasicSubscribeTablets(TSDataType.BOOLEAN, "true", true); + } + + @Test + public void testSubscribeIntData() throws Exception { + testBasicSubscribeTablets(TSDataType.INT32, "1", 1); + } + + @Test + public void testSubscribeLongData() throws Exception { + testBasicSubscribeTablets(TSDataType.INT64, "1", 1L); + } + + @Test + public void testSubscribeFloatData() throws Exception { + testBasicSubscribeTablets(TSDataType.FLOAT, "1.0", 1.0F); + } + + @Test + public void testSubscribeDoubleData() throws Exception { + testBasicSubscribeTablets(TSDataType.DOUBLE, "1.0", 1.0); + } + + @Test + public void testSubscribeTextData() throws Exception { + testBasicSubscribeTablets(TSDataType.TEXT, "'a'", new Binary("a", TSFileConfig.STRING_CHARSET)); + } + + @Test + public void testSubscribeTimestampData() throws Exception { + testBasicSubscribeTablets(TSDataType.TIMESTAMP, "123", 123L); + } + + @Test + public void testSubscribeDateData() throws Exception { + testBasicSubscribeTablets(TSDataType.DATE, "'2011-03-01'", LocalDate.of(2011, 3, 1)); + } + + @Test + public void testSubscribeBlobData() throws Exception { + testBasicSubscribeTablets( + TSDataType.BLOB, "X'f013'", new Binary(new byte[] {(byte) 0xf0, 0x13})); + } + + @Test + public void testSubscribeStringData() throws Exception { + testBasicSubscribeTablets( + TSDataType.STRING, "'a'", new Binary("a", TSFileConfig.STRING_CHARSET)); + } + + private void testBasicSubscribeTablets( + final TSDataType type, final String valueStr, final Object expectedData) throws Exception { // Insert some historical data try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + String.format("create timeseries root.db.d1.s1 %s", type.toString())); for (int i = 0; i < 100; ++i) { session.executeNonQueryStatement( - String.format("insert into root.db.d1(time, s1) values (%s, 1)", i)); + String.format("insert into root.db.d1(time, s1) values (%s, %s)", i, valueStr)); } session.executeNonQueryStatement("flush"); } catch (final Exception e) { @@ -132,7 +192,12 @@ public void testBasicSubscribeTablets() throws Exception { for (final SubscriptionSessionDataSet dataSet : message.getSessionDataSetsHandler()) { while (dataSet.hasNext()) { - dataSet.next(); + final RowRecord record = dataSet.next(); + Assert.assertEquals(type.toString(), dataSet.getColumnTypes().get(1)); + Assert.assertEquals(type, record.getFields().get(0).getDataType()); + Assert.assertEquals(expectedData, getValue(type, dataSet.getTablet())); + Assert.assertEquals( + expectedData, record.getFields().get(0).getObjectValue(type)); rowCount.addAndGet(1); } } @@ -167,6 +232,30 @@ public void testBasicSubscribeTablets() throws Exception { } } + private Object getValue(final TSDataType type, final Tablet tablet) { + switch (type) { + case BOOLEAN: + return ((boolean[]) tablet.values[0])[0]; + case INT32: + return ((int[]) tablet.values[0])[0]; + case INT64: + case TIMESTAMP: + return ((long[]) tablet.values[0])[0]; + case FLOAT: + return ((float[]) tablet.values[0])[0]; + case DOUBLE: + return ((double[]) tablet.values[0])[0]; + case TEXT: + case BLOB: + case STRING: + return ((Binary[]) tablet.values[0])[0]; + case DATE: + return ((LocalDate[]) tablet.values[0])[0]; + default: + return null; + } + } + @Test public void testBasicSubscribeTsFile() throws Exception { // Insert some historical data diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java index c9e9fc61cebb5..ac54941f5151e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java @@ -25,10 +25,12 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -84,7 +86,7 @@ public List getColumnTypes() { final List schemas = tablet.getSchemas(); columnTypeList.addAll( - schemas.stream().map((schema) -> schema.getType().toString()).collect(Collectors.toList())); + schemas.stream().map(schema -> schema.getType().toString()).collect(Collectors.toList())); return columnTypeList; } @@ -149,10 +151,13 @@ private static Field generateFieldFromTabletValue( field.setBoolV(booleanValue); break; case INT32: - case DATE: final int intValue = ((int[]) value)[index]; field.setIntV(intValue); break; + case DATE: + final LocalDate dateValue = ((LocalDate[]) value)[index]; + field.setIntV(DateUtils.parseDateExpressionToInt(dateValue)); + break; case INT64: case TIMESTAMP: final long longValue = ((long[]) value)[index];