From 6b192950d0b2fe0ba4c3282ca1e82edcf8c11602 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 14:22:20 +0800
Subject: [PATCH 1/6] Pipe module
---
.../org/apache/iotdb/pipe/api/access/Row.java | 46 ++++---
.../org/apache/iotdb/pipe/api/type/Type.java | 15 ++-
.../apache/iotdb/udf/api/utils/RowImpl.java | 8 ++
.../request/PipeTransferTabletRawReq.java | 70 ++++++----
.../protocol/opcua/OpcUaConnector.java | 14 ++
.../common/row/PipeDataTypeTransformer.java | 14 +-
.../db/pipe/event/common/row/PipeRow.java | 53 +++++---
.../tablet/TabletInsertionDataContainer.java | 99 +++++++++++----
.../aggregate/AggregateProcessor.java | 43 ++++++-
.../aggregate/TimeSeriesRuntimeState.java | 94 ++++++++++++++
...CustomizedReadableIntermediateResults.java | 49 +++++--
.../IntermediateResultOperator.java | 41 +++---
.../AbstractSameTypeNumericOperator.java | 46 +++++--
.../FractionPoweredSumOperator.java | 56 +++++---
.../integertype/CountOperator.java | 54 +++++---
.../datastructure/TimeSeriesWindow.java | 111 ++++++++++++++++
.../AbstractSimpleTimeWindowingProcessor.java | 52 ++++++--
.../processor/AbstractWindowingProcessor.java | 45 ++++---
.../statement/crud/InsertTabletStatement.java | 4 +
.../PipeDataNodeThriftRequestTest.java | 120 +++++++++++-------
.../event/PipeDataTypeTransformerTest.java | 13 +-
.../event/PipeTabletInsertionEventTest.java | 40 +++++-
22 files changed, 843 insertions(+), 244 deletions(-)
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java
index b3c1342347da1..f22ac5e026dc6 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/access/Row.java
@@ -25,6 +25,7 @@
import org.apache.tsfile.read.common.Path;
+import java.time.LocalDate;
import java.util.List;
public interface Row {
@@ -44,17 +45,28 @@ public interface Row {
* @param columnIndex index of the specified column
* @return the int value at the specified column in this row
*/
- int getInt(int columnIndex);
+ int getInt(final int columnIndex);
+
+ /**
+ * Returns the localDate value at the specified column in this row.
+ *
+ *
Users need to ensure that the data type of the specified column is {@code Type.DATE}.
+ *
+ * @param columnIndex index of the specified column
+ * @return the int value at the specified column in this row
+ */
+ LocalDate getDate(final int columnIndex);
/**
* Returns the long value at the specified column in this row.
*
- *
Users need to ensure that the data type of the specified column is {@code Type.INT64}.
+ *
Users need to ensure that the data type of the specified column is {@code Type.INT64} or
+ * {@code Type.TIMESTAMP}.
*
* @param columnIndex index of the specified column
* @return the long value at the specified column in this row
*/
- long getLong(int columnIndex);
+ long getLong(final int columnIndex);
/**
* Returns the float value at the specified column in this row.
@@ -64,7 +76,7 @@ public interface Row {
* @param columnIndex index of the specified column
* @return the float value at the specified column in this row
*/
- float getFloat(int columnIndex);
+ float getFloat(final int columnIndex);
/**
* Returns the double value at the specified column in this row.
@@ -74,7 +86,7 @@ public interface Row {
* @param columnIndex index of the specified column
* @return the double value at the specified column in this row
*/
- double getDouble(int columnIndex);
+ double getDouble(final int columnIndex);
/**
* Returns the boolean value at the specified column in this row.
@@ -84,35 +96,37 @@ public interface Row {
* @param columnIndex index of the specified column
* @return the boolean value at the specified column in this row
*/
- boolean getBoolean(int columnIndex);
+ boolean getBoolean(final int columnIndex);
/**
* Returns the Binary value at the specified column in this row.
*
- *
Users need to ensure that the data type of the specified column is {@code Type.TEXT}.
+ *
Users need to ensure that the data type of the specified column is {@code Type.TEXT}, {@code
+ * Type.BLOB} or {@code Type.STRING}.
*
* @param columnIndex index of the specified column
* @return the Binary value at the specified column in this row
*/
- Binary getBinary(int columnIndex);
+ Binary getBinary(final int columnIndex);
/**
* Returns the String value at the specified column in this row.
*
- *
Users need to ensure that the data type of the specified column is {@code Type.TEXT}.
+ *
Users need to ensure that the data type of the specified column is {@code Type.TEXT} or
+ * {@code Type.STRING}.
*
* @param columnIndex index of the specified column
* @return the String value at the specified column in this row
*/
- String getString(int columnIndex);
+ String getString(final int columnIndex);
/**
- * Returns the Object value at the specified column in this row.
+ * Returns the raw Object stored at the specified column in this row.
*
* @param columnIndex index of the specified column
* @return the Object value at the specified column in this row
*/
- Object getObject(int columnIndex);
+ Object getObject(final int columnIndex);
/**
* Returns the actual data type of the value at the specified column in this row.
@@ -120,7 +134,7 @@ public interface Row {
* @param columnIndex index of the specified column
* @return the actual data type of the value at the specified column in this row
*/
- Type getDataType(int columnIndex);
+ Type getDataType(final int columnIndex);
/**
* Returns {@code true} if the value of the specified column is null.
@@ -128,7 +142,7 @@ public interface Row {
* @param columnIndex index of the specified column
* @return {@code true} if the value of the specified column is null
*/
- boolean isNull(int columnIndex);
+ boolean isNull(final int columnIndex);
/**
* Returns the number of columns (excluding the timestamp column).
@@ -144,7 +158,7 @@ public interface Row {
* @return the actual column index of the given column name
* @throws PipeParameterNotValidException if the given column name is not existed in the Row
*/
- int getColumnIndex(Path columnName) throws PipeParameterNotValidException;
+ int getColumnIndex(final Path columnName) throws PipeParameterNotValidException;
/**
* Returns the actual column (measurement) name of the given column index.
@@ -152,7 +166,7 @@ public interface Row {
* @param columnIndex the column index
* @return the actual column (measurement) name of the given column index
*/
- String getColumnName(int columnIndex);
+ String getColumnName(final int columnIndex);
/**
* Returns the column data types in the Row.
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
index 85a4390eae46e..9093133631a0e 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
@@ -36,7 +36,20 @@ public enum Type {
DOUBLE((byte) 4),
/* TEXT */
- TEXT((byte) 5);
+ TEXT((byte) 5),
+
+ /* TsDataType.Vector and TsDataType.UNKNOWN are inner types of TsFile-module, which should not be supported in UDF APIs. To be consistent with TsDataType, the next value starts with 8 */
+ /* TIMESTAMP */
+ TIMESTAMP((byte) 8),
+
+ /* DATE */
+ DATE((byte) 9),
+
+ /* BLOB */
+ BLOB((byte) 10),
+
+ /* STRING */
+ STRING((byte) 11);
private final byte dataType;
diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java
index e0ca3a53fc658..4a323b3a14b26 100644
--- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java
+++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java
@@ -121,6 +121,14 @@ private static Type transformToUDFDataType(TSDataType tsDataType) {
return Type.DOUBLE;
case 5:
return Type.TEXT;
+ case 8:
+ return Type.TIMESTAMP;
+ case 9:
+ return Type.DATE;
+ case 10:
+ return Type.BLOB;
+ case 11:
+ return Type.STRING;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
index e2512fac3ff7a..f6d677389e7dc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
@@ -43,6 +43,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Objects;
@@ -70,7 +71,7 @@ public InsertTabletStatement constructStatement() {
try {
final TSInsertTabletReq request = new TSInsertTabletReq();
- for (IMeasurementSchema measurementSchema : tablet.getSchemas()) {
+ for (final IMeasurementSchema measurementSchema : tablet.getSchemas()) {
request.addToMeasurements(measurementSchema.getMeasurementId());
request.addToTypes(measurementSchema.getType().ordinal());
}
@@ -84,13 +85,13 @@ public InsertTabletStatement constructStatement() {
PathUtils.checkIsLegalSingleMeasurementsAndUpdate(request.getMeasurements()));
return StatementGenerator.createStatement(request);
- } catch (MetadataException e) {
+ } catch (final MetadataException e) {
LOGGER.warn("Generate Statement from tablet {} error.", tablet, e);
return null;
}
}
- private static boolean checkSorted(Tablet tablet) {
+ private static boolean checkSorted(final Tablet tablet) {
for (int i = 1; i < tablet.rowSize; i++) {
if (tablet.timestamps[i] < tablet.timestamps[i - 1]) {
return false;
@@ -99,13 +100,13 @@ private static boolean checkSorted(Tablet tablet) {
return true;
}
- private static void sortTablet(Tablet tablet) {
+ private static void sortTablet(final Tablet tablet) {
/*
* following part of code sort the batch data by time,
* so we can insert continuous data in value list to get a better performance
*/
// sort to get index, and use index to sort value list
- Integer[] index = new Integer[tablet.rowSize];
+ final Integer[] index = new Integer[tablet.rowSize];
for (int i = 0; i < tablet.rowSize; i++) {
index[i] = i;
}
@@ -113,7 +114,7 @@ private static void sortTablet(Tablet tablet) {
Arrays.sort(tablet.timestamps, 0, tablet.rowSize);
int columnIndex = 0;
for (int i = 0; i < tablet.getSchemas().size(); i++) {
- IMeasurementSchema schema = tablet.getSchemas().get(i);
+ final IMeasurementSchema schema = tablet.getSchemas().get(i);
if (schema != null) {
tablet.values[columnIndex] = sortList(tablet.values[columnIndex], schema.getType(), index);
if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) {
@@ -133,46 +134,57 @@ private static void sortTablet(Tablet tablet) {
* @return sorted list
* @throws UnSupportedDataTypeException if dataType is illegal
*/
- private static Object sortList(Object valueList, TSDataType dataType, Integer[] index) {
+ private static Object sortList(
+ final Object valueList, final TSDataType dataType, final Integer[] index) {
switch (dataType) {
case BOOLEAN:
- boolean[] boolValues = (boolean[]) valueList;
- boolean[] sortedValues = new boolean[boolValues.length];
+ final boolean[] boolValues = (boolean[]) valueList;
+ final boolean[] sortedValues = new boolean[boolValues.length];
for (int i = 0; i < index.length; i++) {
sortedValues[i] = boolValues[index[i]];
}
return sortedValues;
case INT32:
- int[] intValues = (int[]) valueList;
- int[] sortedIntValues = new int[intValues.length];
+ final int[] intValues = (int[]) valueList;
+ final int[] sortedIntValues = new int[intValues.length];
for (int i = 0; i < index.length; i++) {
sortedIntValues[i] = intValues[index[i]];
}
return sortedIntValues;
+ case DATE:
+ final LocalDate[] dateValues = (LocalDate[]) valueList;
+ final LocalDate[] sortedDateValues = new LocalDate[dateValues.length];
+ for (int i = 0; i < index.length; i++) {
+ sortedDateValues[i] = dateValues[index[i]];
+ }
+ return sortedDateValues;
case INT64:
- long[] longValues = (long[]) valueList;
- long[] sortedLongValues = new long[longValues.length];
+ case TIMESTAMP:
+ final long[] longValues = (long[]) valueList;
+ final long[] sortedLongValues = new long[longValues.length];
for (int i = 0; i < index.length; i++) {
sortedLongValues[i] = longValues[index[i]];
}
return sortedLongValues;
case FLOAT:
- float[] floatValues = (float[]) valueList;
- float[] sortedFloatValues = new float[floatValues.length];
+ final float[] floatValues = (float[]) valueList;
+ final float[] sortedFloatValues = new float[floatValues.length];
for (int i = 0; i < index.length; i++) {
sortedFloatValues[i] = floatValues[index[i]];
}
return sortedFloatValues;
case DOUBLE:
- double[] doubleValues = (double[]) valueList;
- double[] sortedDoubleValues = new double[doubleValues.length];
+ final double[] doubleValues = (double[]) valueList;
+ final double[] sortedDoubleValues = new double[doubleValues.length];
for (int i = 0; i < index.length; i++) {
sortedDoubleValues[i] = doubleValues[index[i]];
}
return sortedDoubleValues;
case TEXT:
- Binary[] binaryValues = (Binary[]) valueList;
- Binary[] sortedBinaryValues = new Binary[binaryValues.length];
+ case BLOB:
+ case STRING:
+ final Binary[] binaryValues = (Binary[]) valueList;
+ final Binary[] sortedBinaryValues = new Binary[binaryValues.length];
for (int i = 0; i < index.length; i++) {
sortedBinaryValues[i] = binaryValues[index[i]];
}
@@ -190,8 +202,8 @@ private static Object sortList(Object valueList, TSDataType dataType, Integer[]
* @param index index
* @return sorted bitMap
*/
- private static BitMap sortBitMap(BitMap bitMap, Integer[] index) {
- BitMap sortedBitMap = new BitMap(bitMap.getSize());
+ private static BitMap sortBitMap(final BitMap bitMap, final Integer[] index) {
+ final BitMap sortedBitMap = new BitMap(bitMap.getSize());
for (int i = 0; i < index.length; i++) {
if (bitMap.isMarked(index[i])) {
sortedBitMap.mark(i);
@@ -202,7 +214,8 @@ private static BitMap sortBitMap(BitMap bitMap, Integer[] index) {
/////////////////////////////// WriteBack & Batch ///////////////////////////////
- public static PipeTransferTabletRawReq toTPipeTransferRawReq(Tablet tablet, boolean isAligned) {
+ public static PipeTransferTabletRawReq toTPipeTransferRawReq(
+ final Tablet tablet, final boolean isAligned) {
final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
tabletReq.tablet = tablet;
@@ -213,8 +226,8 @@ public static PipeTransferTabletRawReq toTPipeTransferRawReq(Tablet tablet, bool
/////////////////////////////// Thrift ///////////////////////////////
- public static PipeTransferTabletRawReq toTPipeTransferReq(Tablet tablet, boolean isAligned)
- throws IOException {
+ public static PipeTransferTabletRawReq toTPipeTransferReq(
+ final Tablet tablet, final boolean isAligned) throws IOException {
final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
tabletReq.tablet = tablet;
@@ -233,7 +246,7 @@ public static PipeTransferTabletRawReq toTPipeTransferReq(Tablet tablet, boolean
return tabletReq;
}
- public static PipeTransferTabletRawReq fromTPipeTransferReq(TPipeTransferReq transferReq) {
+ public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) {
final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq();
tabletReq.tablet = Tablet.deserialize(transferReq.body);
@@ -248,7 +261,8 @@ public static PipeTransferTabletRawReq fromTPipeTransferReq(TPipeTransferReq tra
/////////////////////////////// Air Gap ///////////////////////////////
- public static byte[] toTPipeTransferBytes(Tablet tablet, boolean isAligned) throws IOException {
+ public static byte[] toTPipeTransferBytes(final Tablet tablet, final boolean isAligned)
+ throws IOException {
try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBConnectorRequestVersion.VERSION_1.getVersion(), outputStream);
@@ -262,14 +276,14 @@ public static byte[] toTPipeTransferBytes(Tablet tablet, boolean isAligned) thro
/////////////////////////////// Object ///////////////////////////////
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
- PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj;
+ final PipeTransferTabletRawReq that = (PipeTransferTabletRawReq) obj;
return Objects.equals(tablet, that.tablet)
&& isAligned == that.isAligned
&& version == that.version
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java
index fa1c372db2854..158d8e84bb907 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaConnector.java
@@ -45,6 +45,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDate;
import java.util.Arrays;
import java.util.Map;
import java.util.UUID;
@@ -260,7 +261,13 @@ private void transferTablet(final OpcUaServer server, final Tablet tablet) throw
LocalizedText.english(
Integer.toString(((int[]) tablet.values[columnIndex])[rowIndex])));
break;
+ case DATE:
+ eventNode.setMessage(
+ LocalizedText.english(
+ (((LocalDate[]) tablet.values[columnIndex])[rowIndex]).toString()));
+ break;
case INT64:
+ case TIMESTAMP:
eventNode.setMessage(
LocalizedText.english(
Long.toString(((long[]) tablet.values[columnIndex])[rowIndex])));
@@ -276,6 +283,8 @@ private void transferTablet(final OpcUaServer server, final Tablet tablet) throw
Double.toString(((double[]) tablet.values[columnIndex])[rowIndex])));
break;
case TEXT:
+ case BLOB:
+ case STRING:
eventNode.setMessage(
LocalizedText.english(
((Binary[]) tablet.values[columnIndex])[rowIndex].toString()));
@@ -300,13 +309,18 @@ private NodeId convertToOpcDataType(final TSDataType type) {
return Identifiers.Boolean;
case INT32:
return Identifiers.Int32;
+ case DATE:
+ return Identifiers.DateTime;
case INT64:
+ case TIMESTAMP:
return Identifiers.Int64;
case FLOAT:
return Identifiers.Float;
case DOUBLE:
return Identifiers.Double;
case TEXT:
+ case BLOB:
+ case STRING:
return Identifiers.String;
case VECTOR:
case UNKNOWN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java
index 41ffd7f169109..caa98cdad6ace 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeDataTypeTransformer.java
@@ -29,7 +29,7 @@
/** Transform between {@link TSDataType} and {@link org.apache.iotdb.pipe.api.type.Type}. */
public class PipeDataTypeTransformer {
- public static List transformToPipeDataTypeList(List tsDataTypeList) {
+ public static List transformToPipeDataTypeList(final List tsDataTypeList) {
return tsDataTypeList == null
? null
: tsDataTypeList.stream()
@@ -37,11 +37,11 @@ public static List transformToPipeDataTypeList(List tsDataType
.collect(Collectors.toList());
}
- public static Type transformToPipeDataType(TSDataType tsDataType) {
+ public static Type transformToPipeDataType(final TSDataType tsDataType) {
return tsDataType == null ? null : getPipeDataType(tsDataType.getType());
}
- private static Type getPipeDataType(byte type) {
+ private static Type getPipeDataType(final byte type) {
switch (type) {
case 0:
return Type.BOOLEAN;
@@ -55,6 +55,14 @@ private static Type getPipeDataType(byte type) {
return Type.DOUBLE;
case 5:
return Type.TEXT;
+ case 8:
+ return Type.TIMESTAMP;
+ case 9:
+ return Type.DATE;
+ case 10:
+ return Type.BLOB;
+ case 11:
+ return Type.STRING;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java
index 0296935e7bb1c..6200f66536ee3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java
@@ -29,6 +29,7 @@
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import java.time.LocalDate;
import java.util.Arrays;
import java.util.List;
@@ -48,15 +49,15 @@ public class PipeRow implements Row {
protected final String[] columnNameStringList;
public PipeRow(
- int rowIndex,
- String deviceId,
- boolean isAligned,
- MeasurementSchema[] measurementSchemaList,
- long[] timestampColumn,
- TSDataType[] valueColumnTypes,
- Object[] valueColumns,
- BitMap[] bitMaps,
- String[] columnNameStringList) {
+ final int rowIndex,
+ final String deviceId,
+ final boolean isAligned,
+ final MeasurementSchema[] measurementSchemaList,
+ final long[] timestampColumn,
+ final TSDataType[] valueColumnTypes,
+ final Object[] valueColumns,
+ final BitMap[] bitMaps,
+ final String[] columnNameStringList) {
this.rowIndex = rowIndex;
this.deviceId = deviceId;
this.isAligned = isAligned;
@@ -74,49 +75,57 @@ public long getTime() {
}
@Override
- public int getInt(int columnIndex) {
+ public int getInt(final int columnIndex) {
return ((int[]) valueColumns[columnIndex])[rowIndex];
}
@Override
- public long getLong(int columnIndex) {
+ public LocalDate getDate(final int columnIndex) {
+ return ((LocalDate[]) valueColumns[columnIndex])[rowIndex];
+ }
+
+ @Override
+ public long getLong(final int columnIndex) {
return ((long[]) valueColumns[columnIndex])[rowIndex];
}
@Override
- public float getFloat(int columnIndex) {
+ public float getFloat(final int columnIndex) {
return ((float[]) valueColumns[columnIndex])[rowIndex];
}
@Override
- public double getDouble(int columnIndex) {
+ public double getDouble(final int columnIndex) {
return ((double[]) valueColumns[columnIndex])[rowIndex];
}
@Override
- public boolean getBoolean(int columnIndex) {
+ public boolean getBoolean(final int columnIndex) {
return ((boolean[]) valueColumns[columnIndex])[rowIndex];
}
@Override
- public org.apache.iotdb.pipe.api.type.Binary getBinary(int columnIndex) {
+ public org.apache.iotdb.pipe.api.type.Binary getBinary(final int columnIndex) {
return PipeBinaryTransformer.transformToPipeBinary(
((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex]);
}
@Override
- public String getString(int columnIndex) {
+ public String getString(final int columnIndex) {
final org.apache.tsfile.utils.Binary binary =
((org.apache.tsfile.utils.Binary[]) valueColumns[columnIndex])[rowIndex];
return binary == null ? null : binary.getStringValue(TSFileConfig.STRING_CHARSET);
}
@Override
- public Object getObject(int columnIndex) {
+ public Object getObject(final int columnIndex) {
switch (getDataType(columnIndex)) {
case INT32:
return getInt(columnIndex);
+ case DATE:
+ return getDate(columnIndex);
case INT64:
+ case TIMESTAMP:
return getLong(columnIndex);
case FLOAT:
return getFloat(columnIndex);
@@ -125,6 +134,8 @@ public Object getObject(int columnIndex) {
case BOOLEAN:
return getBoolean(columnIndex);
case TEXT:
+ case BLOB:
+ case STRING:
return getBinary(columnIndex);
default:
throw new UnsupportedOperationException(
@@ -135,12 +146,12 @@ public Object getObject(int columnIndex) {
}
@Override
- public Type getDataType(int columnIndex) {
+ public Type getDataType(final int columnIndex) {
return PipeDataTypeTransformer.transformToPipeDataType(valueColumnTypes[columnIndex]);
}
@Override
- public boolean isNull(int columnIndex) {
+ public boolean isNull(final int columnIndex) {
return bitMaps[columnIndex].isMarked(rowIndex);
}
@@ -150,7 +161,7 @@ public int size() {
}
@Override
- public int getColumnIndex(Path columnName) throws PipeParameterNotValidException {
+ public int getColumnIndex(final Path columnName) throws PipeParameterNotValidException {
for (int i = 0; i < columnNameStringList.length; i++) {
if (columnNameStringList[i].equals(columnName.getFullPath())) {
return i;
@@ -161,7 +172,7 @@ public int getColumnIndex(Path columnName) throws PipeParameterNotValidException
}
@Override
- public String getColumnName(int columnIndex) {
+ public String getColumnName(final int columnIndex) {
return columnNameStringList[columnIndex];
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
index 83fa0497f18f1..26ae666ed2394 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
@@ -35,6 +35,7 @@
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -42,6 +43,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -141,7 +143,7 @@ private void parse(final InsertRowNode insertRowNode, final PipePattern pattern)
this.isAligned = insertRowNode.isAligned();
final long[] originTimestampColumn = new long[] {insertRowNode.getTime()};
- List rowIndexList = generateRowIndexList(originTimestampColumn);
+ final List rowIndexList = generateRowIndexList(originTimestampColumn);
this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray();
generateColumnIndexMapper(
@@ -170,7 +172,7 @@ private void parse(final InsertRowNode insertRowNode, final PipePattern pattern)
this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i];
this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i];
this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i];
- BitMap bitMap = new BitMap(this.timestampColumn.length);
+ final BitMap bitMap = new BitMap(this.timestampColumn.length);
if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) {
this.valueColumns[filteredColumnIndex] = null;
bitMap.markAll();
@@ -209,7 +211,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa
final long[] originTimestampColumn = insertTabletNode.getTimes();
final int originRowSize = originTimestampColumn.length;
- List rowIndexList = generateRowIndexList(originTimestampColumn);
+ final List rowIndexList = generateRowIndexList(originTimestampColumn);
this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray();
generateColumnIndexMapper(
@@ -253,7 +255,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa
this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i];
this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i];
this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i];
- BitMap bitMap = new BitMap(this.timestampColumn.length);
+ final BitMap bitMap = new BitMap(this.timestampColumn.length);
if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) {
this.valueColumns[filteredColumnIndex] = null;
bitMap.markAll();
@@ -293,7 +295,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte
final long[] originTimestampColumn =
Arrays.copyOf(
tablet.timestamps, tablet.rowSize); // tablet.timestamps.length == tablet.maxRowNumber
- List rowIndexList = generateRowIndexList(originTimestampColumn);
+ final List rowIndexList = generateRowIndexList(originTimestampColumn);
this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray();
final List originMeasurementSchemaList = tablet.getSchemas();
@@ -344,7 +346,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte
this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList.get(i);
this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i];
this.valueColumnTypes[filteredColumnIndex] = originValueColumnTypes[i];
- BitMap bitMap = new BitMap(this.timestampColumn.length);
+ final BitMap bitMap = new BitMap(this.timestampColumn.length);
if (Objects.isNull(originValueColumns[i]) || Objects.isNull(originValueColumnTypes[i])) {
this.valueColumns[filteredColumnIndex] = null;
bitMap.markAll();
@@ -414,7 +416,7 @@ private List generateRowIndexList(final long[] originTimestampColumn) {
return generateFullRowIndexList(rowCount);
}
- List rowIndexList = new ArrayList<>();
+ final List rowIndexList = new ArrayList<>();
// We assume that `originTimestampColumn` is ordered.
if (originTimestampColumn[originTimestampColumn.length - 1] < sourceEvent.getStartTime()
|| originTimestampColumn[0] > sourceEvent.getEndTime()) {
@@ -431,7 +433,7 @@ private List generateRowIndexList(final long[] originTimestampColumn) {
return rowIndexList;
}
- private static List generateFullRowIndexList(int rowCount) {
+ private static List generateFullRowIndexList(final int rowCount) {
if (rowCount <= CACHED_FULL_ROW_INDEX_LIST_ROW_COUNT_UPPER) {
return cachedFullRowIndexList.get(rowCount);
}
@@ -439,20 +441,20 @@ private static List generateFullRowIndexList(int rowCount) {
}
private static Object filterValueColumnsByRowIndexList(
- @NonNull TSDataType type,
- @NonNull Object originValueColumn,
- @NonNull List rowIndexList,
- boolean isSingleOriginValueColumn,
- @NonNull BitMap originNullValueColumnBitmap,
- @NonNull BitMap nullValueColumnBitmap /* output parameters */) {
+ @NonNull final TSDataType type,
+ @NonNull final Object originValueColumn,
+ @NonNull final List rowIndexList,
+ final boolean isSingleOriginValueColumn,
+ @NonNull final BitMap originNullValueColumnBitmap,
+ @NonNull final BitMap nullValueColumnBitmap /* output parameters */) {
switch (type) {
case INT32:
{
- int[] intValueColumns =
+ final int[] intValueColumns =
isSingleOriginValueColumn
? new int[] {(int) originValueColumn}
: (int[]) originValueColumn;
- int[] valueColumns = new int[rowIndexList.size()];
+ final int[] valueColumns = new int[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = 0;
@@ -463,13 +465,52 @@ private static Object filterValueColumnsByRowIndexList(
}
return valueColumns;
}
+ case DATE:
+ {
+ // Always store 'LocalDate[]' to help convert to tablet
+ final LocalDate[] valueColumns = new LocalDate[rowIndexList.size()];
+ if (isSingleOriginValueColumn && originValueColumn instanceof LocalDate
+ || !isSingleOriginValueColumn && originValueColumn instanceof LocalDate[]) {
+ // For tablet
+ final LocalDate[] dateValueColumns =
+ isSingleOriginValueColumn
+ ? new LocalDate[] {(LocalDate) originValueColumn}
+ : (LocalDate[]) originValueColumn;
+
+ for (int i = 0; i < rowIndexList.size(); ++i) {
+ if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
+ valueColumns[i] = LocalDate.MIN;
+ nullValueColumnBitmap.mark(i);
+ } else {
+ valueColumns[i] = dateValueColumns[rowIndexList.get(i)];
+ }
+ }
+ } else {
+ // For insertRowNode / insertTabletNode
+ final int[] intValueColumns =
+ isSingleOriginValueColumn
+ ? new int[] {(int) originValueColumn}
+ : (int[]) originValueColumn;
+ for (int i = 0; i < rowIndexList.size(); ++i) {
+ if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
+ valueColumns[i] = LocalDate.MIN;
+ nullValueColumnBitmap.mark(i);
+ } else {
+ valueColumns[i] =
+ DateUtils.parseIntToLocalDate(intValueColumns[rowIndexList.get(i)]);
+ }
+ }
+ }
+ return valueColumns;
+ }
case INT64:
+ case TIMESTAMP:
{
- long[] longValueColumns =
+ final long[] longValueColumns =
isSingleOriginValueColumn
? new long[] {(long) originValueColumn}
: (long[]) originValueColumn;
- long[] valueColumns = new long[rowIndexList.size()];
+ final long[] valueColumns = new long[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = 0L;
@@ -482,11 +523,11 @@ private static Object filterValueColumnsByRowIndexList(
}
case FLOAT:
{
- float[] floatValueColumns =
+ final float[] floatValueColumns =
isSingleOriginValueColumn
? new float[] {(float) originValueColumn}
: (float[]) originValueColumn;
- float[] valueColumns = new float[rowIndexList.size()];
+ final float[] valueColumns = new float[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = 0F;
@@ -499,11 +540,11 @@ private static Object filterValueColumnsByRowIndexList(
}
case DOUBLE:
{
- double[] doubleValueColumns =
+ final double[] doubleValueColumns =
isSingleOriginValueColumn
? new double[] {(double) originValueColumn}
: (double[]) originValueColumn;
- double[] valueColumns = new double[rowIndexList.size()];
+ final double[] valueColumns = new double[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = 0D;
@@ -516,11 +557,11 @@ private static Object filterValueColumnsByRowIndexList(
}
case BOOLEAN:
{
- boolean[] booleanValueColumns =
+ final boolean[] booleanValueColumns =
isSingleOriginValueColumn
? new boolean[] {(boolean) originValueColumn}
: (boolean[]) originValueColumn;
- boolean[] valueColumns = new boolean[rowIndexList.size()];
+ final boolean[] valueColumns = new boolean[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) {
valueColumns[i] = false;
@@ -532,12 +573,14 @@ private static Object filterValueColumnsByRowIndexList(
return valueColumns;
}
case TEXT:
+ case BLOB:
+ case STRING:
{
- Binary[] binaryValueColumns =
+ final Binary[] binaryValueColumns =
isSingleOriginValueColumn
? new Binary[] {(Binary) originValueColumn}
: (Binary[]) originValueColumn;
- Binary[] valueColumns = new Binary[rowIndexList.size()];
+ final Binary[] valueColumns = new Binary[rowIndexList.size()];
for (int i = 0; i < rowIndexList.size(); ++i) {
if (Objects.isNull(binaryValueColumns[rowIndexList.get(i)])
|| Objects.isNull(binaryValueColumns[rowIndexList.get(i)].getValues())
@@ -558,7 +601,7 @@ private static Object filterValueColumnsByRowIndexList(
//////////////////////////// process ////////////////////////////
- public List processRowByRow(BiConsumer consumer) {
+ public List processRowByRow(final BiConsumer consumer) {
if (valueColumns.length == 0 || timestampColumn.length == 0) {
return Collections.emptyList();
}
@@ -581,7 +624,7 @@ public List processRowByRow(BiConsumer
return rowCollector.convertToTabletInsertionEvents(shouldReport);
}
- public List processTablet(BiConsumer consumer) {
+ public List processTablet(final BiConsumer consumer) {
final PipeRowCollector rowCollector = new PipeRowCollector(pipeTaskMeta, sourceEvent);
consumer.accept(convertToTablet(), rowCollector);
return rowCollector.convertToTabletInsertionEvents(shouldReport);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
index dde1fab5a2802..0aeae73da75b2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
@@ -56,14 +56,17 @@
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -450,7 +453,13 @@ private Map> processRow(
state.updateWindows(
timestamp, row.getInt(index), outputMinReportIntervalMilliseconds);
break;
+ case DATE:
+ result =
+ state.updateWindows(
+ timestamp, row.getDate(index), outputMinReportIntervalMilliseconds);
+ break;
case INT64:
+ case TIMESTAMP:
result =
state.updateWindows(
timestamp, row.getLong(index), outputMinReportIntervalMilliseconds);
@@ -466,10 +475,16 @@ private Map> processRow(
timestamp, row.getDouble(index), outputMinReportIntervalMilliseconds);
break;
case TEXT:
+ case STRING:
result =
state.updateWindows(
timestamp, row.getString(index), outputMinReportIntervalMilliseconds);
break;
+ case BLOB:
+ result =
+ state.updateWindows(
+ timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds);
+ break;
default:
throw new UnsupportedOperationException(
String.format("The type %s is not supported", row.getDataType(index)));
@@ -620,7 +635,11 @@ public void collectWindowOutputs(
case INT32:
valueColumns[columnIndex] = new int[distinctOutputs.size()];
break;
+ case DATE:
+ valueColumns[columnIndex] = new LocalDate[distinctOutputs.size()];
+ break;
case INT64:
+ case TIMESTAMP:
valueColumns[columnIndex] = new long[distinctOutputs.size()];
break;
case FLOAT:
@@ -630,7 +649,9 @@ public void collectWindowOutputs(
valueColumns[columnIndex] = new double[distinctOutputs.size()];
break;
case TEXT:
- valueColumns[columnIndex] = new String[distinctOutputs.size()];
+ case BLOB:
+ case STRING:
+ valueColumns[columnIndex] = new Binary[distinctOutputs.size()];
break;
default:
throw new UnsupportedOperationException(
@@ -649,7 +670,12 @@ public void collectWindowOutputs(
((int[]) valueColumns[columnIndex])[rowIndex] =
(int) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
break;
+ case DATE:
+ ((LocalDate[]) valueColumns[columnIndex])[rowIndex] =
+ (LocalDate) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
+ break;
case INT64:
+ case TIMESTAMP:
((long[]) valueColumns[columnIndex])[rowIndex] =
(long) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
break;
@@ -662,8 +688,19 @@ public void collectWindowOutputs(
(double) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
break;
case TEXT:
- ((String[]) valueColumns[columnIndex])[rowIndex] =
- (String) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
+ case STRING:
+ ((Binary[]) valueColumns[columnIndex])[rowIndex] =
+ aggregatedResults.get(columnNameStringList[columnIndex]).getRight()
+ instanceof Binary
+ ? (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight()
+ : new Binary(
+ (String)
+ aggregatedResults.get(columnNameStringList[columnIndex]).getRight(),
+ TSFileConfig.STRING_CHARSET);
+ break;
+ case BLOB:
+ ((Binary[]) valueColumns[columnIndex])[rowIndex] =
+ (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
break;
default:
throw new UnsupportedOperationException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java
index 64864e087250e..2cb9a41469032 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/TimeSeriesRuntimeState.java
@@ -25,6 +25,7 @@
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput;
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState;
import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.PublicBAOS;
@@ -33,6 +34,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -165,6 +167,52 @@ public Pair, Pair> updateWindows(
return output;
}
+ public Pair, Pair> updateWindows(
+ final long timestamp, final LocalDate value, final long outputMinReportIntervalMilliseconds)
+ throws IOException {
+ Pair, Pair> output = null;
+
+ if (timestamp <= lastReportTimeStamp) {
+ return null;
+ }
+ final Set addedWindows =
+ windowingProcessor.mayAddWindow(currentOpeningWindows, timestamp, value);
+ if (Objects.nonNull(addedWindows)) {
+ addedWindows.forEach(
+ window ->
+ window.initWindow(
+ intermediateResultName2OperatorSupplierMap,
+ aggregatorOutputName2OperatorMap,
+ systemParameters));
+ }
+ final Iterator windowIterator = currentOpeningWindows.iterator();
+ while (windowIterator.hasNext()) {
+ final TimeSeriesWindow window = windowIterator.next();
+ final Pair stateWindowOutputPair =
+ window.updateIntermediateResult(timestamp, value);
+ if (Objects.isNull(stateWindowOutputPair)) {
+ continue;
+ }
+ if (stateWindowOutputPair.getLeft().isEmit()
+ && Objects.nonNull(stateWindowOutputPair.getRight())) {
+ outputList.add(stateWindowOutputPair.getRight());
+ lastReportTimeStamp =
+ Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
+ }
+ if (stateWindowOutputPair.getLeft().isPurge()) {
+ windowIterator.remove();
+ }
+ }
+ if (!outputList.isEmpty()) {
+ output =
+ new Pair<>(
+ new ArrayList<>(outputList),
+ getTimestampWindowBufferPair(outputMinReportIntervalMilliseconds));
+ outputList.clear();
+ }
+ return output;
+ }
+
public Pair, Pair> updateWindows(
final long timestamp, final long value, final long outputMinReportIntervalMilliseconds)
throws IOException {
@@ -349,6 +397,52 @@ public Pair, Pair> updateWindows(
return output;
}
+ public Pair, Pair> updateWindows(
+ final long timestamp, final Binary value, final long outputMinReportIntervalMilliseconds)
+ throws IOException {
+ Pair, Pair> output = null;
+
+ if (timestamp <= lastReportTimeStamp) {
+ return null;
+ }
+ final Set addedWindows =
+ windowingProcessor.mayAddWindow(currentOpeningWindows, timestamp, value);
+ if (Objects.nonNull(addedWindows)) {
+ addedWindows.forEach(
+ window ->
+ window.initWindow(
+ intermediateResultName2OperatorSupplierMap,
+ aggregatorOutputName2OperatorMap,
+ systemParameters));
+ }
+ final Iterator windowIterator = currentOpeningWindows.iterator();
+ while (windowIterator.hasNext()) {
+ final TimeSeriesWindow window = windowIterator.next();
+ final Pair stateWindowOutputPair =
+ window.updateIntermediateResult(timestamp, value);
+ if (Objects.isNull(stateWindowOutputPair)) {
+ continue;
+ }
+ if (stateWindowOutputPair.getLeft().isEmit()
+ && Objects.nonNull(stateWindowOutputPair.getRight())) {
+ outputList.add(stateWindowOutputPair.getRight());
+ lastReportTimeStamp =
+ Math.max(lastReportTimeStamp, stateWindowOutputPair.getRight().getProgressTime());
+ }
+ if (stateWindowOutputPair.getLeft().isPurge()) {
+ windowIterator.remove();
+ }
+ }
+ if (!outputList.isEmpty()) {
+ output =
+ new Pair<>(
+ new ArrayList<>(outputList),
+ getTimestampWindowBufferPair(outputMinReportIntervalMilliseconds));
+ outputList.clear();
+ }
+ return output;
+ }
+
public List forceOutput() {
return currentOpeningWindows.stream()
.map(TimeSeriesWindow::forceOutput)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java
index 9af6bcae73631..f9c9c5f828809 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/CustomizedReadableIntermediateResults.java
@@ -19,20 +19,28 @@
package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.pipe.api.type.Binary;
+import org.apache.iotdb.rpc.RpcUtils;
+
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.BytesUtils;
+import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.utils.Pair;
+import java.time.LocalDate;
+import java.time.ZoneId;
import java.util.Map;
public class CustomizedReadableIntermediateResults {
private final Map> intermediateResults;
public CustomizedReadableIntermediateResults(
- Map> intermediateResults) {
+ final Map> intermediateResults) {
this.intermediateResults = intermediateResults;
}
- public boolean getBoolean(String key) {
+ public boolean getBoolean(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
if (typeResultPair.getLeft() == TSDataType.BOOLEAN) {
return (boolean) typeResultPair.getRight();
@@ -41,7 +49,7 @@ public boolean getBoolean(String key) {
String.format("The type %s cannot be casted to boolean.", typeResultPair.getLeft()));
}
- public int getInt(String key) {
+ public int getInt(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
final TSDataType type = typeResultPair.getLeft();
@@ -49,7 +57,10 @@ public int getInt(String key) {
switch (type) {
case INT32:
return (int) value;
+ case DATE:
+ return DateUtils.parseDateExpressionToInt((LocalDate) value);
case INT64:
+ case TIMESTAMP:
return (int) (long) value;
case FLOAT:
return (int) (float) value;
@@ -61,7 +72,7 @@ public int getInt(String key) {
}
}
- public long getLong(String key) {
+ public long getLong(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
final TSDataType type = typeResultPair.getLeft();
@@ -69,7 +80,10 @@ public long getLong(String key) {
switch (type) {
case INT32:
return (int) value;
+ case DATE:
+ return DateUtils.parseDateExpressionToInt((LocalDate) value);
case INT64:
+ case TIMESTAMP:
return (long) value;
case FLOAT:
return (long) (float) value;
@@ -81,7 +95,7 @@ public long getLong(String key) {
}
}
- public float getFloat(String key) {
+ public float getFloat(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
final TSDataType type = typeResultPair.getLeft();
@@ -89,7 +103,10 @@ public float getFloat(String key) {
switch (type) {
case INT32:
return (int) value;
+ case DATE:
+ return DateUtils.parseDateExpressionToInt((LocalDate) value);
case INT64:
+ case TIMESTAMP:
return (long) value;
case FLOAT:
return (float) value;
@@ -101,7 +118,7 @@ public float getFloat(String key) {
}
}
- public double getDouble(String key) {
+ public double getDouble(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
final TSDataType type = typeResultPair.getLeft();
@@ -109,7 +126,10 @@ public double getDouble(String key) {
switch (type) {
case INT32:
return (int) value;
+ case DATE:
+ return DateUtils.parseDateExpressionToInt((LocalDate) value);
case INT64:
+ case TIMESTAMP:
return (long) value;
case FLOAT:
return (float) value;
@@ -123,7 +143,7 @@ public double getDouble(String key) {
// Note: This method will cast any decimal types to string without throwing
// any exceptions.
- public String getString(String key) {
+ public String getString(final String key) {
final Pair typeResultPair = intermediateResults.get(key);
final TSDataType type = typeResultPair.getLeft();
@@ -133,14 +153,25 @@ public String getString(String key) {
return Boolean.toString((boolean) value);
case INT32:
return Integer.toString((int) value);
+ case DATE:
+ return ((LocalDate) value).toString();
case INT64:
return Long.toString((long) value);
+ case TIMESTAMP:
+ return RpcUtils.formatDatetime(
+ RpcUtils.DEFAULT_TIME_FORMAT,
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision(),
+ (long) value,
+ ZoneId.systemDefault());
case FLOAT:
return Float.toString((float) value);
case DOUBLE:
return Double.toString((double) value);
case TEXT:
+ case STRING:
return (String) value;
+ case BLOB:
+ return BytesUtils.parseBlobByteArrayToString(((Binary) value).getValues());
default:
throw new UnsupportedOperationException(
String.format("The type %s cannot be casted to string.", typeResultPair.getLeft()));
@@ -148,11 +179,11 @@ public String getString(String key) {
}
// The caller may cast the object by itself.
- public Object getObject(String key) {
+ public Object getObject(final String key) {
return intermediateResults.get(key).getRight();
}
- public TSDataType getType(String key) {
+ public TSDataType getType(final String key) {
return intermediateResults.get(key).getLeft();
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java
index d68191885775a..95e1a9cb0d1e6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/IntermediateResultOperator.java
@@ -19,12 +19,15 @@
package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult;
+import org.apache.iotdb.pipe.api.type.Binary;
+
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Map;
/**
@@ -53,7 +56,7 @@ public interface IntermediateResultOperator {
*
* @param systemParams the system parameters
*/
- void configureSystemParameters(Map systemParams);
+ void configureSystemParameters(final Map systemParams);
/**
* The operator may init its value and typically set the output intermediate value type by the
@@ -66,17 +69,21 @@ public interface IntermediateResultOperator {
* @param initialInput the initial data
* @return if the input type is supported
*/
- boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp);
+
+ boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp);
+
+ boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp);
- boolean initAndGetIsSupport(int initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp);
- boolean initAndGetIsSupport(long initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp);
- boolean initAndGetIsSupport(float initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp);
- boolean initAndGetIsSupport(double initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp);
- boolean initAndGetIsSupport(String initialInput, long initialTimestamp);
+ boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp);
/**
* Use the input to update the intermediate result. The input is all raw types instead of Object
@@ -84,17 +91,21 @@ public interface IntermediateResultOperator {
*
* @param input the inputs
*/
- void updateValue(boolean input, long timestamp);
+ void updateValue(final boolean input, final long timestamp);
+
+ void updateValue(final int input, final long timestamp);
+
+ void updateValue(final LocalDate input, final long timestamp);
- void updateValue(int input, long timestamp);
+ void updateValue(final long input, final long timestamp);
- void updateValue(long input, long timestamp);
+ void updateValue(final float input, final long timestamp);
- void updateValue(float input, long timestamp);
+ void updateValue(final double input, final long timestamp);
- void updateValue(double input, long timestamp);
+ void updateValue(final String input, final long timestamp);
- void updateValue(String input, long timestamp);
+ void updateValue(final Binary input, final long timestamp);
/**
* Get the result and its type to calculate the aggregated value. If the type is List, Map or
@@ -106,8 +117,8 @@ public interface IntermediateResultOperator {
* Serialize its intermediate result to the outputStream to allow shutdown restart. The operator
* may as well serialize its own status.
*/
- void serialize(DataOutputStream outputStream) throws IOException;
+ void serialize(final DataOutputStream outputStream) throws IOException;
/** Deserialize the object from byteBuffer to allow shutdown restart */
- void deserialize(ByteBuffer byteBuffer) throws IOException;
+ void deserialize(final ByteBuffer byteBuffer) throws IOException;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java
index 8069c17c03728..eab8b64256047 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/sametype/numeric/AbstractSameTypeNumericOperator.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.sametype.numeric;
import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
@@ -28,6 +29,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Map;
/**
@@ -43,56 +45,78 @@ public abstract class AbstractSameTypeNumericOperator implements IntermediateRes
protected double doubleValue;
@Override
- public void configureSystemParameters(Map systemParams) {
+ public void configureSystemParameters(final Map systemParams) {
// Do nothing
}
@Override
- public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) {
return false;
}
@Override
- public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) {
outPutDataType = TSDataType.INT32;
return true;
}
@Override
- public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) {
+ return false;
+ }
+
+ @Override
+ public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) {
outPutDataType = TSDataType.INT64;
return true;
}
@Override
- public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) {
outPutDataType = TSDataType.FLOAT;
return true;
}
@Override
- public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) {
outPutDataType = TSDataType.DOUBLE;
return true;
}
@Override
- public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) {
+ return false;
+ }
+
+ @Override
+ public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) {
return false;
}
@Override
- public void updateValue(boolean input, long timestamp) {
+ public void updateValue(final boolean input, final long timestamp) {
throw new UnsupportedOperationException(
"AbstractSameTypeNumericOperator does not support boolean input");
}
@Override
- public void updateValue(String input, long timestamp) {
+ public void updateValue(final LocalDate input, final long timestamp) {
+ throw new UnsupportedOperationException(
+ "AbstractSameTypeNumericOperator does not support date input");
+ }
+
+ @Override
+ public void updateValue(final String input, final long timestamp) {
throw new UnsupportedOperationException(
"AbstractSameTypeNumericOperator does not support string input");
}
+ @Override
+ public void updateValue(final Binary input, final long timestamp) {
+ throw new UnsupportedOperationException(
+ "AbstractSameTypeNumericOperator does not support binary input");
+ }
+
@Override
public Pair getResult() {
switch (outPutDataType) {
@@ -110,7 +134,7 @@ public Pair getResult() {
}
@Override
- public void serialize(DataOutputStream outputStream) throws IOException {
+ public void serialize(final DataOutputStream outputStream) throws IOException {
outPutDataType.serializeTo(outputStream);
switch (outPutDataType) {
case INT32:
@@ -131,7 +155,7 @@ public void serialize(DataOutputStream outputStream) throws IOException {
}
@Override
- public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ public void deserialize(final ByteBuffer byteBuffer) throws IOException {
outPutDataType = TSDataType.deserializeFrom(byteBuffer);
switch (outPutDataType) {
case INT32:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java
index e80f05e455ada..a49d90c6ede7b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/doubletype/FractionPoweredSumOperator.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.doubletype;
import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
@@ -28,6 +29,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Map;
public class FractionPoweredSumOperator implements IntermediateResultOperator {
@@ -41,7 +43,7 @@ public class FractionPoweredSumOperator implements IntermediateResultOperator {
*
* @param power the power to use
*/
- public FractionPoweredSumOperator(double power) {
+ public FractionPoweredSumOperator(final double power) {
this.power = power;
}
@@ -51,88 +53,110 @@ public String getName() {
}
@Override
- public void configureSystemParameters(Map systemParams) {
+ public void configureSystemParameters(final Map systemParams) {
// Do nothing
}
@Override
- public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) {
return false;
}
@Override
- public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) {
sum = Math.pow(initialInput, power);
return true;
}
@Override
- public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) {
+ return false;
+ }
+
+ @Override
+ public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) {
sum = Math.pow(initialInput, power);
return true;
}
@Override
- public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) {
sum = Math.pow(initialInput, power);
return true;
}
@Override
- public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) {
sum = Math.pow(initialInput, power);
return true;
}
@Override
- public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) {
+ return false;
+ }
+
+ @Override
+ public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) {
return false;
}
@Override
- public void updateValue(boolean input, long timestamp) {
+ public void updateValue(final boolean input, final long timestamp) {
throw new UnsupportedOperationException(
"FractionPoweredSumOperator does not support boolean input");
}
@Override
- public void updateValue(int input, long timestamp) {
+ public void updateValue(final int input, final long timestamp) {
sum += Math.pow(input, power);
}
@Override
- public void updateValue(long input, long timestamp) {
+ public void updateValue(final LocalDate input, final long timestamp) {
+ throw new UnsupportedOperationException(
+ "FractionPoweredSumOperator does not support date input");
+ }
+
+ @Override
+ public void updateValue(final long input, final long timestamp) {
sum += Math.pow(input, power);
}
@Override
- public void updateValue(float input, long timestamp) {
+ public void updateValue(final float input, final long timestamp) {
sum += Math.pow(input, power);
}
@Override
- public void updateValue(double input, long timestamp) {
+ public void updateValue(final double input, final long timestamp) {
sum += Math.pow(input, power);
}
@Override
- public void updateValue(String input, long timestamp) {
+ public void updateValue(final String input, final long timestamp) {
throw new UnsupportedOperationException(
"FractionPoweredSumOperator does not support string input");
}
+ @Override
+ public void updateValue(final Binary initialInput, final long initialTimestamp) {
+ throw new UnsupportedOperationException(
+ "FractionPoweredSumOperator does not support binary input");
+ }
+
@Override
public Pair getResult() {
return new Pair<>(TSDataType.DOUBLE, sum);
}
@Override
- public void serialize(DataOutputStream outputStream) throws IOException {
+ public void serialize(final DataOutputStream outputStream) throws IOException {
ReadWriteIOUtils.write(sum, outputStream);
}
@Override
- public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ public void deserialize(final ByteBuffer byteBuffer) throws IOException {
sum = ReadWriteIOUtils.readDouble(byteBuffer);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java
index 4e5397fe6735f..a42dfa179425a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/operator/intermediateresult/specifictype/integertype/CountOperator.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.specifictype.integertype;
import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
@@ -28,6 +29,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Map;
public class CountOperator implements IntermediateResultOperator {
@@ -39,73 +41,95 @@ public String getName() {
}
@Override
- public void configureSystemParameters(Map systemParams) {
+ public void configureSystemParameters(final Map systemParams) {
// Do nothing
}
@Override
- public boolean initAndGetIsSupport(boolean initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final boolean initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public boolean initAndGetIsSupport(int initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final int initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public boolean initAndGetIsSupport(long initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final LocalDate initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public boolean initAndGetIsSupport(float initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final long initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public boolean initAndGetIsSupport(double initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final float initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public boolean initAndGetIsSupport(String initialInput, long initialTimestamp) {
+ public boolean initAndGetIsSupport(final double initialInput, final long initialTimestamp) {
count = 1;
return true;
}
@Override
- public void updateValue(boolean input, long timestamp) {
+ public boolean initAndGetIsSupport(final String initialInput, final long initialTimestamp) {
+ count = 1;
+ return true;
+ }
+
+ @Override
+ public boolean initAndGetIsSupport(final Binary initialInput, final long initialTimestamp) {
+ count = 1;
+ return true;
+ }
+
+ @Override
+ public void updateValue(final boolean input, final long timestamp) {
+ ++count;
+ }
+
+ @Override
+ public void updateValue(final int input, final long timestamp) {
+ ++count;
+ }
+
+ @Override
+ public void updateValue(LocalDate input, long timestamp) {
++count;
}
@Override
- public void updateValue(int input, long timestamp) {
+ public void updateValue(final long input, final long timestamp) {
++count;
}
@Override
- public void updateValue(long input, long timestamp) {
+ public void updateValue(final float input, final long timestamp) {
++count;
}
@Override
- public void updateValue(float input, long timestamp) {
+ public void updateValue(final double input, final long timestamp) {
++count;
}
@Override
- public void updateValue(double input, long timestamp) {
+ public void updateValue(final String input, final long timestamp) {
++count;
}
@Override
- public void updateValue(String input, long timestamp) {
+ public void updateValue(Binary input, long timestamp) {
++count;
}
@@ -115,12 +139,12 @@ public Pair getResult() {
}
@Override
- public void serialize(DataOutputStream outputStream) throws IOException {
+ public void serialize(final DataOutputStream outputStream) throws IOException {
ReadWriteIOUtils.write(count, outputStream);
}
@Override
- public void deserialize(ByteBuffer byteBuffer) throws IOException {
+ public void deserialize(final ByteBuffer byteBuffer) throws IOException {
count = ReadWriteIOUtils.readInt(byteBuffer);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java
index 511d9a61fb00e..6cd046fb594fc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/datastructure/TimeSeriesWindow.java
@@ -23,6 +23,7 @@
import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.CustomizedReadableIntermediateResults;
import org.apache.iotdb.db.pipe.processor.aggregate.operator.intermediateresult.IntermediateResultOperator;
import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.AbstractWindowingProcessor;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Pair;
@@ -33,6 +34,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -213,6 +215,61 @@ public Pair updateIntermediateResult(
return state.isEmit() ? stateOutputPair : null;
}
+ public Pair updateIntermediateResult(
+ final long timestamp, final LocalDate value) {
+ final Pair stateOutputPair =
+ processor.updateAndMaySetWindowState(this, timestamp, value);
+ final WindowState state = stateOutputPair.getLeft();
+
+ if (state.isEmitWithoutCompute()) {
+ stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.DATE));
+ }
+
+ if (state.isCalculate()) {
+ final Iterator>> iterator =
+ intermediateResultName2tsTypeAndOperatorMap.entrySet().iterator();
+ Map.Entry> entry;
+
+ while (iterator.hasNext()) {
+ entry = iterator.next();
+ final IntermediateResultOperator operator = entry.getValue().getRight();
+ if (entry.getValue().getLeft() == TSDataType.UNKNOWN) {
+ if (!operator.initAndGetIsSupport(value, timestamp)) {
+ // Remove unsupported aggregated results
+ aggregatedOutputName2OperatorMap
+ .entrySet()
+ .removeIf(
+ entry1 ->
+ entry1
+ .getValue()
+ .getDeclaredIntermediateValueNames()
+ .contains(operator.getName()));
+ // If no aggregated values can be calculated, purge the window
+ if (aggregatedOutputName2OperatorMap.isEmpty()) {
+ return new Pair<>(WindowState.PURGE, null);
+ }
+ // Remove unsupported intermediate values
+ iterator.remove();
+ continue;
+ }
+ entry.getValue().setLeft(TSDataType.DATE);
+ } else if (entry.getValue().getLeft() != TSDataType.DATE) {
+ LOGGER.warn(
+ "Different data type encountered in one window, will purge. Previous type: {}, now type: {}",
+ entry.getValue().getLeft(),
+ TSDataType.DATE);
+ return new Pair<>(WindowState.PURGE, null);
+ } else {
+ operator.updateValue(value, timestamp);
+ }
+ }
+ }
+ if (state.isEmitWithCompute()) {
+ stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.DATE));
+ }
+ return state.isEmit() ? stateOutputPair : null;
+ }
+
public Pair updateIntermediateResult(
final long timestamp, final long value) {
final Pair stateOutputPair =
@@ -431,6 +488,60 @@ public Pair updateIntermediateResult(
return state.isEmit() ? stateOutputPair : null;
}
+ public Pair updateIntermediateResult(
+ final long timestamp, final Binary value) {
+ final Pair stateOutputPair =
+ processor.updateAndMaySetWindowState(this, timestamp, value);
+ final WindowState state = stateOutputPair.getLeft();
+
+ if (state.isEmitWithoutCompute()) {
+ stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.BLOB));
+ }
+ if (state.isCalculate()) {
+ final Iterator>> iterator =
+ intermediateResultName2tsTypeAndOperatorMap.entrySet().iterator();
+ Map.Entry> entry;
+
+ while (iterator.hasNext()) {
+ entry = iterator.next();
+ final IntermediateResultOperator operator = entry.getValue().getRight();
+ if (entry.getValue().getLeft() == TSDataType.UNKNOWN) {
+ if (!operator.initAndGetIsSupport(value, timestamp)) {
+ // Remove unsupported aggregated results
+ aggregatedOutputName2OperatorMap
+ .entrySet()
+ .removeIf(
+ entry1 ->
+ entry1
+ .getValue()
+ .getDeclaredIntermediateValueNames()
+ .contains(operator.getName()));
+ // If no aggregated values can be calculated, purge the window
+ if (aggregatedOutputName2OperatorMap.isEmpty()) {
+ return new Pair<>(WindowState.PURGE, null);
+ }
+ // Remove unsupported intermediate values
+ iterator.remove();
+ continue;
+ }
+ entry.getValue().setLeft(TSDataType.BLOB);
+ } else if (entry.getValue().getLeft() != TSDataType.BLOB) {
+ LOGGER.warn(
+ "Different data type encountered in one window, will purge. Previous type: {}, now type: {}",
+ entry.getValue().getLeft(),
+ TSDataType.BLOB);
+ return new Pair<>(WindowState.PURGE, null);
+ } else {
+ operator.updateValue(value, timestamp);
+ }
+ }
+ }
+ if (state.isEmitWithCompute()) {
+ stateOutputPair.getRight().setAggregatedResults(getAggregatedResults(TSDataType.BLOB));
+ }
+ return state.isEmit() ? stateOutputPair : null;
+ }
+
public WindowOutput forceOutput() {
return processor
.forceOutput(this)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java
index b74cbb09181fe..c7acd0441c84f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractSimpleTimeWindowingProcessor.java
@@ -22,82 +22,106 @@
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow;
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput;
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.utils.Pair;
+import java.time.LocalDate;
import java.util.List;
import java.util.Set;
public abstract class AbstractSimpleTimeWindowingProcessor extends AbstractWindowingProcessor {
public Set mayAddWindow(
- List windowList, long timeStamp, boolean value) {
+ final List windowList, final long timeStamp, final boolean value) {
return mayAddWindow(windowList, timeStamp);
}
public Set mayAddWindow(
- List windowList, long timeStamp, int value) {
+ final List windowList, final long timeStamp, final int value) {
return mayAddWindow(windowList, timeStamp);
}
public Set mayAddWindow(
- List windowList, long timeStamp, long value) {
+ final List windowList, final long timeStamp, final LocalDate value) {
return mayAddWindow(windowList, timeStamp);
}
public Set mayAddWindow(
- List windowList, long timeStamp, float value) {
+ final List windowList, final long timeStamp, final long value) {
return mayAddWindow(windowList, timeStamp);
}
public Set mayAddWindow(
- List windowList, long timeStamp, double value) {
+ final List windowList, final long timeStamp, final float value) {
return mayAddWindow(windowList, timeStamp);
}
public Set mayAddWindow(
- List windowList, long timeStamp, String value) {
+ final List windowList, final long timeStamp, final double value) {
+ return mayAddWindow(windowList, timeStamp);
+ }
+
+ public Set mayAddWindow(
+ final List windowList, final long timeStamp, final String value) {
+ return mayAddWindow(windowList, timeStamp);
+ }
+
+ public Set mayAddWindow(
+ final List windowList, final long timeStamp, final Binary value) {
return mayAddWindow(windowList, timeStamp);
}
public abstract Set mayAddWindow(
- List windowList, long timeStamp);
+ final List windowList, final long timeStamp);
+
+ @Override
+ public Pair updateAndMaySetWindowState(
+ final TimeSeriesWindow window, final long timeStamp, final boolean value) {
+ return updateAndMaySetWindowState(window, timeStamp);
+ }
+
+ @Override
+ public Pair updateAndMaySetWindowState(
+ final TimeSeriesWindow window, final long timeStamp, final int value) {
+ return updateAndMaySetWindowState(window, timeStamp);
+ }
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, boolean value) {
+ final TimeSeriesWindow window, final long timeStamp, final LocalDate value) {
return updateAndMaySetWindowState(window, timeStamp);
}
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, int value) {
+ final TimeSeriesWindow window, final long timeStamp, final long value) {
return updateAndMaySetWindowState(window, timeStamp);
}
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, long value) {
+ final TimeSeriesWindow window, final long timeStamp, final float value) {
return updateAndMaySetWindowState(window, timeStamp);
}
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, float value) {
+ final TimeSeriesWindow window, final long timeStamp, final double value) {
return updateAndMaySetWindowState(window, timeStamp);
}
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, double value) {
+ final TimeSeriesWindow window, final long timeStamp, final String value) {
return updateAndMaySetWindowState(window, timeStamp);
}
@Override
public Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, String value) {
+ final TimeSeriesWindow window, final long timeStamp, final Binary value) {
return updateAndMaySetWindowState(window, timeStamp);
}
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp);
+ final TimeSeriesWindow window, final long timeStamp);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java
index 7f8d1ba70a5af..384db345b8592 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/AbstractWindowingProcessor.java
@@ -24,11 +24,13 @@
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow;
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput;
import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState;
+import org.apache.iotdb.pipe.api.type.Binary;
import org.apache.tsfile.utils.Pair;
import java.io.DataOutputStream;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.List;
import java.util.Set;
@@ -47,22 +49,28 @@ public abstract class AbstractWindowingProcessor extends AbstractFormalProcessor
* @return The added windows
*/
public abstract Set mayAddWindow(
- List windowList, long timeStamp, boolean value);
+ final List windowList, final long timeStamp, final boolean value);
public abstract Set mayAddWindow(
- List windowList, long timeStamp, int value);
+ final List windowList, final long timeStamp, final int value);
public abstract Set mayAddWindow(
- List windowList, long timeStamp, long value);
+ final List windowList, final long timeStamp, final LocalDate value);
public abstract Set mayAddWindow(
- List windowList, long timeStamp, float value);
+ final List windowList, final long timeStamp, final long value);
public abstract Set mayAddWindow(
- List windowList, long timeStamp, double value);
+ final List windowList, final long timeStamp, final float value);
public abstract Set mayAddWindow(
- List windowList, long timeStamp, String value);
+ final List windowList, final long timeStamp, final double value);
+
+ public abstract Set mayAddWindow(
+ final List windowList, final long timeStamp, final String value);
+
+ public abstract Set mayAddWindow(
+ final List windowList, final long timeStamp, final Binary value);
/**
* The {@link AbstractWindowingProcessor} may decide whether a {@link TimeSeriesWindow} shall be
@@ -76,24 +84,30 @@ public abstract Set mayAddWindow(
* restarted.
*/
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, boolean value);
+ final TimeSeriesWindow window, final long timeStamp, final boolean value);
+
+ public abstract Pair updateAndMaySetWindowState(
+ final TimeSeriesWindow window, final long timeStamp, final int value);
+
+ public abstract Pair updateAndMaySetWindowState(
+ final TimeSeriesWindow window, final long timeStamp, final LocalDate value);
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, int value);
+ final TimeSeriesWindow window, final long timeStamp, final long value);
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, long value);
+ final TimeSeriesWindow window, final long timeStamp, final float value);
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, float value);
+ final TimeSeriesWindow window, final long timeStamp, final double value);
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, double value);
+ final TimeSeriesWindow window, final long timeStamp, final String value);
public abstract Pair updateAndMaySetWindowState(
- TimeSeriesWindow window, long timeStamp, String value);
+ final TimeSeriesWindow window, final long timeStamp, final Binary value);
- public abstract WindowOutput forceOutput(TimeSeriesWindow window);
+ public abstract WindowOutput forceOutput(final TimeSeriesWindow window);
/**
* Serialize the customized attributes in a window to an output stream
@@ -102,7 +116,7 @@ public abstract Pair updateAndMaySetWindowState(
* @param outputStream the outputStream
*/
public void serializeCustomizedAttributes(
- TimeSeriesWindow window, DataOutputStream outputStream) {
+ final TimeSeriesWindow window, final DataOutputStream outputStream) {
// Do nothing by default
}
@@ -111,7 +125,8 @@ public void serializeCustomizedAttributes(
*
* @param byteBuffer the customized attributes in one window
*/
- public void deserializeCustomizedAttributes(TimeSeriesWindow window, ByteBuffer byteBuffer) {
+ public void deserializeCustomizedAttributes(
+ final TimeSeriesWindow window, final ByteBuffer byteBuffer) {
// Do nothing by default
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index a5dd5fbf511f2..b38997ef9c0c4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -280,10 +280,12 @@ public Object getFirstValueOfIndex(int index) {
Object value;
switch (dataTypes[index]) {
case INT32:
+ case DATE:
int[] intValues = (int[]) columns[index];
value = intValues[0];
break;
case INT64:
+ case TIMESTAMP:
long[] longValues = (long[]) columns[index];
value = longValues[0];
break;
@@ -300,6 +302,8 @@ public Object getFirstValueOfIndex(int index) {
value = boolValues[0];
break;
case TEXT:
+ case BLOB:
+ case STRING:
Binary[] binaryValues = (Binary[]) columns[index];
value = binaryValues[0];
break;
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
index 0d8e5718c9eb2..8e69473b95c6e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
@@ -38,9 +38,11 @@
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;
@@ -51,6 +53,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -61,9 +64,9 @@ public class PipeDataNodeThriftRequestTest {
@Test
public void testPipeTransferDataNodeHandshakeReq() throws IOException {
- PipeTransferDataNodeHandshakeV1Req req =
+ final PipeTransferDataNodeHandshakeV1Req req =
PipeTransferDataNodeHandshakeV1Req.toTPipeTransferReq(TIME_PRECISION);
- PipeTransferDataNodeHandshakeV1Req deserializeReq =
+ final PipeTransferDataNodeHandshakeV1Req deserializeReq =
PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -75,7 +78,7 @@ public void testPipeTransferDataNodeHandshakeReq() throws IOException {
@Test
public void testPipeTransferInsertNodeReq() {
- PipeTransferTabletInsertNodeReq req =
+ final PipeTransferTabletInsertNodeReq req =
PipeTransferTabletInsertNodeReq.toTPipeTransferReq(
new InsertRowNode(
new PlanNodeId(""),
@@ -86,7 +89,7 @@ public void testPipeTransferInsertNodeReq() {
1,
new Object[] {1},
false));
- PipeTransferTabletInsertNodeReq deserializeReq =
+ final PipeTransferTabletInsertNodeReq deserializeReq =
PipeTransferTabletInsertNodeReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -95,8 +98,8 @@ public void testPipeTransferInsertNodeReq() {
Assert.assertEquals(req.getInsertNode(), deserializeReq.getInsertNode());
- Statement statement = req.constructStatement();
- List paths = new ArrayList<>();
+ final Statement statement = req.constructStatement();
+ final List paths = new ArrayList<>();
paths.add(new PartialPath(new String[] {"root", "sg", "d", "s"}));
Assert.assertEquals(statement.getPaths(), paths);
}
@@ -104,9 +107,9 @@ public void testPipeTransferInsertNodeReq() {
@Test
public void testPipeTransferTabletBinaryReq() {
// Not do real test here since "serializeToWal" needs private inner class of walBuffer
- PipeTransferTabletBinaryReq req =
+ final PipeTransferTabletBinaryReq req =
PipeTransferTabletBinaryReq.toTPipeTransferReq(ByteBuffer.wrap(new byte[] {'a', 'b'}));
- PipeTransferTabletBinaryReq deserializeReq =
+ final PipeTransferTabletBinaryReq deserializeReq =
PipeTransferTabletBinaryReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -116,7 +119,7 @@ public void testPipeTransferTabletBinaryReq() {
@Test
public void testPipeTransferSchemaPlanReq() {
- PipeTransferPlanNodeReq req =
+ final PipeTransferPlanNodeReq req =
PipeTransferPlanNodeReq.toTPipeTransferReq(
new CreateAlignedTimeSeriesNode(
new PlanNodeId(""),
@@ -129,7 +132,8 @@ public void testPipeTransferSchemaPlanReq() {
null,
null));
- PipeTransferPlanNodeReq deserializeReq = PipeTransferPlanNodeReq.fromTPipeTransferReq(req);
+ final PipeTransferPlanNodeReq deserializeReq =
+ PipeTransferPlanNodeReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
@@ -141,29 +145,40 @@ public void testPipeTransferSchemaPlanReq() {
@Test
public void testPipeTransferTabletReq() {
try {
- List schemaList = new ArrayList<>();
+ final List schemaList = new ArrayList<>();
schemaList.add(new MeasurementSchema("s1", TSDataType.INT32));
schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
schemaList.add(new MeasurementSchema("s3", TSDataType.FLOAT));
schemaList.add(new MeasurementSchema("s4", TSDataType.DOUBLE));
schemaList.add(new MeasurementSchema("s5", TSDataType.BOOLEAN));
schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT));
- Tablet t = new Tablet("root.sg.d", schemaList, 1024);
+ schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP));
+ schemaList.add(new MeasurementSchema("s8", TSDataType.DATE));
+ schemaList.add(new MeasurementSchema("s9", TSDataType.BLOB));
+ schemaList.add(new MeasurementSchema("s10", TSDataType.STRING));
+ final Tablet t = new Tablet("root.sg.d", schemaList, 1024);
t.rowSize = 2;
t.addTimestamp(0, 2000);
t.addTimestamp(1, 1000);
t.addValue("s1", 0, 2);
t.addValue("s6", 0, "2");
+ t.addValue("s8", 0, LocalDate.of(2024, 2, 2));
+ t.addValue("s9", 0, new Binary("2", TSFileConfig.STRING_CHARSET));
+ t.addValue("s10", 0, "2");
t.addValue("s1", 1, 1);
t.addValue("s6", 1, "1");
- PipeTransferTabletRawReq req = PipeTransferTabletRawReq.toTPipeTransferReq(t, false);
- PipeTransferTabletRawReq deserializeReq = PipeTransferTabletRawReq.fromTPipeTransferReq(req);
+ t.addValue("s8", 1, LocalDate.of(2024, 1, 1));
+ t.addValue("s9", 1, new Binary("1", TSFileConfig.STRING_CHARSET));
+ t.addValue("s10", 1, "1");
+ final PipeTransferTabletRawReq req = PipeTransferTabletRawReq.toTPipeTransferReq(t, false);
+ final PipeTransferTabletRawReq deserializeReq =
+ PipeTransferTabletRawReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
- Statement statement =
+ final Statement statement =
req.constructStatement(); // will call PipeTransferTabletRawReq.sortTablet() here
List paths = new ArrayList<>();
paths.add(new PartialPath(new String[] {"root", "sg", "d", "s1"}));
@@ -172,8 +187,12 @@ public void testPipeTransferTabletReq() {
paths.add(new PartialPath(new String[] {"root", "sg", "d", "s4"}));
paths.add(new PartialPath(new String[] {"root", "sg", "d", "s5"}));
paths.add(new PartialPath(new String[] {"root", "sg", "d", "s6"}));
+ paths.add(new PartialPath(new String[] {"root", "sg", "d", "s7"}));
+ paths.add(new PartialPath(new String[] {"root", "sg", "d", "s8"}));
+ paths.add(new PartialPath(new String[] {"root", "sg", "d", "s9"}));
+ paths.add(new PartialPath(new String[] {"root", "sg", "d", "s10"}));
Assert.assertEquals(statement.getPaths(), paths);
- } catch (IOException e) {
+ } catch (final IOException e) {
Assert.fail();
}
}
@@ -184,7 +203,7 @@ public void testPipeTransferTabletBatchReq() throws IOException {
final List insertNodeBuffers = new ArrayList<>();
final List tabletBuffers = new ArrayList<>();
- InsertRowNode node =
+ final InsertRowNode node =
new InsertRowNode(
new PlanNodeId(""),
new PartialPath(new String[] {"root", "sg", "d"}),
@@ -203,21 +222,32 @@ public void testPipeTransferTabletBatchReq() throws IOException {
binaryBuffers.add(ByteBuffer.wrap(new byte[] {'a', 'b'}));
// Raw buffer
- List schemaList = new ArrayList<>();
+ final List schemaList = new ArrayList<>();
schemaList.add(new MeasurementSchema("s1", TSDataType.INT32));
schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
schemaList.add(new MeasurementSchema("s3", TSDataType.FLOAT));
schemaList.add(new MeasurementSchema("s4", TSDataType.DOUBLE));
schemaList.add(new MeasurementSchema("s5", TSDataType.BOOLEAN));
schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT));
- Tablet t = new Tablet("root.sg.d", schemaList, 1024);
+ schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP));
+ schemaList.add(new MeasurementSchema("s8", TSDataType.DATE));
+ schemaList.add(new MeasurementSchema("s9", TSDataType.BLOB));
+ schemaList.add(new MeasurementSchema("s10", TSDataType.STRING));
+
+ final Tablet t = new Tablet("root.sg.d", schemaList, 1024);
t.rowSize = 2;
t.addTimestamp(0, 2000);
t.addTimestamp(1, 1000);
t.addValue("s1", 0, 2);
t.addValue("s6", 0, "2");
+ t.addValue("s8", 0, LocalDate.of(2024, 2, 2));
+ t.addValue("s9", 0, new Binary("2", TSFileConfig.STRING_CHARSET));
+ t.addValue("s10", 0, "2");
t.addValue("s1", 1, 1);
t.addValue("s6", 1, "1");
+ t.addValue("s8", 1, LocalDate.of(2024, 1, 1));
+ t.addValue("s9", 1, new Binary("1", TSFileConfig.STRING_CHARSET));
+ t.addValue("s10", 1, "1");
try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
@@ -227,11 +257,11 @@ public void testPipeTransferTabletBatchReq() throws IOException {
ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()));
}
- PipeTransferTabletBatchReq req =
+ final PipeTransferTabletBatchReq req =
PipeTransferTabletBatchReq.toTPipeTransferReq(
binaryBuffers, insertNodeBuffers, tabletBuffers);
- PipeTransferTabletBatchReq deserializedReq =
+ final PipeTransferTabletBatchReq deserializedReq =
PipeTransferTabletBatchReq.fromTPipeTransferReq(req);
Assert.assertArrayEquals(
@@ -243,12 +273,12 @@ public void testPipeTransferTabletBatchReq() throws IOException {
@Test
public void testPipeTransferFilePieceReq() throws IOException {
- byte[] body = "testPipeTransferFilePieceReq".getBytes();
- String fileName = "1.tsfile";
+ final byte[] body = "testPipeTransferFilePieceReq".getBytes();
+ final String fileName = "1.tsfile";
- PipeTransferTsFilePieceReq req =
+ final PipeTransferTsFilePieceReq req =
PipeTransferTsFilePieceReq.toTPipeTransferReq(fileName, 0, body);
- PipeTransferTsFilePieceReq deserializeReq =
+ final PipeTransferTsFilePieceReq deserializeReq =
PipeTransferTsFilePieceReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -262,12 +292,12 @@ public void testPipeTransferFilePieceReq() throws IOException {
@Test
public void testPipeTransferFilePieceWithModReq() throws IOException {
- byte[] body = "testPipeTransferFilePieceWithModReq".getBytes();
- String fileName = "1.tsfile.mod";
+ final byte[] body = "testPipeTransferFilePieceWithModReq".getBytes();
+ final String fileName = "1.tsfile.mod";
- PipeTransferTsFilePieceWithModReq req =
+ final PipeTransferTsFilePieceWithModReq req =
PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, 0, body);
- PipeTransferTsFilePieceWithModReq deserializeReq =
+ final PipeTransferTsFilePieceWithModReq deserializeReq =
PipeTransferTsFilePieceWithModReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -281,12 +311,12 @@ public void testPipeTransferFilePieceWithModReq() throws IOException {
@Test
public void testPipeTransferSchemaSnapshotPieceReq() throws IOException {
- byte[] body = "testPipeTransferSchemaSnapshotPieceReq".getBytes();
- String fileName = "1.temp";
+ final byte[] body = "testPipeTransferSchemaSnapshotPieceReq".getBytes();
+ final String fileName = "1.temp";
- PipeTransferSchemaSnapshotPieceReq req =
+ final PipeTransferSchemaSnapshotPieceReq req =
PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(fileName, 0, body);
- PipeTransferSchemaSnapshotPieceReq deserializeReq =
+ final PipeTransferSchemaSnapshotPieceReq deserializeReq =
PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -300,10 +330,12 @@ public void testPipeTransferSchemaSnapshotPieceReq() throws IOException {
@Test
public void testPipeTransferTsFileSealReq() throws IOException {
- String fileName = "1.tsfile";
+ final String fileName = "1.tsfile";
- PipeTransferTsFileSealReq req = PipeTransferTsFileSealReq.toTPipeTransferReq(fileName, 100);
- PipeTransferTsFileSealReq deserializeReq = PipeTransferTsFileSealReq.fromTPipeTransferReq(req);
+ final PipeTransferTsFileSealReq req =
+ PipeTransferTsFileSealReq.toTPipeTransferReq(fileName, 100);
+ final PipeTransferTsFileSealReq deserializeReq =
+ PipeTransferTsFileSealReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
@@ -315,16 +347,16 @@ public void testPipeTransferTsFileSealReq() throws IOException {
@Test
public void testPipeTransferSchemaSnapshotSealReq() throws IOException {
- String mTreeSnapshotName = "mtree.snapshot";
- String tLogName = "tlog.txt";
- String databaseName = "root.db";
+ final String mTreeSnapshotName = "mtree.snapshot";
+ final String tLogName = "tlog.txt";
+ final String databaseName = "root.db";
// CREATE_TIME_SERIES
- String typeString = "19";
+ final String typeString = "19";
- PipeTransferSchemaSnapshotSealReq req =
+ final PipeTransferSchemaSnapshotSealReq req =
PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq(
"root.**", mTreeSnapshotName, 100, tLogName, 10, databaseName, typeString);
- PipeTransferSchemaSnapshotSealReq deserializeReq =
+ final PipeTransferSchemaSnapshotSealReq deserializeReq =
PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
@@ -338,9 +370,9 @@ public void testPipeTransferSchemaSnapshotSealReq() throws IOException {
@Test
public void testPipeTransferFilePieceResp() throws IOException {
- PipeTransferFilePieceResp resp =
+ final PipeTransferFilePieceResp resp =
PipeTransferFilePieceResp.toTPipeTransferResp(RpcUtils.SUCCESS_STATUS, 100);
- PipeTransferFilePieceResp deserializeResp =
+ final PipeTransferFilePieceResp deserializeResp =
PipeTransferFilePieceResp.fromTPipeTransferResp(resp);
Assert.assertEquals(resp.getStatus(), deserializeResp.getStatus());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
index ee05f26fb82be..5ba92b46a4e7c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
@@ -50,6 +50,10 @@ public void createDataTypeList() {
tsDataTypeList.add(TSDataType.DOUBLE);
tsDataTypeList.add(TSDataType.BOOLEAN);
tsDataTypeList.add(TSDataType.TEXT);
+ tsDataTypeList.add(TSDataType.TIMESTAMP);
+ tsDataTypeList.add(TSDataType.DATE);
+ tsDataTypeList.add(TSDataType.BLOB);
+ tsDataTypeList.add(TSDataType.STRING);
tsDataTypeList.add(null);
nullTypeList = null;
@@ -60,14 +64,19 @@ public void createDataTypeList() {
typeList.add(Type.DOUBLE);
typeList.add(Type.BOOLEAN);
typeList.add(Type.TEXT);
+ typeList.add(Type.INT64);
+ typeList.add(Type.INT32);
+ typeList.add(Type.TEXT);
+ typeList.add(Type.TEXT);
typeList.add(null);
}
@Test
public void testDataTypeTransformer() {
- List nullResultList =
+ final List nullResultList =
PipeDataTypeTransformer.transformToPipeDataTypeList(nullTsDataTypeList);
- List resultList = PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypeList);
+ final List resultList =
+ PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypeList);
Assert.assertEquals(nullResultList, nullTypeList);
Assert.assertEquals(resultList, typeList);
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
index 8c4d9ce691746..44fa83cca6235 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
@@ -38,6 +38,7 @@
import org.junit.Before;
import org.junit.Test;
+import java.time.LocalDate;
import java.util.Arrays;
public class PipeTabletInsertionEventTest {
@@ -49,7 +50,8 @@ public class PipeTabletInsertionEventTest {
final String deviceId = "root.sg.d1";
final long[] times = new long[] {110L, 111L, 112L, 113L, 114L};
- final String[] measurementIds = new String[] {"s1", "s2", "s3", "s4", "s5", "s6"};
+ final String[] measurementIds =
+ new String[] {"s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10"};
final TSDataType[] dataTypes =
new TSDataType[] {
TSDataType.INT32,
@@ -57,10 +59,14 @@ public class PipeTabletInsertionEventTest {
TSDataType.FLOAT,
TSDataType.DOUBLE,
TSDataType.BOOLEAN,
- TSDataType.TEXT
+ TSDataType.TEXT,
+ TSDataType.TIMESTAMP,
+ TSDataType.DATE,
+ TSDataType.BLOB,
+ TSDataType.STRING,
};
- final MeasurementSchema[] schemas = new MeasurementSchema[6];
+ final MeasurementSchema[] schemas = new MeasurementSchema[10];
final String pattern = "root.sg.d1";
@@ -90,6 +96,10 @@ private void createInsertRowNode() throws IllegalPathException {
values[3] = 1D;
values[4] = false;
values[5] = BytesUtils.valueOf("text");
+ values[6] = 20000L;
+ values[7] = LocalDate.of(2024, 1, 1);
+ values[8] = BytesUtils.valueOf("blob");
+ values[9] = BytesUtils.valueOf("string");
insertRowNode =
new InsertRowNode(
@@ -125,6 +135,10 @@ private void createInsertTabletNode() throws IllegalPathException {
values[3] = new double[times.length];
values[4] = new boolean[times.length];
values[5] = new Binary[times.length];
+ values[6] = new long[times.length];
+ values[7] = new LocalDate[times.length];
+ values[8] = new Binary[times.length];
+ values[9] = new Binary[times.length];
for (int r = 0; r < times.length; r++) {
((int[]) values[0])[r] = 100;
@@ -133,6 +147,10 @@ private void createInsertTabletNode() throws IllegalPathException {
((double[]) values[3])[r] = 1D;
((boolean[]) values[4])[r] = false;
((Binary[]) values[5])[r] = BytesUtils.valueOf("text");
+ ((long[]) values[6])[r] = 20000L;
+ ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1);
+ ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob");
+ ((Binary[]) values[9])[r] = BytesUtils.valueOf("string");
}
this.insertTabletNode =
@@ -177,6 +195,10 @@ private void createTablet() {
values[3] = new double[1];
values[4] = new boolean[1];
values[5] = new Binary[1];
+ values[6] = new long[times.length];
+ values[7] = new LocalDate[times.length];
+ values[8] = new Binary[times.length];
+ values[9] = new Binary[times.length];
for (int r = 0; r < 1; r++) {
((int[]) values[0])[r] = 100;
@@ -185,6 +207,10 @@ private void createTablet() {
((double[]) values[3])[r] = 1D;
((boolean[]) values[4])[r] = false;
((Binary[]) values[5])[r] = BytesUtils.valueOf("text");
+ ((long[]) values[6])[r] = 20000L;
+ ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1);
+ ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob");
+ ((Binary[]) values[9])[r] = BytesUtils.valueOf("string");
}
tabletForInsertRowNode = new Tablet(deviceId, Arrays.asList(schemas), 1);
@@ -205,6 +231,10 @@ private void createTablet() {
values[3] = new double[times.length];
values[4] = new boolean[times.length];
values[5] = new Binary[times.length];
+ values[6] = new long[times.length];
+ values[7] = new LocalDate[times.length];
+ values[8] = new Binary[times.length];
+ values[9] = new Binary[times.length];
for (int r = 0; r < times.length; r++) {
((int[]) values[0])[r] = 100;
@@ -213,6 +243,10 @@ private void createTablet() {
((double[]) values[3])[r] = 1D;
((boolean[]) values[4])[r] = false;
((Binary[]) values[5])[r] = BytesUtils.valueOf("text");
+ ((long[]) values[6])[r] = 20000L;
+ ((LocalDate[]) values[7])[r] = LocalDate.of(2024, 1, 1);
+ ((Binary[]) values[8])[r] = BytesUtils.valueOf("blob");
+ ((Binary[]) values[9])[r] = BytesUtils.valueOf("string");
}
tabletForInsertTabletNode = new Tablet(deviceId, Arrays.asList(schemas), times.length);
From e65335a8557bcf0688fabbf7b1ca1eb58f5d6e68 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 15:32:16 +0800
Subject: [PATCH 2/6] load module
---
.../iotdb/it/utils/TsFileGenerator.java | 103 +++--
...adTsFileIT.java => IoTDBLoadTsFileIT.java} | 394 +++++++++++-------
.../execution/load/AlignedChunkData.java | 124 +++---
.../execution/load/NonAlignedChunkData.java | 61 +--
4 files changed, 421 insertions(+), 261 deletions(-)
rename integration-test/src/test/java/org/apache/iotdb/db/it/{IOTDBLoadTsFileIT.java => IoTDBLoadTsFileIT.java} (65%)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java
index 31f6fecdd0738..d30c0c36d5048 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java
@@ -37,6 +37,7 @@
import java.io.File;
import java.io.IOException;
+import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -52,7 +53,7 @@ public class TsFileGenerator implements AutoCloseable {
private final Map> device2MeasurementSchema;
private Random random;
- public TsFileGenerator(File tsFile) throws IOException {
+ public TsFileGenerator(final File tsFile) throws IOException {
this.tsFile = tsFile;
this.writer = new TsFileWriter(tsFile);
this.device2TimeSet = new HashMap<>();
@@ -64,11 +65,12 @@ public void resetRandom() {
random = new Random();
}
- public void resetRandom(long seed) {
+ public void resetRandom(final long seed) {
random = new Random(seed);
}
- public void registerTimeseries(String path, List measurementSchemaList) {
+ public void registerTimeseries(
+ final String path, final List measurementSchemaList) {
if (device2MeasurementSchema.containsKey(path)) {
LOGGER.error("Register same device {}.", path);
return;
@@ -78,7 +80,8 @@ public void registerTimeseries(String path, List measurementS
device2MeasurementSchema.put(path, measurementSchemaList);
}
- public void registerAlignedTimeseries(String path, List measurementSchemaList)
+ public void registerAlignedTimeseries(
+ final String path, final List measurementSchemaList)
throws WriteProcessException {
if (device2MeasurementSchema.containsKey(path)) {
LOGGER.error("Register same device {}.", path);
@@ -89,14 +92,15 @@ public void registerAlignedTimeseries(String path, List measu
device2MeasurementSchema.put(path, measurementSchemaList);
}
- public void generateData(String device, int number, long timeGap, boolean isAligned)
+ public void generateData(
+ final String device, final int number, final long timeGap, final boolean isAligned)
throws IOException, WriteProcessException {
- List schemas = device2MeasurementSchema.get(device);
- TreeSet timeSet = device2TimeSet.get(device);
- Tablet tablet = new Tablet(device, schemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
- long sensorNum = schemas.size();
+ final List schemas = device2MeasurementSchema.get(device);
+ final TreeSet timeSet = device2TimeSet.get(device);
+ final Tablet tablet = new Tablet(device, schemas);
+ final long[] timestamps = tablet.timestamps;
+ final Object[] values = tablet.values;
+ final long sensorNum = schemas.size();
long startTime = timeSet.isEmpty() ? 0L : timeSet.last();
for (long r = 0; r < number; r++) {
@@ -131,18 +135,22 @@ public void generateData(String device, int number, long timeGap, boolean isAlig
}
public void generateData(
- String device, int number, long timeGap, boolean isAligned, long startTimestamp)
+ final String device,
+ final int number,
+ final long timeGap,
+ final boolean isAligned,
+ final long startTimestamp)
throws IOException, WriteProcessException {
- List schemas = device2MeasurementSchema.get(device);
- TreeSet timeSet = device2TimeSet.get(device);
- Tablet tablet = new Tablet(device, schemas);
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
- long sensorNum = schemas.size();
+ final List schemas = device2MeasurementSchema.get(device);
+ final TreeSet timeSet = device2TimeSet.get(device);
+ final Tablet tablet = new Tablet(device, schemas);
+ final long[] timestamps = tablet.timestamps;
+ final Object[] values = tablet.values;
+ final long sensorNum = schemas.size();
long startTime = startTimestamp;
for (long r = 0; r < number; r++) {
- int row = tablet.rowSize++;
+ final int row = tablet.rowSize++;
startTime += timeGap;
timestamps[row] = startTime;
timeSet.add(startTime);
@@ -172,12 +180,16 @@ public void generateData(
LOGGER.info("Write {} points into device {}", number, device);
}
- private void generateDataPoint(Object obj, int row, MeasurementSchema schema) {
+ private void generateDataPoint(final Object obj, final int row, final MeasurementSchema schema) {
switch (schema.getType()) {
case INT32:
generateINT32(obj, row);
break;
+ case DATE:
+ generateDATE(obj, row);
+ break;
case INT64:
+ case TIMESTAMP:
generateINT64(obj, row);
break;
case FLOAT:
@@ -190,6 +202,8 @@ private void generateDataPoint(Object obj, int row, MeasurementSchema schema) {
generateBOOLEAN(obj, row);
break;
case TEXT:
+ case BLOB:
+ case STRING:
generateTEXT(obj, row);
break;
default:
@@ -197,53 +211,60 @@ private void generateDataPoint(Object obj, int row, MeasurementSchema schema) {
}
}
- private void generateINT32(Object obj, int row) {
- int[] ints = (int[]) obj;
+ private void generateINT32(final Object obj, final int row) {
+ final int[] ints = (int[]) obj;
ints[row] = random.nextInt();
}
- private void generateINT64(Object obj, int row) {
- long[] longs = (long[]) obj;
+ private void generateDATE(final Object obj, final int row) {
+ final LocalDate[] dates = (LocalDate[]) obj;
+ dates[row] =
+ LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28));
+ }
+
+ private void generateINT64(final Object obj, final int row) {
+ final long[] longs = (long[]) obj;
longs[row] = random.nextLong();
}
- private void generateFLOAT(Object obj, int row) {
- float[] floats = (float[]) obj;
+ private void generateFLOAT(final Object obj, final int row) {
+ final float[] floats = (float[]) obj;
floats[row] = random.nextFloat();
}
- private void generateDOUBLE(Object obj, int row) {
- double[] doubles = (double[]) obj;
+ private void generateDOUBLE(final Object obj, final int row) {
+ final double[] doubles = (double[]) obj;
doubles[row] = random.nextDouble();
}
- private void generateBOOLEAN(Object obj, int row) {
- boolean[] booleans = (boolean[]) obj;
+ private void generateBOOLEAN(final Object obj, final int row) {
+ final boolean[] booleans = (boolean[]) obj;
booleans[row] = random.nextBoolean();
}
- private void generateTEXT(Object obj, int row) {
- Binary[] binaries = (Binary[]) obj;
+ private void generateTEXT(final Object obj, final int row) {
+ final Binary[] binaries = (Binary[]) obj;
binaries[row] =
new Binary(String.format("test point %d", random.nextInt()), TSFileConfig.STRING_CHARSET);
}
- public void generateDeletion(String device, int number) throws IOException, IllegalPathException {
- try (ModificationFile modificationFile =
+ public void generateDeletion(final String device, final int number)
+ throws IOException, IllegalPathException {
+ try (final ModificationFile modificationFile =
new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) {
writer.flushAllChunkGroups();
- TreeSet timeSet = device2TimeSet.get(device);
+ final TreeSet timeSet = device2TimeSet.get(device);
if (timeSet.isEmpty()) {
return;
}
- long fileOffset = tsFile.length();
- long maxTime = timeSet.last() - 1;
+ final long fileOffset = tsFile.length();
+ final long maxTime = timeSet.last() - 1;
for (int i = 0; i < number; i++) {
- int endTime = random.nextInt((int) (maxTime)) + 1;
- int startTime = random.nextInt(endTime);
- for (MeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) {
- Deletion deletion =
+ final int endTime = random.nextInt((int) (maxTime)) + 1;
+ final int startTime = random.nextInt(endTime);
+ for (final MeasurementSchema measurementSchema : device2MeasurementSchema.get(device)) {
+ final Deletion deletion =
new Deletion(
new PartialPath(
device
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
similarity index 65%
rename from integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
rename to integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index 0a3a44ee629ab..c1b1fdf91091f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -64,8 +64,8 @@
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
-public class IOTDBLoadTsFileIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(IOTDBLoadTsFileIT.class);
+public class IoTDBLoadTsFileIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBLoadTsFileIT.class);
private static final long PARTITION_INTERVAL = 10 * 1000L;
private static final int connectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(300);
private static final long loadTsFileAnalyzeSchemaMemorySizeInBytes = 10 * 1024L;
@@ -95,8 +95,8 @@ public void tearDown() throws Exception {
}
private void registerSchema() throws SQLException {
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_0);
statement.execute("CREATE DATABASE " + SchemaConfig.STORAGE_GROUP_1);
@@ -113,19 +113,24 @@ private void registerSchema() throws SQLException {
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13)));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17)));
statement.execute(convert2SQL(SchemaConfig.DEVICE_2, SchemaConfig.MEASUREMENT_20));
statement.execute(convert2SQL(SchemaConfig.DEVICE_3, SchemaConfig.MEASUREMENT_30));
statement.execute(
- convert2AlignedSQL(SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40)));
+ convert2AlignedSQL(
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40)));
}
}
- private String convert2SQL(String device, MeasurementSchema schema) {
- String sql =
+ private String convert2SQL(final String device, final MeasurementSchema schema) {
+ final String sql =
String.format(
"create timeseries %s %s",
new Path(device, schema.getMeasurementId(), true).getFullPath(),
@@ -134,10 +139,10 @@ private String convert2SQL(String device, MeasurementSchema schema) {
return sql;
}
- private String convert2AlignedSQL(String device, List schemas) {
+ private String convert2AlignedSQL(final String device, final List schemas) {
String sql = String.format("create aligned timeseries %s(", device);
for (int i = 0; i < schemas.size(); i++) {
- MeasurementSchema schema = schemas.get(i);
+ final MeasurementSchema schema = schemas.get(i);
sql += (String.format("%s %s", schema.getMeasurementId(), schema.getType().name()));
sql += (i == schemas.size() - 1 ? ")" : ",");
}
@@ -146,17 +151,17 @@ private String convert2AlignedSQL(String device, List schemas
}
private void deleteSG() throws SQLException {
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_0));
statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_1));
- } catch (IoTDBSQLException ignored) {
+ } catch (final IoTDBSQLException ignored) {
}
}
private boolean deleteDir() {
- for (File file : tmpDir.listFiles()) {
+ for (final File file : tmpDir.listFiles()) {
if (!file.delete()) {
return false;
}
@@ -168,49 +173,59 @@ private boolean deleteDir() {
public void testLoad() throws Exception {
registerSchema();
- long writtenPoint1 = 0;
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true);
writtenPoint1 = generator.getTotalNumber();
}
- long writtenPoint2 = 0;
+ final long writtenPoint2;
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint2 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
@@ -223,9 +238,9 @@ public void testLoad() throws Exception {
}
}
- // try delete after loading. Expect no deadlock
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ // Try to delete after loading. Expect no deadlock
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(
String.format(
"delete timeseries %s.%s",
@@ -235,29 +250,38 @@ public void testLoad() throws Exception {
@Test
public void testLoadWithExtendTemplate() throws Exception {
- long writtenPoint1 = 0;
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint1 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute("create database root.sg.test_0");
statement.execute(
@@ -266,17 +290,17 @@ public void testLoadWithExtendTemplate() throws Exception {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
} else {
Assert.fail("This ResultSet is empty.");
}
}
- Set nodes =
+ final Set nodes =
new HashSet<>(
Arrays.asList(
"lat",
@@ -284,14 +308,18 @@ public void testLoadWithExtendTemplate() throws Exception {
SchemaConfig.MEASUREMENT_00.getMeasurementId(),
SchemaConfig.MEASUREMENT_01.getMeasurementId(),
SchemaConfig.MEASUREMENT_02.getMeasurementId(),
- SchemaConfig.MEASUREMENT_03.getMeasurementId()));
- try (ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) {
+ SchemaConfig.MEASUREMENT_03.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_04.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_05.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_06.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_07.getMeasurementId()));
+ try (final ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) {
while (resultSet.next()) {
String device = resultSet.getString(ColumnHeaderConstant.CHILD_NODES);
Assert.assertTrue(nodes.remove(device));
}
Assert.assertTrue(nodes.isEmpty());
- } catch (Exception e) {
+ } catch (final Exception e) {
e.printStackTrace();
Assert.fail("Parse result set error.");
}
@@ -300,67 +328,77 @@ public void testLoadWithExtendTemplate() throws Exception {
@Test
public void testLoadWithAutoRegister() throws Exception {
- long writtenPoint1 = 0;
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint1 = generator.getTotalNumber();
}
- long writtenPoint2 = 0;
+ final long writtenPoint2;
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint2 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
- long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
+ final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
Assert.assertEquals(writtenPoint2, sg2Count);
} else {
Assert.fail("This ResultSet is empty.");
}
}
- Map isAligned = new HashMap<>();
+ final Map isAligned = new HashMap<>();
isAligned.put(SchemaConfig.DEVICE_0, "false");
isAligned.put(SchemaConfig.DEVICE_1, "true");
isAligned.put(SchemaConfig.DEVICE_2, "false");
isAligned.put(SchemaConfig.DEVICE_3, "false");
isAligned.put(SchemaConfig.DEVICE_4, "true");
- try (ResultSet resultSet = statement.executeQuery("show devices")) {
+ try (final ResultSet resultSet = statement.executeQuery("show devices")) {
int size = 0;
while (resultSet.next()) {
size += 1;
@@ -369,7 +407,7 @@ public void testLoadWithAutoRegister() throws Exception {
isAligned.get(device), resultSet.getString(ColumnHeaderConstant.IS_ALIGNED));
}
Assert.assertEquals(isAligned.size(), size);
- } catch (Exception e) {
+ } catch (final Exception e) {
e.printStackTrace();
Assert.fail("Parse result set error.");
}
@@ -381,33 +419,43 @@ public void testAuth() throws Exception {
createUser("test", "test123");
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "test1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "test1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
}
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "test2-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "test2-0-0-0.tsfile"))) {
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
@@ -443,33 +491,41 @@ public void testAuth() throws Exception {
@Test
public void testLoadWithOnSuccess() throws Exception {
- File file1 = new File(tmpDir, "1-0-0-0.tsfile");
- File file2 = new File(tmpDir, "2-0-0-0.tsfile");
+ final File file1 = new File(tmpDir, "1-0-0-0.tsfile");
+ final File file2 = new File(tmpDir, "2-0-0-0.tsfile");
long writtenPoint1 = 0;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(file1)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file1)) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint1 = generator.getTotalNumber();
}
- long writtenPoint2 = 0;
+ final long writtenPoint2;
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(file2)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file2)) {
generator.registerTimeseries(
SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
@@ -482,16 +538,16 @@ public void testLoadWithOnSuccess() throws Exception {
writtenPoint2 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(
String.format("load \"%s\" sglevel=2 onSuccess=none", file1.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
} else {
Assert.fail("This ResultSet is empty.");
@@ -500,13 +556,13 @@ public void testLoadWithOnSuccess() throws Exception {
Assert.assertTrue(file1.exists());
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(
String.format("load \"%s\" sglevel=2 onSuccess=delete", file2.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
@@ -525,19 +581,19 @@ public void testLoadWithOnSuccess() throws Exception {
public void testLoadWithLastCache() throws Exception {
registerSchema();
- String device = SchemaConfig.DEVICE_0;
- String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId();
+ final String device = SchemaConfig.DEVICE_0;
+ final String measurement = SchemaConfig.MEASUREMENT_00.getMeasurementId();
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(
String.format("insert into %s(timestamp, %s) values(100, 100)", device, measurement));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery(String.format("select last %s from %s", measurement, device))) {
if (resultSet.next()) {
- String lastValue = resultSet.getString(ColumnHeaderConstant.VALUE);
+ final String lastValue = resultSet.getString(ColumnHeaderConstant.VALUE);
Assert.assertEquals("100", lastValue);
} else {
Assert.fail("This ResultSet is empty.");
@@ -545,50 +601,58 @@ public void testLoadWithLastCache() throws Exception {
}
}
- File file1 = new File(tmpDir, "1-0-0-0.tsfile");
- File file2 = new File(tmpDir, "2-0-0-0.tsfile");
+ final File file1 = new File(tmpDir, "1-0-0-0.tsfile");
+ final File file2 = new File(tmpDir, "2-0-0-0.tsfile");
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(file1)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file1)) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
}
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(file2)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file2)) {
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery(String.format("select last %s from %s", measurement, device))) {
if (resultSet.next()) {
- String lastTime = resultSet.getString(ColumnHeaderConstant.TIME);
+ final String lastTime = resultSet.getString(ColumnHeaderConstant.TIME);
Assert.assertEquals(String.valueOf(PARTITION_INTERVAL), lastTime);
} else {
Assert.fail("This ResultSet is empty.");
@@ -599,56 +663,64 @@ public void testLoadWithLastCache() throws Exception {
@Test
public void testLoadWithOnNonStandardTsFileName() throws Exception {
- File file1 = new File(tmpDir, "1-0-0-0.tsfile");
- File file2 = new File(tmpDir, "a-1.tsfile");
- long writtenPoint1 = 0;
+ final File file1 = new File(tmpDir, "1-0-0-0.tsfile");
+ final File file2 = new File(tmpDir, "a-1.tsfile");
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(file1)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file1)) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint1 = generator.getTotalNumber();
}
- long writtenPoint2 = 0;
+ final long writtenPoint2;
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(file2)) {
+ try (final TsFileGenerator generator = new TsFileGenerator(file2)) {
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
writtenPoint2 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
- long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
+ final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
Assert.assertEquals(writtenPoint2, sg2Count);
} else {
Assert.fail("This ResultSet is empty.");
@@ -659,23 +731,32 @@ public void testLoadWithOnNonStandardTsFileName() throws Exception {
@Test
public void testLoadWithMods() throws Exception {
- long writtenPoint1 = 0;
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true);
generator.generateDeletion(SchemaConfig.DEVICE_0, 10);
@@ -683,16 +764,17 @@ public void testLoadWithMods() throws Exception {
writtenPoint1 = generator.getTotalNumber();
}
- long writtenPoint2 = 0;
+ final long writtenPoint2;
// device 2, device 3, device4, sg 1
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
generator.resetRandom(1000);
generator.registerTimeseries(
- SchemaConfig.DEVICE_2, Arrays.asList(SchemaConfig.MEASUREMENT_20));
+ SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
generator.registerTimeseries(
- SchemaConfig.DEVICE_3, Arrays.asList(SchemaConfig.MEASUREMENT_30));
+ SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
generator.registerAlignedTimeseries(
- SchemaConfig.DEVICE_4, Arrays.asList(SchemaConfig.MEASUREMENT_40));
+ SchemaConfig.DEVICE_4, Collections.singletonList(SchemaConfig.MEASUREMENT_40));
generator.generateData(SchemaConfig.DEVICE_2, 100, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 100, PARTITION_INTERVAL / 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 100, PARTITION_INTERVAL / 10_000, true);
@@ -705,17 +787,17 @@ public void testLoadWithMods() throws Exception {
writtenPoint2 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
- long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
+ final long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
Assert.assertEquals(writtenPoint2, sg2Count);
} else {
Assert.fail("This ResultSet is empty.");
@@ -726,14 +808,14 @@ public void testLoadWithMods() throws Exception {
@Test
public void testLoadWithEmptyTsFile() throws Exception {
- try (TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {}
+ try (final TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {}
- try (Connection connection = EnvFactory.getEnv().getConnection();
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet = statement.executeQuery("show timeseries")) {
+ try (final ResultSet resultSet = statement.executeQuery("show timeseries")) {
Assert.assertFalse(resultSet.next());
}
}
@@ -741,21 +823,30 @@ public void testLoadWithEmptyTsFile() throws Exception {
@Test
public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0,
Arrays.asList(
SchemaConfig.MEASUREMENT_00,
SchemaConfig.MEASUREMENT_01,
SchemaConfig.MEASUREMENT_02,
- SchemaConfig.MEASUREMENT_03));
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
generator.registerAlignedTimeseries(
SchemaConfig.DEVICE_1,
Arrays.asList(
SchemaConfig.MEASUREMENT_10,
SchemaConfig.MEASUREMENT_11,
SchemaConfig.MEASUREMENT_12,
- SchemaConfig.MEASUREMENT_13));
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
// generate ns timestamp
generator.generateData(
SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false, 1694689856546000000L);
@@ -763,11 +854,11 @@ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true, 1694689856546000000L);
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath()));
- } catch (IoTDBSQLException e) {
+ } catch (final IoTDBSQLException e) {
Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
}
}
@@ -776,24 +867,25 @@ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
public void testLoadLocally() throws Exception {
registerSchema();
- long writtenPoint1 = 0;
+ final long writtenPoint1;
// device 0, device 1, sg 0
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
generator.registerTimeseries(
SchemaConfig.DEVICE_0, Collections.singletonList(SchemaConfig.MEASUREMENT_00));
generator.generateData(SchemaConfig.DEVICE_0, 1, PARTITION_INTERVAL / 10_000, false);
writtenPoint1 = generator.getTotalNumber();
}
- try (Connection connection = EnvFactory.getEnv().getConnection();
- Statement statement = connection.createStatement()) {
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
- try (ResultSet resultSet =
+ try (final ResultSet resultSet =
statement.executeQuery("select count(*) from root.** group by level=1,2")) {
if (resultSet.next()) {
- long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
Assert.assertEquals(writtenPoint1, sg1Count);
} else {
Assert.fail("This ResultSet is empty.");
@@ -816,6 +908,14 @@ private static class SchemaConfig {
new MeasurementSchema("sensor_02", TSDataType.DOUBLE, TSEncoding.GORILLA);
private static final MeasurementSchema MEASUREMENT_03 =
new MeasurementSchema("sensor_03", TSDataType.TEXT, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_04 =
+ new MeasurementSchema("sensor_04", TSDataType.TIMESTAMP, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_05 =
+ new MeasurementSchema("sensor_05", TSDataType.DATE, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_06 =
+ new MeasurementSchema("sensor_06", TSDataType.BLOB, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_07 =
+ new MeasurementSchema("sensor_07", TSDataType.STRING, TSEncoding.PLAIN);
// device 1, aligned, sg 0
private static final String DEVICE_1 = "root.sg.test_0.a_1";
@@ -827,6 +927,14 @@ private static class SchemaConfig {
new MeasurementSchema("sensor_12", TSDataType.DOUBLE, TSEncoding.GORILLA);
private static final MeasurementSchema MEASUREMENT_13 =
new MeasurementSchema("sensor_13", TSDataType.TEXT, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_14 =
+ new MeasurementSchema("sensor_14", TSDataType.TIMESTAMP, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_15 =
+ new MeasurementSchema("sensor_15", TSDataType.DATE, TSEncoding.RLE);
+ private static final MeasurementSchema MEASUREMENT_16 =
+ new MeasurementSchema("sensor_16", TSDataType.BLOB, TSEncoding.PLAIN);
+ private static final MeasurementSchema MEASUREMENT_17 =
+ new MeasurementSchema("sensor_17", TSDataType.STRING, TSEncoding.PLAIN);
// device 2, non aligned, sg 1
private static final String DEVICE_2 = "root.sg.test_1.d_2";
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java
index 46bd37e630b9e..00d8a605a7548 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/AlignedChunkData.java
@@ -75,7 +75,9 @@ public class AlignedChunkData implements ChunkData {
private List chunkList;
public AlignedChunkData(
- String device, ChunkHeader chunkHeader, TTimePartitionSlot timePartitionSlot) {
+ final String device,
+ final ChunkHeader chunkHeader,
+ final TTimePartitionSlot timePartitionSlot) {
this.dataSize = 0;
this.device = device;
this.chunkHeaderList = new ArrayList<>();
@@ -94,7 +96,7 @@ public AlignedChunkData(
private void addAttrDataSize() { // Should be init before serialize, corresponding serializeAttr
dataSize += 2 * Byte.BYTES; // isModification and isAligned
dataSize += Long.BYTES; // timePartitionSlot
- int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
+ final int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength);
dataSize += deviceLength; // device
dataSize += Integer.BYTES; // chunkHeaderListSize
@@ -127,9 +129,9 @@ public boolean isAligned() {
}
@Override
- public void writeToFileWriter(TsFileIOWriter writer) throws IOException {
+ public void writeToFileWriter(final TsFileIOWriter writer) throws IOException {
if (chunkList != null) {
- for (Chunk chunk : chunkList) {
+ for (final Chunk chunk : chunkList) {
writer.writeChunk(chunk);
}
} else {
@@ -137,7 +139,7 @@ public void writeToFileWriter(TsFileIOWriter writer) throws IOException {
}
}
- public void addValueChunk(ChunkHeader chunkHeader) {
+ public void addValueChunk(final ChunkHeader chunkHeader) {
this.chunkHeaderList.add(chunkHeader);
this.pageNumbers.add(0);
dataSize += chunkHeader.getSerializedSize();
@@ -147,7 +149,7 @@ public void addValueChunk(ChunkHeader chunkHeader) {
}
@Override
- public void serialize(DataOutputStream stream) throws IOException {
+ public void serialize(final DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(isModification(), stream);
ReadWriteIOUtils.write(isAligned(), stream);
serializeAttr(stream);
@@ -155,31 +157,32 @@ public void serialize(DataOutputStream stream) throws IOException {
close();
}
- private void serializeAttr(DataOutputStream stream) throws IOException {
+ private void serializeAttr(final DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream);
ReadWriteIOUtils.write(device, stream);
ReadWriteIOUtils.write(dataSize, stream);
ReadWriteIOUtils.write(needDecodeChunk, stream);
ReadWriteIOUtils.write(chunkHeaderList.size(), stream);
- for (ChunkHeader chunkHeader : chunkHeaderList) {
+ for (final ChunkHeader chunkHeader : chunkHeaderList) {
chunkHeader.serializeTo(stream); // Chunk header already serialize chunk type
}
if (needDecodeChunk) {
- for (Integer pageNumber : pageNumbers) {
+ for (final Integer pageNumber : pageNumbers) {
ReadWriteIOUtils.write(pageNumber, stream);
}
}
}
@Override
- public void writeEntireChunk(ByteBuffer chunkData, IChunkMetadata chunkMetadata)
+ public void writeEntireChunk(final ByteBuffer chunkData, final IChunkMetadata chunkMetadata)
throws IOException {
dataSize += ReadWriteIOUtils.write(chunkData, stream);
dataSize += chunkMetadata.getStatistics().serialize(stream);
}
@Override
- public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws IOException {
+ public void writeEntirePage(final PageHeader pageHeader, final ByteBuffer pageData)
+ throws IOException {
pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1);
// serialize needDecode==false
dataSize += ReadWriteIOUtils.write(false, stream);
@@ -190,17 +193,17 @@ public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws I
}
@Override
- public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
+ public void writeDecodePage(final long[] times, final Object[] values, final int satisfiedLength)
throws IOException {
pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1);
satisfiedLengthQueue.offer(satisfiedLength);
- long startTime = timePartitionSlot.getStartTime();
- long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
+ final long startTime = timePartitionSlot.getStartTime();
+ final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
// serialize needDecode==true
dataSize += ReadWriteIOUtils.write(true, stream);
dataSize += ReadWriteIOUtils.write(satisfiedLength, stream);
- for (long time : times) {
+ for (final long time : times) {
if (time >= endTime) {
break;
}
@@ -210,12 +213,13 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
}
}
- public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataType dataType)
+ public void writeDecodeValuePage(
+ final long[] times, final TsPrimitiveType[] values, final TSDataType dataType)
throws IOException {
pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1);
- long startTime = timePartitionSlot.getStartTime();
- long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
- int satisfiedLength = satisfiedLengthQueue.poll();
+ final long startTime = timePartitionSlot.getStartTime();
+ final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
+ final int satisfiedLength = satisfiedLengthQueue.poll();
// serialize needDecode==true
dataSize += ReadWriteIOUtils.write(true, stream);
dataSize += ReadWriteIOUtils.write(satisfiedLength, stream);
@@ -232,9 +236,11 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT
dataSize += ReadWriteIOUtils.write(false, stream);
switch (dataType) {
case INT32:
+ case DATE:
dataSize += ReadWriteIOUtils.write(values[i].getInt(), stream);
break;
case INT64:
+ case TIMESTAMP:
dataSize += ReadWriteIOUtils.write(values[i].getLong(), stream);
break;
case FLOAT:
@@ -247,6 +253,8 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT
dataSize += ReadWriteIOUtils.write(values[i].getBoolean(), stream);
break;
case TEXT:
+ case BLOB:
+ case STRING:
dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream);
break;
default:
@@ -258,7 +266,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT
}
}
- private void deserializeTsFileData(InputStream stream) throws IOException, PageException {
+ private void deserializeTsFileData(final InputStream stream) throws IOException, PageException {
if (needDecodeChunk) {
buildChunkWriter(stream);
} else {
@@ -266,21 +274,21 @@ private void deserializeTsFileData(InputStream stream) throws IOException, PageE
}
}
- private void deserializeEntireChunk(InputStream stream) throws IOException {
+ private void deserializeEntireChunk(final InputStream stream) throws IOException {
chunkList = new ArrayList<>();
- for (ChunkHeader chunkHeader : chunkHeaderList) {
- ByteBuffer chunkData =
+ for (final ChunkHeader chunkHeader : chunkHeaderList) {
+ final ByteBuffer chunkData =
ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream));
- Statistics extends Serializable> statistics =
+ final Statistics extends Serializable> statistics =
Statistics.deserialize(stream, chunkHeader.getDataType());
chunkList.add(new Chunk(chunkHeader, chunkData, null, statistics));
}
}
- private void buildChunkWriter(InputStream stream) throws IOException, PageException {
- List measurementSchemaList = new ArrayList<>();
+ private void buildChunkWriter(final InputStream stream) throws IOException, PageException {
+ final List measurementSchemaList = new ArrayList<>();
IMeasurementSchema timeSchema = null;
- for (ChunkHeader chunkHeader : chunkHeaderList) {
+ for (final ChunkHeader chunkHeader : chunkHeaderList) {
if (TSDataType.VECTOR.equals(chunkHeader.getDataType())) {
timeSchema =
new MeasurementSchema(
@@ -299,7 +307,7 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
}
chunkWriter = new AlignedChunkWriterImpl(timeSchema, measurementSchemaList);
timeBatch = new ArrayList<>();
- int chunkHeaderSize = chunkHeaderList.size();
+ final int chunkHeaderSize = chunkHeaderList.size();
for (int i = 0; i < chunkHeaderSize; i++) {
buildChunk(stream, chunkHeaderList.get(i), pageNumbers.get(i), i - 1, i == 0);
}
@@ -308,52 +316,60 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
@SuppressWarnings({"squid:S6541", "squid:S3776"})
private void buildChunk(
- InputStream stream,
- ChunkHeader chunkHeader,
- int pageNumber,
- int valueChunkIndex,
- boolean isTimeChunk)
+ final InputStream stream,
+ final ChunkHeader chunkHeader,
+ final int pageNumber,
+ final int valueChunkIndex,
+ final boolean isTimeChunk)
throws IOException, PageException {
boolean needDecode;
int decodePageIndex = 0;
for (int j = 0; j < pageNumber; j++) {
needDecode = ReadWriteIOUtils.readBool(stream);
if (needDecode) {
- int length = ReadWriteIOUtils.readInt(stream);
+ final int length = ReadWriteIOUtils.readInt(stream);
long[] timePageBatch = new long[length];
if (!isTimeChunk) {
timePageBatch = timeBatch.get(decodePageIndex);
}
for (int i = 0; i < length; i++) {
if (isTimeChunk) {
- long time = ReadWriteIOUtils.readLong(stream);
+ final long time = ReadWriteIOUtils.readLong(stream);
timePageBatch[i] = time;
chunkWriter.writeTime(time);
} else {
- boolean isNull = ReadWriteIOUtils.readBool(stream);
+ final boolean isNull = ReadWriteIOUtils.readBool(stream);
switch (chunkHeader.getDataType()) {
case INT32:
- int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream);
+ case DATE:
+ final int int32Value = isNull ? DEFAULT_INT32 : ReadWriteIOUtils.readInt(stream);
chunkWriter.write(timePageBatch[i], int32Value, isNull, valueChunkIndex);
break;
case INT64:
- long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream);
+ case TIMESTAMP:
+ final long int64Value = isNull ? DEFAULT_INT64 : ReadWriteIOUtils.readLong(stream);
chunkWriter.write(timePageBatch[i], int64Value, isNull, valueChunkIndex);
break;
case FLOAT:
- float floatValue = isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream);
+ final float floatValue =
+ isNull ? DEFAULT_FLOAT : ReadWriteIOUtils.readFloat(stream);
chunkWriter.write(timePageBatch[i], floatValue, isNull, valueChunkIndex);
break;
case DOUBLE:
- double doubleValue = isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream);
+ final double doubleValue =
+ isNull ? DEFAULT_DOUBLE : ReadWriteIOUtils.readDouble(stream);
chunkWriter.write(timePageBatch[i], doubleValue, isNull, valueChunkIndex);
break;
case BOOLEAN:
- boolean boolValue = isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream);
+ final boolean boolValue =
+ isNull ? DEFAULT_BOOLEAN : ReadWriteIOUtils.readBool(stream);
chunkWriter.write(timePageBatch[i], boolValue, isNull, valueChunkIndex);
break;
case TEXT:
- Binary binaryValue = isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream);
+ case BLOB:
+ case STRING:
+ final Binary binaryValue =
+ isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream);
chunkWriter.write(timePageBatch[i], binaryValue, isNull, valueChunkIndex);
break;
default:
@@ -373,7 +389,8 @@ private void buildChunk(
chunkWriter.sealCurrentValuePage(valueChunkIndex);
}
} else {
- PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true);
+ final PageHeader pageHeader =
+ PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true);
if (isTimeChunk) {
chunkWriter.writePageHeaderAndDataIntoTimeBuff(
ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)),
@@ -388,26 +405,27 @@ private void buildChunk(
}
}
- public static AlignedChunkData deserialize(InputStream stream) throws IOException, PageException {
- TTimePartitionSlot timePartitionSlot =
+ public static AlignedChunkData deserialize(final InputStream stream)
+ throws IOException, PageException {
+ final TTimePartitionSlot timePartitionSlot =
TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream));
- String device = ReadWriteIOUtils.readString(stream);
- long dataSize = ReadWriteIOUtils.readLong(stream);
- boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream);
- int chunkHeaderListSize = ReadWriteIOUtils.readInt(stream);
- List chunkHeaderList = new ArrayList<>();
+ final String device = ReadWriteIOUtils.readString(stream);
+ final long dataSize = ReadWriteIOUtils.readLong(stream);
+ final boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream);
+ final int chunkHeaderListSize = ReadWriteIOUtils.readInt(stream);
+ final List chunkHeaderList = new ArrayList<>();
for (int i = 0; i < chunkHeaderListSize; i++) {
- byte chunkType = ReadWriteIOUtils.readByte(stream);
+ final byte chunkType = ReadWriteIOUtils.readByte(stream);
chunkHeaderList.add(ChunkHeader.deserializeFrom(stream, chunkType));
}
- List pageNumbers = new ArrayList<>();
+ final List pageNumbers = new ArrayList<>();
if (needDecodeChunk) {
for (int i = 0; i < chunkHeaderListSize; i++) {
pageNumbers.add(ReadWriteIOUtils.readInt(stream));
}
}
- AlignedChunkData chunkData =
+ final AlignedChunkData chunkData =
new AlignedChunkData(device, chunkHeaderList.get(0), timePartitionSlot);
chunkData.needDecodeChunk = needDecodeChunk;
chunkData.chunkHeaderList = chunkHeaderList;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
index 77467d55e98b1..371e97726c5e5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
@@ -60,7 +60,9 @@ public class NonAlignedChunkData implements ChunkData {
private Chunk chunk;
public NonAlignedChunkData(
- String device, ChunkHeader chunkHeader, TTimePartitionSlot timePartitionSlot) {
+ final String device,
+ final ChunkHeader chunkHeader,
+ final TTimePartitionSlot timePartitionSlot) {
this.dataSize = 0;
this.device = device;
this.chunkHeader = chunkHeader;
@@ -76,7 +78,7 @@ public NonAlignedChunkData(
private void addAttrDataSize() { // should be init before serialize, corresponding serializeAttr
dataSize += 2 * Byte.BYTES; // isModification and isAligned
dataSize += Long.BYTES; // timePartitionSlot
- int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
+ final int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength);
dataSize += deviceLength; // device
dataSize += chunkHeader.getSerializedSize(); // timeChunkHeader
@@ -108,7 +110,7 @@ public boolean isAligned() {
}
@Override
- public void writeToFileWriter(TsFileIOWriter writer) throws IOException {
+ public void writeToFileWriter(final TsFileIOWriter writer) throws IOException {
if (chunk != null) {
writer.writeChunk(chunk);
} else {
@@ -117,7 +119,7 @@ public void writeToFileWriter(TsFileIOWriter writer) throws IOException {
}
@Override
- public void serialize(DataOutputStream stream) throws IOException {
+ public void serialize(final DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(isModification(), stream);
ReadWriteIOUtils.write(isAligned(), stream);
serializeAttr(stream);
@@ -125,7 +127,7 @@ public void serialize(DataOutputStream stream) throws IOException {
close();
}
- private void serializeAttr(DataOutputStream stream) throws IOException {
+ private void serializeAttr(final DataOutputStream stream) throws IOException {
ReadWriteIOUtils.write(timePartitionSlot.getStartTime(), stream);
ReadWriteIOUtils.write(device, stream);
ReadWriteIOUtils.write(dataSize, stream);
@@ -137,14 +139,15 @@ private void serializeAttr(DataOutputStream stream) throws IOException {
}
@Override
- public void writeEntireChunk(ByteBuffer chunkData, IChunkMetadata chunkMetadata)
+ public void writeEntireChunk(final ByteBuffer chunkData, final IChunkMetadata chunkMetadata)
throws IOException {
dataSize += ReadWriteIOUtils.write(chunkData, stream);
dataSize += chunkMetadata.getStatistics().serialize(stream);
}
@Override
- public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws IOException {
+ public void writeEntirePage(final PageHeader pageHeader, final ByteBuffer pageData)
+ throws IOException {
pageNumber += 1;
dataSize += ReadWriteIOUtils.write(false, stream);
dataSize += pageHeader.serializeTo(stream);
@@ -152,11 +155,11 @@ public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData) throws I
}
@Override
- public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
+ public void writeDecodePage(final long[] times, final Object[] values, final int satisfiedLength)
throws IOException {
pageNumber += 1;
- long startTime = timePartitionSlot.getStartTime();
- long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
+ final long startTime = timePartitionSlot.getStartTime();
+ final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval();
dataSize += ReadWriteIOUtils.write(true, stream);
dataSize += ReadWriteIOUtils.write(satisfiedLength, stream);
@@ -168,9 +171,11 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
dataSize += ReadWriteIOUtils.write(times[i], stream);
switch (chunkHeader.getDataType()) {
case INT32:
+ case DATE:
dataSize += ReadWriteIOUtils.write((int) values[i], stream);
break;
case INT64:
+ case TIMESTAMP:
dataSize += ReadWriteIOUtils.write((long) values[i], stream);
break;
case FLOAT:
@@ -183,6 +188,8 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
dataSize += ReadWriteIOUtils.write((boolean) values[i], stream);
break;
case TEXT:
+ case BLOB:
+ case STRING:
dataSize += ReadWriteIOUtils.write((Binary) values[i], stream);
break;
default:
@@ -193,7 +200,7 @@ public void writeDecodePage(long[] times, Object[] values, int satisfiedLength)
}
}
- private void deserializeTsFileData(InputStream stream) throws IOException, PageException {
+ private void deserializeTsFileData(final InputStream stream) throws IOException, PageException {
if (needDecodeChunk) {
buildChunkWriter(stream);
} else {
@@ -201,15 +208,15 @@ private void deserializeTsFileData(InputStream stream) throws IOException, PageE
}
}
- private void deserializeEntireChunk(InputStream stream) throws IOException {
- ByteBuffer chunkData =
+ private void deserializeEntireChunk(final InputStream stream) throws IOException {
+ final ByteBuffer chunkData =
ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream));
- Statistics extends Serializable> statistics =
+ final Statistics extends Serializable> statistics =
Statistics.deserialize(stream, chunkHeader.getDataType());
chunk = new Chunk(chunkHeader, chunkData, null, statistics);
}
- private void buildChunkWriter(InputStream stream) throws IOException, PageException {
+ private void buildChunkWriter(final InputStream stream) throws IOException, PageException {
chunkWriter =
new ChunkWriterImpl(
new MeasurementSchema(
@@ -226,9 +233,11 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
long time = ReadWriteIOUtils.readLong(stream);
switch (chunkHeader.getDataType()) {
case INT32:
+ case DATE:
chunkWriter.write(time, ReadWriteIOUtils.readInt(stream));
break;
case INT64:
+ case TIMESTAMP:
chunkWriter.write(time, ReadWriteIOUtils.readLong(stream));
break;
case FLOAT:
@@ -241,6 +250,8 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
chunkWriter.write(time, ReadWriteIOUtils.readBool(stream));
break;
case TEXT:
+ case BLOB:
+ case STRING:
chunkWriter.write(time, ReadWriteIOUtils.readBinary(stream));
break;
default:
@@ -251,7 +262,8 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
chunkWriter.sealCurrentPage();
} else {
- PageHeader pageHeader = PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true);
+ final PageHeader pageHeader =
+ PageHeader.deserializeFrom(stream, chunkHeader.getDataType(), true);
chunkWriter.writePageHeaderAndDataIntoBuff(
ByteBuffer.wrap(ReadWriteIOUtils.readBytesWithSelfDescriptionLength(stream)),
pageHeader);
@@ -259,21 +271,22 @@ private void buildChunkWriter(InputStream stream) throws IOException, PageExcept
}
}
- public static NonAlignedChunkData deserialize(InputStream stream)
+ public static NonAlignedChunkData deserialize(final InputStream stream)
throws IOException, PageException {
- TTimePartitionSlot timePartitionSlot =
+ final TTimePartitionSlot timePartitionSlot =
TimePartitionUtils.getTimePartitionSlot(ReadWriteIOUtils.readLong(stream));
- String device = ReadWriteIOUtils.readString(stream);
- long dataSize = ReadWriteIOUtils.readLong(stream);
- boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream);
- byte chunkType = ReadWriteIOUtils.readByte(stream);
- ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType);
+ final String device = ReadWriteIOUtils.readString(stream);
+ final long dataSize = ReadWriteIOUtils.readLong(stream);
+ final boolean needDecodeChunk = ReadWriteIOUtils.readBool(stream);
+ final byte chunkType = ReadWriteIOUtils.readByte(stream);
+ final ChunkHeader chunkHeader = ChunkHeader.deserializeFrom(stream, chunkType);
int pageNumber = 0;
if (needDecodeChunk) {
pageNumber = ReadWriteIOUtils.readInt(stream);
}
- NonAlignedChunkData chunkData = new NonAlignedChunkData(device, chunkHeader, timePartitionSlot);
+ final NonAlignedChunkData chunkData =
+ new NonAlignedChunkData(device, chunkHeader, timePartitionSlot);
chunkData.needDecodeChunk = needDecodeChunk;
chunkData.pageNumber = pageNumber;
chunkData.deserializeTsFileData(stream);
From a659eaf03859d362edf1fe5356471f95cc6fdc5e Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 15:52:08 +0800
Subject: [PATCH 3/6] fix
---
.../changing/ChangingValueFilter.java | 22 ++++++++++++-------
.../sdt/SwingingDoorTrendingFilter.java | 20 +++++++++++------
2 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java
index cc83fbeb21ec8..7dc8c87c09b2d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java
@@ -21,6 +21,7 @@
import org.apache.iotdb.pipe.api.type.Binary;
+import java.time.LocalDate;
import java.util.Objects;
public class ChangingValueFilter {
@@ -36,26 +37,28 @@ public class ChangingValueFilter {
private T lastStoredValue;
public ChangingValueFilter(
- ChangingValueSamplingProcessor processor, long firstTimestamp, T firstValue) {
+ final ChangingValueSamplingProcessor processor,
+ final long firstTimestamp,
+ final T firstValue) {
this.processor = processor;
init(firstTimestamp, firstValue);
}
- private void init(long firstTimestamp, T firstValue) {
+ private void init(final long firstTimestamp, final T firstValue) {
lastStoredTimestamp = firstTimestamp;
lastStoredValue = firstValue;
}
- public boolean filter(long timestamp, T value) {
+ public boolean filter(final long timestamp, final T value) {
try {
return tryFilter(timestamp, value);
- } catch (Exception e) {
+ } catch (final Exception e) {
init(timestamp, value);
return true;
}
}
- private boolean tryFilter(long timestamp, T value) {
+ private boolean tryFilter(final long timestamp, final T value) {
final long timeDiff = Math.abs(timestamp - lastStoredTimestamp);
if (timeDiff <= processor.getCompressionMinTimeInterval()) {
@@ -67,8 +70,11 @@ private boolean tryFilter(long timestamp, T value) {
return true;
}
- // For boolean and string type, we only compare the value
- if (value instanceof Boolean || value instanceof String || value instanceof Binary) {
+ // For non-numerical types, we only compare the value
+ if (value instanceof Boolean
+ || value instanceof String
+ || value instanceof Binary
+ || value instanceof LocalDate) {
if (Objects.equals(lastStoredValue, value)) {
return false;
}
@@ -88,7 +94,7 @@ private boolean tryFilter(long timestamp, T value) {
return false;
}
- private void reset(long timestamp, T value) {
+ private void reset(final long timestamp, final T value) {
lastStoredTimestamp = timestamp;
lastStoredValue = value;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java
index 8d9e73deb0701..850cbd3ed0729 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java
@@ -21,6 +21,7 @@
import org.apache.iotdb.pipe.api.type.Binary;
+import java.time.LocalDate;
import java.util.Objects;
public class SwingingDoorTrendingFilter {
@@ -56,12 +57,14 @@ public class SwingingDoorTrendingFilter {
private T lastStoredValue;
public SwingingDoorTrendingFilter(
- SwingingDoorTrendingSamplingProcessor processor, long firstTimestamp, T firstValue) {
+ final SwingingDoorTrendingSamplingProcessor processor,
+ final long firstTimestamp,
+ final T firstValue) {
this.processor = processor;
init(firstTimestamp, firstValue);
}
- private void init(long firstTimestamp, T firstValue) {
+ private void init(final long firstTimestamp, final T firstValue) {
upperDoor = Double.MIN_VALUE;
lowerDoor = Double.MAX_VALUE;
@@ -72,16 +75,16 @@ private void init(long firstTimestamp, T firstValue) {
lastStoredValue = firstValue;
}
- public boolean filter(long timestamp, T value) {
+ public boolean filter(final long timestamp, final T value) {
try {
return tryFilter(timestamp, value);
- } catch (Exception e) {
+ } catch (final Exception e) {
init(timestamp, value);
return true;
}
}
- private boolean tryFilter(long timestamp, T value) {
+ private boolean tryFilter(final long timestamp, final T value) {
final long timeDiff = timestamp - lastStoredTimestamp;
final long absTimeDiff = Math.abs(timeDiff);
@@ -95,7 +98,10 @@ private boolean tryFilter(long timestamp, T value) {
}
// For boolean and string type, we only compare the value
- if (value instanceof Boolean || value instanceof String || value instanceof Binary) {
+ if (value instanceof Boolean
+ || value instanceof String
+ || value instanceof Binary
+ || value instanceof LocalDate) {
if (Objects.equals(lastStoredValue, value)) {
return false;
}
@@ -138,7 +144,7 @@ private boolean tryFilter(long timestamp, T value) {
return false;
}
- private void reset(long timestamp, T value) {
+ private void reset(final long timestamp, final T value) {
upperDoor = Double.MIN_VALUE;
lowerDoor = Double.MAX_VALUE;
From 74c9364dbc3f4a992f300fc7b902dd148751b554 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 17:12:11 +0800
Subject: [PATCH 4/6] fix ut
---
.../queryengine/execution/load/NonAlignedChunkData.java | 4 ++--
.../iotdb/db/pipe/event/PipeDataTypeTransformerTest.java | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
index 371e97726c5e5..2ae0acd2180bd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/NonAlignedChunkData.java
@@ -228,9 +228,9 @@ private void buildChunkWriter(final InputStream stream) throws IOException, Page
for (int j = 0; j < pageNumber; j++) {
needDecode = ReadWriteIOUtils.readBool(stream);
if (needDecode) {
- int length = ReadWriteIOUtils.readInt(stream);
+ final int length = ReadWriteIOUtils.readInt(stream);
for (int i = 0; i < length; i++) {
- long time = ReadWriteIOUtils.readLong(stream);
+ final long time = ReadWriteIOUtils.readLong(stream);
switch (chunkHeader.getDataType()) {
case INT32:
case DATE:
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
index 5ba92b46a4e7c..a2249a35e50e6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java
@@ -64,10 +64,10 @@ public void createDataTypeList() {
typeList.add(Type.DOUBLE);
typeList.add(Type.BOOLEAN);
typeList.add(Type.TEXT);
- typeList.add(Type.INT64);
- typeList.add(Type.INT32);
- typeList.add(Type.TEXT);
- typeList.add(Type.TEXT);
+ typeList.add(Type.TIMESTAMP);
+ typeList.add(Type.DATE);
+ typeList.add(Type.BLOB);
+ typeList.add(Type.STRING);
typeList.add(null);
}
From 612b4a8ba4a675ef68f82c50d166e1d4b0fa1d09 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 18:38:39 +0800
Subject: [PATCH 5/6] Added ITs
---
.../it/autocreate/IoTDBPipeDataSinkIT.java | 105 ++++++++++++++++++
.../pipe/it/single/IoTDBPipeAggregateIT.java | 14 +++
2 files changed, 119 insertions(+)
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
index 50a9b0d2ec5b2..8cd8d66d38791 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
@@ -148,4 +148,109 @@ public void testLegacyConnector() throws Exception {
Collections.singleton("0,1.0,"));
}
}
+
+ @Test
+ public void testReceiverAutoCreateByLog() throws Exception {
+ testReceiverAutoCreate(
+ new HashMap() {
+ {
+ put("source.realtime.mode", "forced-log");
+ }
+ });
+ }
+
+ @Test
+ public void testReceiverAutoCreateByFile() throws Exception {
+ testReceiverAutoCreate(
+ new HashMap() {
+ {
+ put("source.realtime.mode", "batch");
+ }
+ });
+ }
+
+ @Test
+ public void testReceiverAutoCreateWithPattern() throws Exception {
+ testReceiverAutoCreate(
+ new HashMap() {
+ {
+ put("source.realtime.mode", "batch");
+ put("source.path", "root.ln.wf01.wt0*.*");
+ }
+ });
+ }
+
+ private void testReceiverAutoCreate(final Map extractorAttributes)
+ throws Exception {
+ final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
+
+ final String receiverIp = receiverDataNode.getIp();
+ final int receiverPort = receiverDataNode.getPort();
+
+ try (final SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
+ final Map processorAttributes = new HashMap<>();
+ final Map connectorAttributes = new HashMap<>();
+
+ connectorAttributes.put("sink", "iotdb-thrift-sink");
+ connectorAttributes.put("sink.batch.enable", "false");
+ connectorAttributes.put("sink.ip", receiverIp);
+ connectorAttributes.put("sink.port", Integer.toString(receiverPort));
+
+ final TSStatus status =
+ client.createPipe(
+ new TCreatePipeReq("testPipe", connectorAttributes)
+ .setExtractorAttributes(extractorAttributes)
+ .setProcessorAttributes(processorAttributes));
+
+ Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+
+ // Do not fail if the failure has nothing to do with pipe
+ // Because the failures will randomly generate due to resource limitation
+ if (!TestUtils.tryExecuteNonQueriesWithRetry(
+ senderEnv,
+ Arrays.asList(
+ "create timeSeries root.ln.wf01.wt01.boolean boolean",
+ "create timeSeries root.ln.wf01.wt01.int32 int32",
+ "create timeSeries root.ln.wf01.wt01.int64 int64",
+ "create timeSeries root.ln.wf01.wt01.float float",
+ "create timeSeries root.ln.wf01.wt01.double double",
+ "create timeSeries root.ln.wf01.wt01.time_stamp timestamp",
+ "create timeSeries root.ln.wf01.wt01.date date",
+ "create timeSeries root.ln.wf01.wt01.text text",
+ "create timeSeries root.ln.wf01.wt01.string string",
+ "create timeSeries root.ln.wf01.wt01.blob blob",
+ "create aligned timeSeries root.ln.wf01.wt02(int32 int32, boolean boolean)",
+ "insert into root.ln.wf01.wt01(time, boolean, int32, int64, float, double, time_stamp, date, text, string, blob) values (20000, false, 123, 321, 13.3, 14.4, now(), '2000-12-13', 'abc', 'def', X'f103')",
+ "insert into root.ln.wf01.wt02(time, int32, boolean) values (20000, 123, false)",
+ // For pattern parse
+ "insert into root.ln.wf01.wt11(time, redundant_data) values (20000, -1)",
+ "flush"))) {
+ return;
+ }
+
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv,
+ "show timeSeries root.ln.wf01.wt01.*",
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,",
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ "root.ln.wf01.wt01.boolean,null,root.ln,BOOLEAN,RLE,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.int32,null,root.ln,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.int64,null,root.ln,INT64,TS_2DIFF,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.float,null,root.ln,FLOAT,GORILLA,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.double,null,root.ln,DOUBLE,GORILLA,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.time_stamp,null,root.ln,TIMESTAMP,TS_2DIFF,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.date,null,root.ln,DATE,TS_2DIFF,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.text,null,root.ln,TEXT,PLAIN,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.string,null,root.ln,STRING,PLAIN,LZ4,null,null,null,null,BASE,",
+ "root.ln.wf01.wt01.blob,null,root.ln,BLOB,PLAIN,LZ4,null,null,null,null,BASE,"))));
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv,
+ "show devices root.ln.wf01.wt02",
+ "Device,IsAligned,Template,TTL,",
+ Collections.singleton("root.ln.wf01.wt02,true,null,INF,"));
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java
index 544509b82894a..cbd9c08f4e5c2 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java
@@ -78,6 +78,20 @@ public void testAggregator() throws Exception {
.setProcessorAttributes(processorAttributes));
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+ // Test unsupported types
+ if (!TestUtils.tryExecuteNonQueriesWithRetry(
+ env,
+ Arrays.asList(
+ "create timeSeries root.ln.wf01.wt01.boolean boolean",
+ "create timeSeries root.ln.wf01.wt01.date date",
+ "create timeSeries root.ln.wf01.wt01.text text",
+ "create timeSeries root.ln.wf01.wt01.string string",
+ "create timeSeries root.ln.wf01.wt01.blob blob",
+ "insert into root.ln.wf01.wt01(time, boolean, date, text, string, blob) values (20000, false, '2000-12-13', 'abc', 'def', X'f103')",
+ "flush"))) {
+ return;
+ }
+
if (!TestUtils.tryExecuteNonQueriesWithRetry(
env,
Arrays.asList(
From 78ff20775428cdb7eb52eb13623852e601bacb98 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 5 Jun 2024 22:10:45 +0800
Subject: [PATCH 6/6] Added subscription tests
---
.../it/local/IoTDBSubscriptionBasicIT.java | 95 ++++++++++++++++++-
.../payload/SubscriptionSessionDataSet.java | 9 +-
2 files changed, 99 insertions(+), 5 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java
index 13a5e8abc2617..c2b67018858ae 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java
@@ -36,10 +36,15 @@
import org.apache.iotdb.session.subscription.payload.SubscriptionTsFileHandler;
import org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.TsFileReader;
import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.record.Tablet;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Assert;
@@ -53,6 +58,7 @@
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.time.Duration;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -83,12 +89,66 @@ public void tearDown() throws Exception {
}
@Test
- public void testBasicSubscribeTablets() throws Exception {
+ public void testSubscribeBooleanData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.BOOLEAN, "true", true);
+ }
+
+ @Test
+ public void testSubscribeIntData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.INT32, "1", 1);
+ }
+
+ @Test
+ public void testSubscribeLongData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.INT64, "1", 1L);
+ }
+
+ @Test
+ public void testSubscribeFloatData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.FLOAT, "1.0", 1.0F);
+ }
+
+ @Test
+ public void testSubscribeDoubleData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.DOUBLE, "1.0", 1.0);
+ }
+
+ @Test
+ public void testSubscribeTextData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.TEXT, "'a'", new Binary("a", TSFileConfig.STRING_CHARSET));
+ }
+
+ @Test
+ public void testSubscribeTimestampData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.TIMESTAMP, "123", 123L);
+ }
+
+ @Test
+ public void testSubscribeDateData() throws Exception {
+ testBasicSubscribeTablets(TSDataType.DATE, "'2011-03-01'", LocalDate.of(2011, 3, 1));
+ }
+
+ @Test
+ public void testSubscribeBlobData() throws Exception {
+ testBasicSubscribeTablets(
+ TSDataType.BLOB, "X'f013'", new Binary(new byte[] {(byte) 0xf0, 0x13}));
+ }
+
+ @Test
+ public void testSubscribeStringData() throws Exception {
+ testBasicSubscribeTablets(
+ TSDataType.STRING, "'a'", new Binary("a", TSFileConfig.STRING_CHARSET));
+ }
+
+ private void testBasicSubscribeTablets(
+ final TSDataType type, final String valueStr, final Object expectedData) throws Exception {
// Insert some historical data
try (final ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement(
+ String.format("create timeseries root.db.d1.s1 %s", type.toString()));
for (int i = 0; i < 100; ++i) {
session.executeNonQueryStatement(
- String.format("insert into root.db.d1(time, s1) values (%s, 1)", i));
+ String.format("insert into root.db.d1(time, s1) values (%s, %s)", i, valueStr));
}
session.executeNonQueryStatement("flush");
} catch (final Exception e) {
@@ -132,7 +192,12 @@ public void testBasicSubscribeTablets() throws Exception {
for (final SubscriptionSessionDataSet dataSet :
message.getSessionDataSetsHandler()) {
while (dataSet.hasNext()) {
- dataSet.next();
+ final RowRecord record = dataSet.next();
+ Assert.assertEquals(type.toString(), dataSet.getColumnTypes().get(1));
+ Assert.assertEquals(type, record.getFields().get(0).getDataType());
+ Assert.assertEquals(expectedData, getValue(type, dataSet.getTablet()));
+ Assert.assertEquals(
+ expectedData, record.getFields().get(0).getObjectValue(type));
rowCount.addAndGet(1);
}
}
@@ -167,6 +232,30 @@ public void testBasicSubscribeTablets() throws Exception {
}
}
+ private Object getValue(final TSDataType type, final Tablet tablet) {
+ switch (type) {
+ case BOOLEAN:
+ return ((boolean[]) tablet.values[0])[0];
+ case INT32:
+ return ((int[]) tablet.values[0])[0];
+ case INT64:
+ case TIMESTAMP:
+ return ((long[]) tablet.values[0])[0];
+ case FLOAT:
+ return ((float[]) tablet.values[0])[0];
+ case DOUBLE:
+ return ((double[]) tablet.values[0])[0];
+ case TEXT:
+ case BLOB:
+ case STRING:
+ return ((Binary[]) tablet.values[0])[0];
+ case DATE:
+ return ((LocalDate[]) tablet.values[0])[0];
+ default:
+ return null;
+ }
+ }
+
@Test
public void testBasicSubscribeTsFile() throws Exception {
// Insert some historical data
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
index c9e9fc61cebb5..ac54941f5151e 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
@@ -25,10 +25,12 @@
import org.apache.tsfile.read.common.Field;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -84,7 +86,7 @@ public List getColumnTypes() {
final List schemas = tablet.getSchemas();
columnTypeList.addAll(
- schemas.stream().map((schema) -> schema.getType().toString()).collect(Collectors.toList()));
+ schemas.stream().map(schema -> schema.getType().toString()).collect(Collectors.toList()));
return columnTypeList;
}
@@ -149,10 +151,13 @@ private static Field generateFieldFromTabletValue(
field.setBoolV(booleanValue);
break;
case INT32:
- case DATE:
final int intValue = ((int[]) value)[index];
field.setIntV(intValue);
break;
+ case DATE:
+ final LocalDate dateValue = ((LocalDate[]) value)[index];
+ field.setIntV(DateUtils.parseDateExpressionToInt(dateValue));
+ break;
case INT64:
case TIMESTAMP:
final long longValue = ((long[]) value)[index];