From 0f475ff5e8ce2e847a45e2602abcfe8de0c3d7c2 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Tue, 11 Jun 2024 21:00:06 +0800 Subject: [PATCH 1/5] load-tsfile script: disable redirection --- .../cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java index 4e74dbe5f594e..320b03334df59 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportTsFile.java @@ -310,7 +310,9 @@ public static int importFromTargetPath() { } sessionPool = - new SessionPool(host, Integer.parseInt(port), username, password, threadNum + 1); + new SessionPool( + host, Integer.parseInt(port), username, password, threadNum + 1, false, false); + sessionPool.setEnableQueryRedirection(false); traverseAndCollectFiles(file); addNoResourceOrModsToQueue(); From e6cb065ef10287b31c0ef50983fac4736356c089 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 15:11:05 +0800 Subject: [PATCH 2/5] refactor --- .../load/LoadTsFileMemoryManager.java | 10 ++-- .../plan/analyze/AnalyzeVisitor.java | 17 +++++-- .../plan/analyze/LoadTsfileAnalyzer.java | 47 ++++++++++--------- .../plan/planner/LocalExecutionPlanner.java | 16 ++++--- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileMemoryManager.java index eff5f790bdb3a..f034a4627b901 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/load/LoadTsFileMemoryManager.java @@ -64,12 +64,12 @@ private synchronized void forceAllocatedFromQuery(long sizeInBytes) throw new LoadRuntimeOutOfMemoryException( String.format( - "forceAllocate: failed to allocate memory from query engine after %d retries, " - + "total query memory %s, available memory for load %s bytes, " - + "used memory size %d bytes, requested memory size %d bytes", + "forceAllocate: failed to allocate memory from query engine after %s retries, " + + "total query memory %s bytes, current available memory for load %s bytes, " + + "current load used memory size %s bytes, load requested memory size %s bytes", MEMORY_ALLOCATE_MAX_RETRIES, - IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForOperators(), - LocalExecutionPlanner.getInstance().getFreeMemoryForOperators(), + QUERY_ENGINE_MEMORY_MANAGER.getAllocateMemoryForOperators(), + QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile(), usedMemorySizeInBytes.get(), sizeInBytes)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 038514ee02f03..e67de934eac0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2825,12 +2825,19 @@ private InsertBaseStatement removeLogicalView( @Override public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { - LoadTsfileAnalyzer loadTsfileAnalyzer = - new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher); - try { + try (final LoadTsfileAnalyzer loadTsfileAnalyzer = + new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher)) { return loadTsfileAnalyzer.analyzeFileByFile(); - } finally { - loadTsfileAnalyzer.close(); + } catch (final Exception e) { + final String exceptionMessage = + String.format( + "Failed to execute load tsfile statement %s. Detail: %s", + loadTsFileStatement, e.getMessage()); + logger.warn(exceptionMessage, e); + final Analysis analysis = new Analysis(); + analysis.setFinishQueryAfterAnalyze(true); + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage)); + return analysis; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java index 91ca66fc0d52d..41690bd67c9de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java @@ -82,7 +82,6 @@ import java.io.File; import java.io.IOException; -import java.nio.BufferUnderflowException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -94,7 +93,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class LoadTsfileAnalyzer { +public class LoadTsfileAnalyzer implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsfileAnalyzer.class); @@ -139,9 +138,10 @@ public class LoadTsfileAnalyzer { public Analysis analyzeFileByFile() { context.setQueryType(QueryType.WRITE); + Analysis analysis = new Analysis(); + // check if the system is read only if (CommonDescriptor.getInstance().getConfig().isReadOnly()) { - Analysis analysis = new Analysis(); analysis.setFinishQueryAfterAnalyze(true); analysis.setFailStatus( RpcUtils.getStatus(TSStatusCode.SYSTEM_READ_ONLY, LoadReadOnlyException.MESSAGE)); @@ -171,25 +171,17 @@ public Analysis analyzeFileByFile() { "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%", i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum)); } - } catch (IllegalArgumentException e) { - LOGGER.warn( - "Parse file {} to resource error, this TsFile maybe empty.", tsFile.getPath(), e); - throw new SemanticException( - String.format("TsFile %s is empty or incomplete.", tsFile.getPath())); } catch (AuthException e) { return createFailAnalysisForAuthException(e); - } catch (BufferUnderflowException e) { - LOGGER.warn( - "The file {} is not a valid tsfile. Please check the input file.", tsFile.getPath(), e); - throw new SemanticException( - String.format( - "The file %s is not a valid tsfile. Please check the input file.", - tsFile.getPath())); } catch (Exception e) { - LOGGER.warn("Parse file {} to resource error.", tsFile.getPath(), e); - throw new SemanticException( + final String exceptionMessage = String.format( - "Parse file %s to resource error, because %s", tsFile.getPath(), e.getMessage())); + "The file %s is not a valid tsfile. Please check the input file. Detail: %s", + tsFile.getPath(), e.getMessage()); + LOGGER.warn(exceptionMessage, e); + analysis.setFinishQueryAfterAnalyze(true); + analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage)); + return analysis; } } @@ -197,16 +189,30 @@ public Analysis analyzeFileByFile() { schemaAutoCreatorAndVerifier.flush(); } catch (AuthException e) { return createFailAnalysisForAuthException(e); + } catch (Exception e) { + final String exceptionMessage = + String.format( + "Auto create or verify schema error when executing statement %s. Detail: %s.", + loadTsFileStatement, e.getMessage()); + LOGGER.warn(exceptionMessage, e); + analysis.setFinishQueryAfterAnalyze(true); + analysis.setFailStatus( + RpcUtils.getStatus( + TSStatusCode.LOAD_FILE_ERROR, + String.format( + "Auto create or verify schema error when executing statement %s.", + loadTsFileStatement))); + return analysis; } LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed."); // data partition will be queried in the scheduler - final Analysis analysis = new Analysis(); analysis.setStatement(loadTsFileStatement); return analysis; } + @Override public void close() { schemaAutoCreatorAndVerifier.close(); } @@ -360,8 +366,7 @@ public void autoCreateAndVerify( * This can only be invoked after all timeseries in the current tsfile have been processed. * Otherwise, the isAligned status may be wrong. */ - public void flushAndClearDeviceIsAlignedCacheIfNecessary() - throws SemanticException, AuthException { + public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticException { // avoid OOM when loading a tsfile with too many timeseries // or loading too many tsfiles at the same time schemaCache.clearDeviceIsAlignedCacheIfNecessary(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index 4789834f35eb7..4db63ed1d4937 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -52,12 +52,12 @@ public class LocalExecutionPlanner { private static final Logger LOGGER = LoggerFactory.getLogger(LocalExecutionPlanner.class); private static final long ALLOCATE_MEMORY_FOR_OPERATORS; - private static final long MAX_REST_MEMORY_FOR_LOAD; + private static final long MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD; static { IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); ALLOCATE_MEMORY_FOR_OPERATORS = CONFIG.getAllocateMemoryForOperators(); - MAX_REST_MEMORY_FOR_LOAD = + MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD = (long) ((ALLOCATE_MEMORY_FOR_OPERATORS) * (1.0 - CONFIG.getMaxAllocateMemoryRatioForLoad())); } @@ -69,6 +69,10 @@ public long getFreeMemoryForOperators() { return freeMemoryForOperators; } + public long getFreeMemoryForLoadTsFile() { + return freeMemoryForOperators - MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD; + } + public static LocalExecutionPlanner getInstance() { return InstanceHolder.INSTANCE; } @@ -211,7 +215,7 @@ private List collectSourcePaths(LocalExecutionPlanContext context) } public synchronized boolean forceAllocateFreeMemoryForOperators(long memoryInBytes) { - if (freeMemoryForOperators - memoryInBytes <= MAX_REST_MEMORY_FOR_LOAD) { + if (freeMemoryForOperators - memoryInBytes <= MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) { return false; } else { freeMemoryForOperators -= memoryInBytes; @@ -220,9 +224,9 @@ public synchronized boolean forceAllocateFreeMemoryForOperators(long memoryInByt } public synchronized long tryAllocateFreeMemoryForOperators(long memoryInBytes) { - if (freeMemoryForOperators - memoryInBytes <= MAX_REST_MEMORY_FOR_LOAD) { - long result = freeMemoryForOperators - MAX_REST_MEMORY_FOR_LOAD; - freeMemoryForOperators = MAX_REST_MEMORY_FOR_LOAD; + if (freeMemoryForOperators - memoryInBytes <= MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) { + long result = freeMemoryForOperators - MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD; + freeMemoryForOperators = MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD; return result; } else { freeMemoryForOperators -= memoryInBytes; From 1e61e11c5755d68aebaf15cbe6a704740f248fb7 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 15:39:06 +0800 Subject: [PATCH 3/5] Update PipeStatementTSStatusVisitor.java --- .../visitor/PipeStatementTSStatusVisitor.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTSStatusVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTSStatusVisitor.java index be4ee14ff02f7..812f8d291512e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTSStatusVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementTSStatusVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; @@ -56,6 +57,20 @@ public TSStatus visitNode(final StatementNode node, final TSStatus context) { return context; } + @Override + public TSStatus visitLoadFile( + final LoadTsFileStatement loadTsFileStatement, final TSStatus context) { + if (context.getCode() == TSStatusCode.SYSTEM_READ_ONLY.getStatusCode() + || context.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode() + && context.getMessage() != null + && context.getMessage().contains("memory")) { + return new TSStatus( + TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode()) + .setMessage(context.getMessage()); + } + return super.visitLoadFile(loadTsFileStatement, context); + } + @Override public TSStatus visitInsertTablet( final InsertTabletStatement insertTabletStatement, final TSStatus context) { @@ -82,7 +97,11 @@ public TSStatus visitInsertMultiTablets( private TSStatus visitInsertBase( final InsertBaseStatement insertBaseStatement, final TSStatus context) { - if (context.getCode() == TSStatusCode.OUT_OF_TTL.getStatusCode()) { + if (context.getCode() == TSStatusCode.SYSTEM_READ_ONLY.getStatusCode()) { + return new TSStatus( + TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode()) + .setMessage(context.getMessage()); + } else if (context.getCode() == TSStatusCode.OUT_OF_TTL.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); } else if (context.getCode() == TSStatusCode.METADATA_ERROR.getStatusCode()) { From 9e6eb887ca0fbb87517f849870e885ccdbfbae7e Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 15:45:57 +0800 Subject: [PATCH 4/5] Update PipeStatementExceptionVisitor.java --- .../db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java index 7cbe5bed4167b..5a65b1a5e0577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java @@ -61,7 +61,7 @@ public TSStatus visitLoadFile( return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); } - return super.visitLoadFile(loadTsFileStatement, context); + return visitStatement(loadTsFileStatement, context); } @Override From c538fe1eb221f3ac7b5ddcb12057a9821fe29621 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 12 Jun 2024 16:00:42 +0800 Subject: [PATCH 5/5] refactor --- .../iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java | 3 ++- .../db/queryengine/plan/analyze/LoadTsfileAnalyzer.java | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index e67de934eac0a..84708e5214394 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2832,7 +2832,8 @@ public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryC final String exceptionMessage = String.format( "Failed to execute load tsfile statement %s. Detail: %s", - loadTsFileStatement, e.getMessage()); + loadTsFileStatement, + e.getMessage() == null ? e.getClass().getName() : e.getMessage()); logger.warn(exceptionMessage, e); final Analysis analysis = new Analysis(); analysis.setFinishQueryAfterAnalyze(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java index 41690bd67c9de..79e84b3400daa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java @@ -177,7 +177,7 @@ public Analysis analyzeFileByFile() { final String exceptionMessage = String.format( "The file %s is not a valid tsfile. Please check the input file. Detail: %s", - tsFile.getPath(), e.getMessage()); + tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage()); LOGGER.warn(exceptionMessage, e); analysis.setFinishQueryAfterAnalyze(true); analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage)); @@ -193,7 +193,8 @@ public Analysis analyzeFileByFile() { final String exceptionMessage = String.format( "Auto create or verify schema error when executing statement %s. Detail: %s.", - loadTsFileStatement, e.getMessage()); + loadTsFileStatement, + e.getMessage() == null ? e.getClass().getName() : e.getMessage()); LOGGER.warn(exceptionMessage, e); analysis.setFinishQueryAfterAnalyze(true); analysis.setFailStatus(