Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,9 @@ public static int importFromTargetPath() {
}

sessionPool =
new SessionPool(host, Integer.parseInt(port), username, password, threadNum + 1);
new SessionPool(
host, Integer.parseInt(port), username, password, threadNum + 1, false, false);
sessionPool.setEnableQueryRedirection(false);

traverseAndCollectFiles(file);
addNoResourceOrModsToQueue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public TSStatus visitLoadFile(
return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode())
.setMessage(context.getMessage());
}
return super.visitLoadFile(loadTsFileStatement, context);
return visitStatement(loadTsFileStatement, context);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement;
Expand All @@ -56,6 +57,20 @@ public TSStatus visitNode(final StatementNode node, final TSStatus context) {
return context;
}

@Override
public TSStatus visitLoadFile(
final LoadTsFileStatement loadTsFileStatement, final TSStatus context) {
if (context.getCode() == TSStatusCode.SYSTEM_READ_ONLY.getStatusCode()
|| context.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()
&& context.getMessage() != null
&& context.getMessage().contains("memory")) {
return new TSStatus(
TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
.setMessage(context.getMessage());
}
return super.visitLoadFile(loadTsFileStatement, context);
}

@Override
public TSStatus visitInsertTablet(
final InsertTabletStatement insertTabletStatement, final TSStatus context) {
Expand All @@ -82,7 +97,11 @@ public TSStatus visitInsertMultiTablets(

private TSStatus visitInsertBase(
final InsertBaseStatement insertBaseStatement, final TSStatus context) {
if (context.getCode() == TSStatusCode.OUT_OF_TTL.getStatusCode()) {
if (context.getCode() == TSStatusCode.SYSTEM_READ_ONLY.getStatusCode()) {
return new TSStatus(
TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
.setMessage(context.getMessage());
} else if (context.getCode() == TSStatusCode.OUT_OF_TTL.getStatusCode()) {
return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())
.setMessage(context.getMessage());
} else if (context.getCode() == TSStatusCode.METADATA_ERROR.getStatusCode()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ private synchronized void forceAllocatedFromQuery(long sizeInBytes)

throw new LoadRuntimeOutOfMemoryException(
String.format(
"forceAllocate: failed to allocate memory from query engine after %d retries, "
+ "total query memory %s, available memory for load %s bytes, "
+ "used memory size %d bytes, requested memory size %d bytes",
"forceAllocate: failed to allocate memory from query engine after %s retries, "
+ "total query memory %s bytes, current available memory for load %s bytes, "
+ "current load used memory size %s bytes, load requested memory size %s bytes",
MEMORY_ALLOCATE_MAX_RETRIES,
IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForOperators(),
LocalExecutionPlanner.getInstance().getFreeMemoryForOperators(),
QUERY_ENGINE_MEMORY_MANAGER.getAllocateMemoryForOperators(),
QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile(),
usedMemorySizeInBytes.get(),
sizeInBytes));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2825,12 +2825,20 @@ private InsertBaseStatement removeLogicalView(

@Override
public Analysis visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
LoadTsfileAnalyzer loadTsfileAnalyzer =
new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher);
try {
try (final LoadTsfileAnalyzer loadTsfileAnalyzer =
new LoadTsfileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher)) {
return loadTsfileAnalyzer.analyzeFileByFile();
} finally {
loadTsfileAnalyzer.close();
} catch (final Exception e) {
final String exceptionMessage =
String.format(
"Failed to execute load tsfile statement %s. Detail: %s",
loadTsFileStatement,
e.getMessage() == null ? e.getClass().getName() : e.getMessage());
logger.warn(exceptionMessage, e);
final Analysis analysis = new Analysis();
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
return analysis;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@

import java.io.File;
import java.io.IOException;
import java.nio.BufferUnderflowException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -94,7 +93,7 @@
import java.util.Set;
import java.util.stream.Collectors;

public class LoadTsfileAnalyzer {
public class LoadTsfileAnalyzer implements AutoCloseable {

private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsfileAnalyzer.class);

Expand Down Expand Up @@ -139,9 +138,10 @@ public class LoadTsfileAnalyzer {
public Analysis analyzeFileByFile() {
context.setQueryType(QueryType.WRITE);

Analysis analysis = new Analysis();

// check if the system is read only
if (CommonDescriptor.getInstance().getConfig().isReadOnly()) {
Analysis analysis = new Analysis();
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(
RpcUtils.getStatus(TSStatusCode.SYSTEM_READ_ONLY, LoadReadOnlyException.MESSAGE));
Expand Down Expand Up @@ -171,42 +171,49 @@ public Analysis analyzeFileByFile() {
"Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%",
i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum));
}
} catch (IllegalArgumentException e) {
LOGGER.warn(
"Parse file {} to resource error, this TsFile maybe empty.", tsFile.getPath(), e);
throw new SemanticException(
String.format("TsFile %s is empty or incomplete.", tsFile.getPath()));
} catch (AuthException e) {
return createFailAnalysisForAuthException(e);
} catch (BufferUnderflowException e) {
LOGGER.warn(
"The file {} is not a valid tsfile. Please check the input file.", tsFile.getPath(), e);
throw new SemanticException(
String.format(
"The file %s is not a valid tsfile. Please check the input file.",
tsFile.getPath()));
} catch (Exception e) {
LOGGER.warn("Parse file {} to resource error.", tsFile.getPath(), e);
throw new SemanticException(
final String exceptionMessage =
String.format(
"Parse file %s to resource error, because %s", tsFile.getPath(), e.getMessage()));
"The file %s is not a valid tsfile. Please check the input file. Detail: %s",
tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage());
LOGGER.warn(exceptionMessage, e);
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
return analysis;
}
}

try {
schemaAutoCreatorAndVerifier.flush();
} catch (AuthException e) {
return createFailAnalysisForAuthException(e);
} catch (Exception e) {
final String exceptionMessage =
String.format(
"Auto create or verify schema error when executing statement %s. Detail: %s.",
loadTsFileStatement,
e.getMessage() == null ? e.getClass().getName() : e.getMessage());
LOGGER.warn(exceptionMessage, e);
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(
RpcUtils.getStatus(
TSStatusCode.LOAD_FILE_ERROR,
String.format(
"Auto create or verify schema error when executing statement %s.",
loadTsFileStatement)));
return analysis;
}

LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed.");

// data partition will be queried in the scheduler
final Analysis analysis = new Analysis();
analysis.setStatement(loadTsFileStatement);
return analysis;
}

@Override
public void close() {
schemaAutoCreatorAndVerifier.close();
}
Expand Down Expand Up @@ -360,8 +367,7 @@ public void autoCreateAndVerify(
* This can only be invoked after all timeseries in the current tsfile have been processed.
* Otherwise, the isAligned status may be wrong.
*/
public void flushAndClearDeviceIsAlignedCacheIfNecessary()
throws SemanticException, AuthException {
public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticException {
// avoid OOM when loading a tsfile with too many timeseries
// or loading too many tsfiles at the same time
schemaCache.clearDeviceIsAlignedCacheIfNecessary();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,12 @@ public class LocalExecutionPlanner {

private static final Logger LOGGER = LoggerFactory.getLogger(LocalExecutionPlanner.class);
private static final long ALLOCATE_MEMORY_FOR_OPERATORS;
private static final long MAX_REST_MEMORY_FOR_LOAD;
private static final long MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;

static {
IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
ALLOCATE_MEMORY_FOR_OPERATORS = CONFIG.getAllocateMemoryForOperators();
MAX_REST_MEMORY_FOR_LOAD =
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD =
(long)
((ALLOCATE_MEMORY_FOR_OPERATORS) * (1.0 - CONFIG.getMaxAllocateMemoryRatioForLoad()));
}
Expand All @@ -69,6 +69,10 @@ public long getFreeMemoryForOperators() {
return freeMemoryForOperators;
}

public long getFreeMemoryForLoadTsFile() {
return freeMemoryForOperators - MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
}

public static LocalExecutionPlanner getInstance() {
return InstanceHolder.INSTANCE;
}
Expand Down Expand Up @@ -211,7 +215,7 @@ private List<PartialPath> collectSourcePaths(LocalExecutionPlanContext context)
}

public synchronized boolean forceAllocateFreeMemoryForOperators(long memoryInBytes) {
if (freeMemoryForOperators - memoryInBytes <= MAX_REST_MEMORY_FOR_LOAD) {
if (freeMemoryForOperators - memoryInBytes <= MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) {
return false;
} else {
freeMemoryForOperators -= memoryInBytes;
Expand All @@ -220,9 +224,9 @@ public synchronized boolean forceAllocateFreeMemoryForOperators(long memoryInByt
}

public synchronized long tryAllocateFreeMemoryForOperators(long memoryInBytes) {
if (freeMemoryForOperators - memoryInBytes <= MAX_REST_MEMORY_FOR_LOAD) {
long result = freeMemoryForOperators - MAX_REST_MEMORY_FOR_LOAD;
freeMemoryForOperators = MAX_REST_MEMORY_FOR_LOAD;
if (freeMemoryForOperators - memoryInBytes <= MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) {
long result = freeMemoryForOperators - MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
freeMemoryForOperators = MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
return result;
} else {
freeMemoryForOperators -= memoryInBytes;
Expand Down