From b14780b93878acef07da2860ed7892b4a5a6ef51 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:21:38 +0800 Subject: [PATCH 1/5] Added log --- .../pipe/pattern/IoTDBPipePattern.java | 4 ++- .../task/subtask/PipeReportableSubtask.java | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/pattern/IoTDBPipePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/pattern/IoTDBPipePattern.java index 6c82c7e3d9ccc..30307d0771121 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/pattern/IoTDBPipePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/pattern/IoTDBPipePattern.java @@ -50,7 +50,9 @@ public IoTDBPipePattern(final String pattern) { public static List applyIndexesOnList( final int[] filteredIndexes, final List originalList) { - return Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()); + return Objects.nonNull(originalList) + ? Arrays.stream(filteredIndexes).mapToObj(originalList::get).collect(Collectors.toList()) + : null; } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeReportableSubtask.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeReportableSubtask.java index 327152dd0c37e..462644db4df02 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeReportableSubtask.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeReportableSubtask.java @@ -31,12 +31,12 @@ public abstract class PipeReportableSubtask extends PipeSubtask { private static final Logger LOGGER = LoggerFactory.getLogger(PipeReportableSubtask.class); - protected PipeReportableSubtask(String taskID, long creationTime) { + protected PipeReportableSubtask(final String taskID, final long creationTime) { super(taskID, creationTime); } @Override - public synchronized void onFailure(Throwable throwable) { + public synchronized void onFailure(final Throwable throwable) { if (isClosed.get()) { LOGGER.info("onFailure in pipe subtask, ignored because pipe is dropped.", throwable); clearReferenceCountAndReleaseLastEvent(); @@ -55,7 +55,7 @@ public synchronized void onFailure(Throwable throwable) { // is dropped or the process is running normally. } - private void onEnrichedEventFailure(Throwable throwable) { + private void onEnrichedEventFailure(final Throwable throwable) { final int maxRetryTimes = throwable instanceof PipeRuntimeConnectorRetryTimesConfigurableException ? ((PipeRuntimeConnectorRetryTimesConfigurableException) throwable).getRetryTimes() @@ -75,15 +75,17 @@ private void onEnrichedEventFailure(Throwable throwable) { retryCount.incrementAndGet(); if (retryCount.get() <= MAX_RETRY_TIMES) { LOGGER.warn( - "Retry executing subtask {} (creation time: {}, simple class: {}), retry count [{}/{}]", + "Retry executing subtask {} (creation time: {}, simple class: {}), retry count [{}/{}], last exception: {}", taskID, creationTime, this.getClass().getSimpleName(), retryCount.get(), - maxRetryTimes); + maxRetryTimes, + throwable.getMessage(), + throwable); try { Thread.sleep(Math.min(1000L * retryCount.get(), 10000)); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { LOGGER.warn( "Interrupted when retrying to execute subtask {} (creation time: {}, simple class: {})", taskID, @@ -123,11 +125,11 @@ private void onEnrichedEventFailure(Throwable throwable) { } } - protected abstract String getRootCause(Throwable throwable); + protected abstract String getRootCause(final Throwable throwable); - protected abstract void report(EnrichedEvent event, PipeRuntimeException exception); + protected abstract void report(final EnrichedEvent event, final PipeRuntimeException exception); - private void onNonEnrichedEventFailure(Throwable throwable) { + private void onNonEnrichedEventFailure(final Throwable throwable) { if (retryCount.get() == 0) { LOGGER.warn( "Failed to execute subtask {} (creation time: {}, simple class: {}), " @@ -141,14 +143,16 @@ private void onNonEnrichedEventFailure(Throwable throwable) { retryCount.incrementAndGet(); LOGGER.warn( - "Retry executing subtask {} (creation time: {}, simple class: {}), retry count {}", + "Retry executing subtask {} (creation time: {}, simple class: {}), retry count {}, last exception: {}", taskID, creationTime, this.getClass().getSimpleName(), - retryCount.get()); + retryCount.get(), + throwable.getMessage(), + throwable); try { Thread.sleep(Math.min(1000L * retryCount.get(), 10000)); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { LOGGER.warn( "Interrupted when retrying to execute subtask {} (creation time: {}, simple class: {})", taskID, From 04932deb488eed06c70b31aa332a50c191674371 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:42:58 +0800 Subject: [PATCH 2/5] Update PipeAbstractConnectorSubtask.java --- .../pipe/task/subtask/PipeAbstractConnectorSubtask.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java index f5bfe36c6c732..791e02ebceb4d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java @@ -68,13 +68,11 @@ public void bindExecutors( @Override public Boolean call() throws Exception { - final boolean hasAtLeastOneEventProcessed = super.call(); - // Wait for the callable to be decorated by Futures.addCallback in the executorService // to make sure that the callback can be submitted again on success or failure. callbackDecoratingLock.waitForDecorated(); - return hasAtLeastOneEventProcessed; + return super.call(); } @Override From a85649080fde78002b91e280abe9fdf123d5cf7c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:46:57 +0800 Subject: [PATCH 3/5] Improved logs --- .../PipeConfigNodeSubtaskExecutor.java | 4 +- .../PipeConnectorSubtaskExecutor.java | 2 + .../threadpool/WrappedThreadPoolExecutor.java | 7 ++- .../executor/PipeSubtaskExecutor.java | 18 ++++---- .../commons/pipe/task/DecoratingLock.java | 44 ------------------- .../subtask/PipeAbstractConnectorSubtask.java | 40 +++++------------ 6 files changed, 32 insertions(+), 83 deletions(-) delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/DecoratingLock.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java index 9a82e72ece3d7..72474d0e168ce 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.manager.pipe.execution; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.pipe.execution.executor.PipeSubtaskExecutor; import org.apache.iotdb.commons.utils.TestOnly; @@ -29,13 +30,14 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor { private PipeConfigNodeSubtaskExecutor() { super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL); + ((WrappedThreadPoolExecutor) subtaskWorkerThreadPoolExecutor).disableErrorLog(); } /** * @param ignored Used to distinguish this constructor from the default constructor. */ @TestOnly - public PipeConfigNodeSubtaskExecutor(Object ignored) { + public PipeConfigNodeSubtaskExecutor(final Object ignored) { super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java index ce832091c1dc4..c885b06a868c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.execution; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.execution.executor.PipeSubtaskExecutor; @@ -29,6 +30,7 @@ public PipeConnectorSubtaskExecutor() { super( PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(), ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL); + ((WrappedThreadPoolExecutor) subtaskWorkerThreadPoolExecutor).disableErrorLog(); } public PipeConnectorSubtaskExecutor(int corePoolSize, ThreadName threadName) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java index f35eba3e7bbe2..2c24f045ca533 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/threadpool/WrappedThreadPoolExecutor.java @@ -41,6 +41,7 @@ public class WrappedThreadPoolExecutor extends ThreadPoolExecutor implements WrappedThreadPoolExecutorMBean { private static final Logger logger = LoggerFactory.getLogger(WrappedThreadPoolExecutor.class); private final String mbeanName; + private volatile boolean disableErrorLog = false; public WrappedThreadPoolExecutor( int corePoolSize, @@ -125,8 +126,12 @@ protected void afterExecute(Runnable r, Throwable t) { Thread.currentThread().interrupt(); } } - if (t != null) { + if (t != null && !disableErrorLog) { logger.error("Exception in thread pool {}", mbeanName, t); } } + + public void disableErrorLog() { + disableErrorLog = true; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java index e990318660b30..c7ae695bb9d8c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java @@ -41,14 +41,14 @@ public abstract class PipeSubtaskExecutor { private static final ExecutorService subtaskCallbackListeningExecutor = IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL.getName()); - private final ListeningExecutorService subtaskWorkerThreadPoolExecutor; + protected final ListeningExecutorService subtaskWorkerThreadPoolExecutor; private final Map registeredIdSubtaskMapper; private final int corePoolSize; private int runningSubtaskNumber; - protected PipeSubtaskExecutor(int corePoolSize, ThreadName threadName) { + protected PipeSubtaskExecutor(final int corePoolSize, final ThreadName threadName) { subtaskWorkerThreadPoolExecutor = MoreExecutors.listeningDecorator( IoTDBThreadPoolFactory.newFixedThreadPool(corePoolSize, threadName.getName())); @@ -61,7 +61,7 @@ protected PipeSubtaskExecutor(int corePoolSize, ThreadName threadName) { /////////////////////// Subtask management /////////////////////// - public final synchronized void register(PipeSubtask subtask) { + public final synchronized void register(final PipeSubtask subtask) { if (registeredIdSubtaskMapper.containsKey(subtask.getTaskID())) { LOGGER.warn("The subtask {} is already registered.", subtask.getTaskID()); return; @@ -74,7 +74,7 @@ public final synchronized void register(PipeSubtask subtask) { new PipeSubtaskScheduler(this)); } - public final synchronized void start(String subTaskID) { + public final synchronized void start(final String subTaskID) { if (!registeredIdSubtaskMapper.containsKey(subTaskID)) { LOGGER.warn("The subtask {} is not registered.", subTaskID); return; @@ -93,7 +93,7 @@ public final synchronized void start(String subTaskID) { } } - public final synchronized void stop(String subTaskID) { + public final synchronized void stop(final String subTaskID) { if (!registeredIdSubtaskMapper.containsKey(subTaskID)) { LOGGER.warn("The subtask {} is not registered.", subTaskID); return; @@ -104,7 +104,7 @@ public final synchronized void stop(String subTaskID) { } } - public final synchronized void deregister(String subTaskID) { + public final synchronized void deregister(final String subTaskID) { stop(subTaskID); final PipeSubtask subtask = registeredIdSubtaskMapper.remove(subTaskID); @@ -113,14 +113,14 @@ public final synchronized void deregister(String subTaskID) { try { subtask.close(); LOGGER.info("The subtask {} is closed successfully.", subTaskID); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.error("Failed to close the subtask {}.", subTaskID, e); } } } @TestOnly - public final boolean isRegistered(String subTaskID) { + public final boolean isRegistered(final String subTaskID) { return registeredIdSubtaskMapper.containsKey(subTaskID); } @@ -137,7 +137,7 @@ public final synchronized void shutdown() { } // stop all subtasks before shutting down the executor - for (PipeSubtask subtask : registeredIdSubtaskMapper.values()) { + for (final PipeSubtask subtask : registeredIdSubtaskMapper.values()) { subtask.disallowSubmittingSelf(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/DecoratingLock.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/DecoratingLock.java deleted file mode 100644 index cc5d0d93e5eee..0000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/DecoratingLock.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.pipe.task; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class DecoratingLock { - private final AtomicBoolean isDecorating = new AtomicBoolean(false); - - public void waitForDecorated() { - while (isDecorating.get()) { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - - public void markAsDecorating() { - isDecorating.set(true); - } - - public void markAsDecorated() { - isDecorating.set(false); - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java index 791e02ebceb4d..78e913d65757c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/subtask/PipeAbstractConnectorSubtask.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.execution.scheduler.PipeSubtaskScheduler; -import org.apache.iotdb.commons.pipe.task.DecoratingLock; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.exception.PipeConnectionException; @@ -43,7 +42,6 @@ public abstract class PipeAbstractConnectorSubtask extends PipeReportableSubtask protected PipeConnector outputPipeConnector; // For thread pool to execute callbacks - protected final DecoratingLock callbackDecoratingLock = new DecoratingLock(); protected ExecutorService subtaskCallbackListeningExecutor; // For controlling subtask submitting, making sure that @@ -51,39 +49,30 @@ public abstract class PipeAbstractConnectorSubtask extends PipeReportableSubtask protected volatile boolean isSubmitted = false; protected PipeAbstractConnectorSubtask( - String taskID, long creationTime, PipeConnector outputPipeConnector) { + final String taskID, final long creationTime, final PipeConnector outputPipeConnector) { super(taskID, creationTime); this.outputPipeConnector = outputPipeConnector; } @Override public void bindExecutors( - ListeningExecutorService subtaskWorkerThreadPoolExecutor, - ExecutorService subtaskCallbackListeningExecutor, - PipeSubtaskScheduler subtaskScheduler) { + final ListeningExecutorService subtaskWorkerThreadPoolExecutor, + final ExecutorService subtaskCallbackListeningExecutor, + final PipeSubtaskScheduler subtaskScheduler) { this.subtaskWorkerThreadPoolExecutor = subtaskWorkerThreadPoolExecutor; this.subtaskCallbackListeningExecutor = subtaskCallbackListeningExecutor; this.subtaskScheduler = subtaskScheduler; } @Override - public Boolean call() throws Exception { - // Wait for the callable to be decorated by Futures.addCallback in the executorService - // to make sure that the callback can be submitted again on success or failure. - callbackDecoratingLock.waitForDecorated(); - - return super.call(); - } - - @Override - public synchronized void onSuccess(Boolean hasAtLeastOneEventProcessed) { + public synchronized void onSuccess(final Boolean hasAtLeastOneEventProcessed) { isSubmitted = false; super.onSuccess(hasAtLeastOneEventProcessed); } @Override - public synchronized void onFailure(Throwable throwable) { + public synchronized void onFailure(final Throwable throwable) { isSubmitted = false; if (isClosed.get()) { @@ -114,7 +103,7 @@ public synchronized void onFailure(Throwable throwable) { /** * @return {@code true} if the {@link PipeSubtask} should be stopped, {@code false} otherwise */ - private boolean onPipeConnectionException(Throwable throwable) { + private boolean onPipeConnectionException(final Throwable throwable) { LOGGER.warn( "PipeConnectionException occurred, {} retries to handshake with the target system.", outputPipeConnector.getClass().getName(), @@ -128,7 +117,7 @@ private boolean onPipeConnectionException(Throwable throwable) { "{} handshakes with the target system successfully.", outputPipeConnector.getClass().getName()); break; - } catch (Exception e) { + } catch (final Exception e) { retry++; LOGGER.warn( "{} failed to handshake with the target system for {} times, " @@ -139,7 +128,7 @@ private boolean onPipeConnectionException(Throwable throwable) { e); try { Thread.sleep(retry * PipeConfig.getInstance().getPipeConnectorRetryIntervalMs()); - } catch (InterruptedException interruptedException) { + } catch (final InterruptedException interruptedException) { LOGGER.info( "Interrupted while sleeping, will retry to handshake with the target system.", interruptedException); @@ -193,13 +182,8 @@ public synchronized void submitSelf() { return; } - callbackDecoratingLock.markAsDecorating(); - try { - final ListenableFuture nextFuture = subtaskWorkerThreadPoolExecutor.submit(this); - Futures.addCallback(nextFuture, this, subtaskCallbackListeningExecutor); - isSubmitted = true; - } finally { - callbackDecoratingLock.markAsDecorated(); - } + final ListenableFuture nextFuture = subtaskWorkerThreadPoolExecutor.submit(this); + Futures.addCallback(nextFuture, this, subtaskCallbackListeningExecutor); + isSubmitted = true; } } From 21abc91f2553de51425b018c585d7bbec5c19b96 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:09:17 +0800 Subject: [PATCH 4/5] bug fix --- .../execution/PipeConfigNodeSubtaskExecutor.java | 6 ++---- .../consensus/PipeConsensusSubtaskExecutor.java | 4 ++-- .../execution/PipeConnectorSubtaskExecutor.java | 9 ++++----- .../execution/PipeProcessorSubtaskExecutor.java | 3 ++- .../pipe/task/builder/PipeDataNodeTaskBuilder.java | 2 +- .../execution/executor/PipeSubtaskExecutor.java | 14 ++++++++++---- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java index 72474d0e168ce..6041c5bf82da8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/execution/PipeConfigNodeSubtaskExecutor.java @@ -20,7 +20,6 @@ package org.apache.iotdb.confignode.manager.pipe.execution; import org.apache.iotdb.commons.concurrent.ThreadName; -import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.pipe.execution.executor.PipeSubtaskExecutor; import org.apache.iotdb.commons.utils.TestOnly; @@ -29,8 +28,7 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor { private static final int THREAD_NUM = 1; private PipeConfigNodeSubtaskExecutor() { - super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL); - ((WrappedThreadPoolExecutor) subtaskWorkerThreadPoolExecutor).disableErrorLog(); + super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true); } /** @@ -38,7 +36,7 @@ private PipeConfigNodeSubtaskExecutor() { */ @TestOnly public PipeConfigNodeSubtaskExecutor(final Object ignored) { - super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL); + super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true); } private static class PipeSchemaSubtaskExecutorHolder { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java index 5d99c5038fc67..175a95992cfa9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java @@ -20,9 +20,9 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.execution.executor.PipeSubtaskExecutor; +import org.apache.iotdb.db.pipe.execution.PipeConnectorSubtaskExecutor; -public class PipeConsensusSubtaskExecutor extends PipeSubtaskExecutor { +public class PipeConsensusSubtaskExecutor extends PipeConnectorSubtaskExecutor { public PipeConsensusSubtaskExecutor() { super( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java index c885b06a868c7..ea03c8ce4a008 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeConnectorSubtaskExecutor.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.pipe.execution; import org.apache.iotdb.commons.concurrent.ThreadName; -import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.execution.executor.PipeSubtaskExecutor; @@ -29,11 +28,11 @@ public class PipeConnectorSubtaskExecutor extends PipeSubtaskExecutor { public PipeConnectorSubtaskExecutor() { super( PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(), - ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL); - ((WrappedThreadPoolExecutor) subtaskWorkerThreadPoolExecutor).disableErrorLog(); + ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL, + true); } - public PipeConnectorSubtaskExecutor(int corePoolSize, ThreadName threadName) { - super(corePoolSize, threadName); + public PipeConnectorSubtaskExecutor(final int corePoolSize, final ThreadName threadName) { + super(corePoolSize, threadName, true); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeProcessorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeProcessorSubtaskExecutor.java index a4b36d3664cea..6d5a8ca89fc9f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeProcessorSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/execution/PipeProcessorSubtaskExecutor.java @@ -28,6 +28,7 @@ public class PipeProcessorSubtaskExecutor extends PipeSubtaskExecutor { public PipeProcessorSubtaskExecutor() { super( PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(), - ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL); + ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL, + false); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java index e84f93e34ce33..9d29993271156 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java @@ -54,7 +54,7 @@ public class PipeDataNodeTaskBuilder { CONNECTOR_EXECUTOR_MAP.put( PipeType.SUBSCRIPTION, PipeSubtaskExecutorManager.getInstance().getSubscriptionExecutor()); CONNECTOR_EXECUTOR_MAP.put( - PipeType.CONSENSUS, PipeSubtaskExecutorManager.getInstance().getConnectorExecutor()); + PipeType.CONSENSUS, PipeSubtaskExecutorManager.getInstance().getConsensusExecutor()); } protected final Map systemParameters = new HashMap<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java index c7ae695bb9d8c..886538050673e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/execution/executor/PipeSubtaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.pipe.execution.scheduler.PipeSubtaskScheduler; import org.apache.iotdb.commons.pipe.task.subtask.PipeSubtask; import org.apache.iotdb.commons.utils.TestOnly; @@ -48,10 +49,15 @@ public abstract class PipeSubtaskExecutor { private final int corePoolSize; private int runningSubtaskNumber; - protected PipeSubtaskExecutor(final int corePoolSize, final ThreadName threadName) { - subtaskWorkerThreadPoolExecutor = - MoreExecutors.listeningDecorator( - IoTDBThreadPoolFactory.newFixedThreadPool(corePoolSize, threadName.getName())); + protected PipeSubtaskExecutor( + final int corePoolSize, final ThreadName threadName, final boolean disableLogInThreadPool) { + final WrappedThreadPoolExecutor executor = + (WrappedThreadPoolExecutor) + IoTDBThreadPoolFactory.newFixedThreadPool(corePoolSize, threadName.getName()); + if (disableLogInThreadPool) { + executor.disableErrorLog(); + } + subtaskWorkerThreadPoolExecutor = MoreExecutors.listeningDecorator(executor); registeredIdSubtaskMapper = new ConcurrentHashMap<>(); From 4e4677d15c8507d6c33952e44550adaaca25d55f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:16:53 +0800 Subject: [PATCH 5/5] Update PipeDataNodeTaskBuilder.java --- .../iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java index 9d29993271156..04d5ece6e3b02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/builder/PipeDataNodeTaskBuilder.java @@ -89,7 +89,7 @@ public PipeDataNodeTask build() { blendUserAndSystemParameters(pipeStaticMeta.getConnectorParameters()), regionId, CONNECTOR_EXECUTOR_MAP.get(pipeType)); - } else { // user pipe + } else { // user pipe or consensus pipe connectorStage = new PipeTaskConnectorStage( pipeStaticMeta.getPipeName(),