From 21bb6219d44646190a50fcf1b42418545e9b8026 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 14:08:12 +0800 Subject: [PATCH 01/11] setup --- .../env/cluster/node/AbstractNodeWrapper.java | 45 +++++++++++++++++++ .../IoTDBSubscriptionConsumerGroupIT.java | 8 ++++ 2 files changed, 53 insertions(+) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index fa5447bc5b071..1ae4918008df7 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -38,13 +38,16 @@ import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -675,4 +678,46 @@ private String getKillPoints() { public abstract String getSystemPropertiesPath(); protected abstract MppJVMConfig initVMConfig(); + + public void executeJstack() throws Exception { + final long pid = getPidOfProcess(this.instance); + if (pid == -1) { + logger.warn("Failed to get pid for {} before executing jstack", this.getId()); + return; + } + final String command = "jstack -l " + pid; + logger.info("Executing command {} for {}", command, this.getId()); + final Process process = Runtime.getRuntime().exec(command); + final StringBuilder output = new StringBuilder(); + try (final BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append(System.lineSeparator()); + } + } + final int exitCode = process.waitFor(); + logger.info("Command {} exited with code {}", command, exitCode); + logger.info(output.toString()); + } + + /** + * @param process The process to obtain the native process ID from. + * @return The native process ID of the process, -1 if failure. + */ + private static synchronized long getPidOfProcess(final Process process) { + long pid = -1; + try { + if (process.getClass().getName().equals("java.lang.UNIXProcess") + || process.getClass().getName().equals("java.lang.ProcessImpl")) { + final Field f = process.getClass().getDeclaredField("pid"); + f.setAccessible(true); + pid = f.getLong(process); + f.setAccessible(false); + } + } catch (final Exception e) { + pid = -1; + } + return pid; + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 650f861f498d0..787e855c18fd3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1010,6 +1010,7 @@ private void pollMessagesAndCheck( } // Check data on receiver + final long[] currentTime = {System.currentTimeMillis()}; try { try (final Connection connection = receiverEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -1020,6 +1021,13 @@ private void pollMessagesAndCheck( LOGGER.info("detect receiver crashed, skipping this test..."); return; } + // potential stuck + if (System.currentTimeMillis() - currentTime[0] > 60_000L) { + currentTime[0] = System.currentTimeMillis(); + for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { + wrapper.executeJstack(); + } + } TestUtils.assertSingleResultSetEqual( TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), expectedHeaderWithResult); From 6b58679d7feda09561362d07a562d5e7f03822bc Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 16:18:58 +0800 Subject: [PATCH 02/11] tmp --- .../env/cluster/node/AbstractNodeWrapper.java | 20 ++++++---- .../IoTDBSubscriptionConsumerGroupIT.java | 5 ++- .../tsfile/PipeTsFileInsertionEvent.java | 39 ++++++++++++------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 1ae4918008df7..975d8450fa194 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -679,7 +679,7 @@ private String getKillPoints() { protected abstract MppJVMConfig initVMConfig(); - public void executeJstack() throws Exception { + public void executeJstack(final String testCaseName) throws Exception { final long pid = getPidOfProcess(this.instance); if (pid == -1) { logger.warn("Failed to get pid for {} before executing jstack", this.getId()); @@ -688,17 +688,23 @@ public void executeJstack() throws Exception { final String command = "jstack -l " + pid; logger.info("Executing command {} for {}", command, this.getId()); final Process process = Runtime.getRuntime().exec(command); - final StringBuilder output = new StringBuilder(); - try (final BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (final PrintWriter output = + new PrintWriter( + getLogDirPath() + + File.separator + + testCaseName + + "_" + + getId() + + "-threads.jstack"); + final BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { String line; while ((line = reader.readLine()) != null) { output.append(line).append(System.lineSeparator()); } + final int exitCode = process.waitFor(); + logger.info("Command {} exited with code {}", command, exitCode); } - final int exitCode = process.waitFor(); - logger.info("Command {} exited with code {}", command, exitCode); - logger.info(output.toString()); } /** diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 787e855c18fd3..3c4a07c710b14 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1023,10 +1023,11 @@ private void pollMessagesAndCheck( } // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { - currentTime[0] = System.currentTimeMillis(); for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - wrapper.executeJstack(); + wrapper.dumpJVMSnapshot(testName.getMethodName() + currentTime[0]); + wrapper.executeJstack(testName.getMethodName() + currentTime[0]); } + currentTime[0] = System.currentTimeMillis(); } TestUtils.assertSingleResultSetEqual( TestUtils.executeQueryWithRetry(statement, "select count(*) from root.**"), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 55739bf8f3291..c2e985670cb7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -56,6 +56,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent implements TsFileIns private final boolean isGeneratedByPipe; private final AtomicBoolean isClosed; + private final AtomicBoolean tmp = new AtomicBoolean(false); private TsFileInsertionDataContainer dataContainer; public PipeTsFileInsertionEvent( @@ -134,27 +135,37 @@ public PipeTsFileInsertionEvent( * otherwise. */ public boolean waitForTsFileClose() throws InterruptedException { - if (!isClosed.get()) { - isClosed.set(resource.isClosed()); - - synchronized (isClosed) { - while (!isClosed.get()) { - isClosed.wait(100); - - final boolean isClosedNow = resource.isClosed(); - if (isClosedNow) { - isClosed.set(true); - isClosed.notifyAll(); - break; - } + if (!tmp.get()) { + synchronized (tmp) { + while (!tmp.get()) { + tmp.wait(); } } } + return true; + + // if (!isClosed.get()) { + // isClosed.set(resource.isClosed()); + // + // synchronized (isClosed) { + // while (!isClosed.get()) { + // isClosed.wait(100); + // + // final boolean isClosedNow = resource.isClosed(); + // if (isClosedNow) { + // isClosed.set(true); + // isClosed.notifyAll(); + // break; + // } + // } + // } + // } + // From illustrations above we know If the status is "closed", then the tsFile is flushed // And here we guarantee that the isEmpty() is set before flushing if tsFile is empty // Then we know: "isClosed" --> tsFile flushed --> (isEmpty() <--> tsFile is empty) - return !resource.isEmpty(); + // return !resource.isEmpty(); } public File getTsFile() { From 9309bdddbeea23806bdf332e20b2132d0071512f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 17:35:14 +0800 Subject: [PATCH 03/11] improve --- .../env/cluster/node/AbstractNodeWrapper.java | 33 ++++++++++--------- .../IoTDBSubscriptionConsumerGroupIT.java | 4 +-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 975d8450fa194..503c492aedfae 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -45,9 +45,9 @@ import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.MonitorInfo; +import java.lang.management.RuntimeMXBean; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; -import java.lang.reflect.Field; import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -680,7 +680,7 @@ private String getKillPoints() { protected abstract MppJVMConfig initVMConfig(); public void executeJstack(final String testCaseName) throws Exception { - final long pid = getPidOfProcess(this.instance); + final long pid = this.getPid(); if (pid == -1) { logger.warn("Failed to get pid for {} before executing jstack", this.getId()); return; @@ -708,22 +708,25 @@ public void executeJstack(final String testCaseName) throws Exception { } /** - * @param process The process to obtain the native process ID from. * @return The native process ID of the process, -1 if failure. */ - private static synchronized long getPidOfProcess(final Process process) { - long pid = -1; + public long getPid() { + final JMXServiceURL url; try { - if (process.getClass().getName().equals("java.lang.UNIXProcess") - || process.getClass().getName().equals("java.lang.ProcessImpl")) { - final Field f = process.getClass().getDeclaredField("pid"); - f.setAccessible(true); - pid = f.getLong(process); - f.setAccessible(false); - } - } catch (final Exception e) { - pid = -1; + url = + new JMXServiceURL( + String.format("service:jmx:rmi:///jndi/rmi://127.0.0.1:%d/jmxrmi", jmxPort)); + } catch (final MalformedURLException ignored) { + return -1; + } + try (final JMXConnector connector = JMXConnectorFactory.connect(url)) { + final MBeanServerConnection mbsc = connector.getMBeanServerConnection(); + final RuntimeMXBean rmbean = + ManagementFactory.newPlatformMXBeanProxy( + mbsc, ManagementFactory.RUNTIME_MXBEAN_NAME, RuntimeMXBean.class); + return Long.parseLong(rmbean.getName().split("@")[0]); + } catch (final Throwable ignored) { + return -1; } - return pid; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 3c4a07c710b14..209cb8be4a197 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1024,8 +1024,8 @@ private void pollMessagesAndCheck( // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - wrapper.dumpJVMSnapshot(testName.getMethodName() + currentTime[0]); - wrapper.executeJstack(testName.getMethodName() + currentTime[0]); + wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), currentTime[0])); + wrapper.executeJstack(String.format("%s_%s", testName.getMethodName(), currentTime[0])); } currentTime[0] = System.currentTimeMillis(); } From 498d246b4a30583b13ff37e66e288807733d0cfd Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 17:36:41 +0800 Subject: [PATCH 04/11] reset --- .../tsfile/PipeTsFileInsertionEvent.java | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index c2e985670cb7d..55739bf8f3291 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -56,7 +56,6 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent implements TsFileIns private final boolean isGeneratedByPipe; private final AtomicBoolean isClosed; - private final AtomicBoolean tmp = new AtomicBoolean(false); private TsFileInsertionDataContainer dataContainer; public PipeTsFileInsertionEvent( @@ -135,37 +134,27 @@ public PipeTsFileInsertionEvent( * otherwise. */ public boolean waitForTsFileClose() throws InterruptedException { - if (!tmp.get()) { - synchronized (tmp) { - while (!tmp.get()) { - tmp.wait(); + if (!isClosed.get()) { + isClosed.set(resource.isClosed()); + + synchronized (isClosed) { + while (!isClosed.get()) { + isClosed.wait(100); + + final boolean isClosedNow = resource.isClosed(); + if (isClosedNow) { + isClosed.set(true); + isClosed.notifyAll(); + break; + } } } } - return true; - - // if (!isClosed.get()) { - // isClosed.set(resource.isClosed()); - // - // synchronized (isClosed) { - // while (!isClosed.get()) { - // isClosed.wait(100); - // - // final boolean isClosedNow = resource.isClosed(); - // if (isClosedNow) { - // isClosed.set(true); - // isClosed.notifyAll(); - // break; - // } - // } - // } - // } - // From illustrations above we know If the status is "closed", then the tsFile is flushed // And here we guarantee that the isEmpty() is set before flushing if tsFile is empty // Then we know: "isClosed" --> tsFile flushed --> (isEmpty() <--> tsFile is empty) - // return !resource.isEmpty(); + return !resource.isEmpty(); } public File getTsFile() { From a955cad9d25c7cc20aabb927bd24cdafb66aa874 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 17:37:08 +0800 Subject: [PATCH 05/11] comment dumpJVMSnapshot --- .../subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 209cb8be4a197..06def50e0591a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1024,7 +1024,7 @@ private void pollMessagesAndCheck( // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), currentTime[0])); + // wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), currentTime[0])); wrapper.executeJstack(String.format("%s_%s", testName.getMethodName(), currentTime[0])); } currentTime[0] = System.currentTimeMillis(); From 3c6426db710f159338f85039a4787aa69179f9d6 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 17:43:35 +0800 Subject: [PATCH 06/11] spotless --- .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 06def50e0591a..7568836ba1ddc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1024,8 +1024,10 @@ private void pollMessagesAndCheck( // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - // wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), currentTime[0])); - wrapper.executeJstack(String.format("%s_%s", testName.getMethodName(), currentTime[0])); + // wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), + // currentTime[0])); + wrapper.executeJstack( + String.format("%s_%s", testName.getMethodName(), currentTime[0])); } currentTime[0] = System.currentTimeMillis(); } From fddf086365dda4f9d90fb6167a43621ea47964b6 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 20:59:21 +0800 Subject: [PATCH 07/11] replace dumpJVMSnapshot with executeJstack --- .../iotdb/it/env/cluster/env/AbstractEnv.java | 4 +- .../env/cluster/node/AbstractNodeWrapper.java | 41 +++++++++++-------- .../iotdb/itbase/env/BaseNodeWrapper.java | 4 ++ .../IoTDBSubscriptionConsumerGroupIT.java | 2 - 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 981d39dccca70..949a7f21fd2be 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -612,10 +612,10 @@ public void setTestMethodName(String testMethodName) { @Override public void dumpTestJVMSnapshot() { for (ConfigNodeWrapper configNodeWrapper : configNodeWrapperList) { - configNodeWrapper.dumpJVMSnapshot(testMethodName); + configNodeWrapper.executeJstack(testMethodName); } for (DataNodeWrapper dataNodeWrapper : dataNodeWrapperList) { - dataNodeWrapper.dumpJVMSnapshot(testMethodName); + dataNodeWrapper.executeJstack(testMethodName); } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 503c492aedfae..f6405c0e97f20 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -679,37 +679,46 @@ private String getKillPoints() { protected abstract MppJVMConfig initVMConfig(); - public void executeJstack(final String testCaseName) throws Exception { + @Override + public void executeJstack(final String testCaseName) { final long pid = this.getPid(); if (pid == -1) { - logger.warn("Failed to get pid for {} before executing jstack", this.getId()); + logger.warn("Failed to get pid for {} before executing Jstack", this.getId()); return; } final String command = "jstack -l " + pid; logger.info("Executing command {} for {}", command, this.getId()); - final Process process = Runtime.getRuntime().exec(command); - try (final PrintWriter output = - new PrintWriter( - getLogDirPath() - + File.separator - + testCaseName - + "_" - + getId() - + "-threads.jstack"); - final BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line; - while ((line = reader.readLine()) != null) { - output.append(line).append(System.lineSeparator()); + final String fileName = getLogDirPath() + + File.separator + + testCaseName + + "_" + + getId() + + "-threads.jstack"; + try { + final Process process = Runtime.getRuntime().exec(command); + try (final PrintWriter output = new PrintWriter(fileName); + final BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append(System.lineSeparator()); + } } final int exitCode = process.waitFor(); logger.info("Command {} exited with code {}", command, exitCode); + logger.info("Jstack execution output can be found at {}", fileName); + } catch (final IOException e) { + logger.warn("IOException occurred when executing Jstack for {}", this.getId(), e); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("InterruptedException occurred when executing Jstack for {}", this.getId(), e); } } /** * @return The native process ID of the process, -1 if failure. */ + @Override public long getPid() { final JMXServiceURL url; try { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java index cd0eb86bde18c..d11244457a0ca 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java @@ -46,4 +46,8 @@ public interface BaseNodeWrapper { String getIpAndPortString(); void dumpJVMSnapshot(String testCaseName); + + void executeJstack(String testCaseName); + + long getPid(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 7568836ba1ddc..2dfeb11d2cba1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1024,8 +1024,6 @@ private void pollMessagesAndCheck( // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - // wrapper.dumpJVMSnapshot(String.format("%s_%s", testName.getMethodName(), - // currentTime[0])); wrapper.executeJstack( String.format("%s_%s", testName.getMethodName(), currentTime[0])); } From 56a707251294ab5be44688fe334f03b46e2f4b7a Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 13 Jun 2024 20:59:36 +0800 Subject: [PATCH 08/11] spotless --- .../iotdb/it/env/cluster/node/AbstractNodeWrapper.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index f6405c0e97f20..c875f08003592 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -688,12 +688,8 @@ public void executeJstack(final String testCaseName) { } final String command = "jstack -l " + pid; logger.info("Executing command {} for {}", command, this.getId()); - final String fileName = getLogDirPath() - + File.separator - + testCaseName - + "_" - + getId() - + "-threads.jstack"; + final String fileName = + getLogDirPath() + File.separator + testCaseName + "_" + getId() + "-threads.jstack"; try { final Process process = Runtime.getRuntime().exec(command); try (final PrintWriter output = new PrintWriter(fileName); From ff392cc0ef8f9598d8fab69e02ec8204086498f8 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 19:05:33 +0800 Subject: [PATCH 09/11] intro executeJstack() --- .../env/cluster/node/AbstractNodeWrapper.java | 28 +++++++++++++------ .../iotdb/itbase/env/BaseNodeWrapper.java | 2 +- .../IoTDBSubscriptionConsumerGroupIT.java | 5 ++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index c875f08003592..6d433ab365308 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -64,6 +64,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.stream.Stream; import static org.apache.iotdb.it.env.cluster.ClusterConstant.CLUSTER_CONFIGURATIONS; @@ -569,7 +570,6 @@ public String getNodePath() { return System.getProperty(USER_DIR) + File.separator + TARGET + File.separator + getId(); } - @Override public void dumpJVMSnapshot(String testCaseName) { JMXServiceURL url; try { @@ -679,8 +679,24 @@ private String getKillPoints() { protected abstract MppJVMConfig initVMConfig(); + @Override + public void executeJstack() { + executeJstack(logger::info); + } + @Override public void executeJstack(final String testCaseName) { + final String fileName = + getLogDirPath() + File.separator + testCaseName + "_" + getId() + "-threads.jstack"; + try (final PrintWriter output = new PrintWriter(fileName)) { + executeJstack(output::println); + } catch (final IOException e) { + logger.warn("IOException occurred when executing Jstack for {}", this.getId(), e); + } + logger.info("Jstack execution output can be found at {}", fileName); + } + + private void executeJstack(Consumer consumer) { final long pid = this.getPid(); if (pid == -1) { logger.warn("Failed to get pid for {} before executing Jstack", this.getId()); @@ -688,21 +704,17 @@ public void executeJstack(final String testCaseName) { } final String command = "jstack -l " + pid; logger.info("Executing command {} for {}", command, this.getId()); - final String fileName = - getLogDirPath() + File.separator + testCaseName + "_" + getId() + "-threads.jstack"; try { final Process process = Runtime.getRuntime().exec(command); - try (final PrintWriter output = new PrintWriter(fileName); - final BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (final BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { String line; while ((line = reader.readLine()) != null) { - output.append(line).append(System.lineSeparator()); + consumer.accept(line); } } final int exitCode = process.waitFor(); logger.info("Command {} exited with code {}", command, exitCode); - logger.info("Jstack execution output can be found at {}", fileName); } catch (final IOException e) { logger.warn("IOException occurred when executing Jstack for {}", this.getId(), e); } catch (final InterruptedException e) { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java index d11244457a0ca..165239d546ff5 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java @@ -45,7 +45,7 @@ public interface BaseNodeWrapper { String getIpAndPortString(); - void dumpJVMSnapshot(String testCaseName); + void executeJstack(); void executeJstack(String testCaseName); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java index 2dfeb11d2cba1..e332574925004 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java @@ -1024,8 +1024,9 @@ private void pollMessagesAndCheck( // potential stuck if (System.currentTimeMillis() - currentTime[0] > 60_000L) { for (final DataNodeWrapper wrapper : senderEnv.getDataNodeWrapperList()) { - wrapper.executeJstack( - String.format("%s_%s", testName.getMethodName(), currentTime[0])); + wrapper.executeJstack(); + // wrapper.executeJstack(String.format("%s_%s", testName.getMethodName(), + // currentTime[0])); } currentTime[0] = System.currentTimeMillis(); } From fc647ddfe6a951f18d5a9278ba1bc3faf13c1385 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 21:17:49 +0800 Subject: [PATCH 10/11] add comments --- .../org/apache/iotdb/itbase/env/BaseNodeWrapper.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java index 165239d546ff5..9fe410703e9bb 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java @@ -45,8 +45,18 @@ public interface BaseNodeWrapper { String getIpAndPortString(); + /** + * Perform jstack on the process corresponding to the wrapper, and use logger to output the + * results. + */ void executeJstack(); + /** + * Perform jstack on the process corresponding to the wrapper, and output the results to a file in + * the log directory. + * + * @param testCaseName the name of test case + */ void executeJstack(String testCaseName); long getPid(); From 13b7e39d0bc32327781d3c5fa91023b460753957 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 14 Jun 2024 21:34:13 +0800 Subject: [PATCH 11/11] add finals --- .../apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java | 2 +- .../main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java index 6d433ab365308..942b6a9e01242 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java @@ -696,7 +696,7 @@ public void executeJstack(final String testCaseName) { logger.info("Jstack execution output can be found at {}", fileName); } - private void executeJstack(Consumer consumer) { + private void executeJstack(final Consumer consumer) { final long pid = this.getPid(); if (pid == -1) { logger.warn("Failed to get pid for {} before executing Jstack", this.getId()); diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java index 9fe410703e9bb..b0ce0d4a22147 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseNodeWrapper.java @@ -57,7 +57,7 @@ public interface BaseNodeWrapper { * * @param testCaseName the name of test case */ - void executeJstack(String testCaseName); + void executeJstack(final String testCaseName); long getPid(); }