From 10311f8c64fbc4ea0e41df1e73aabf97e3cb4fb5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:16:25 +0800 Subject: [PATCH 1/2] bug fix --- .../iotdb/db/pipe/task/connection/PipeEventCollector.java | 7 ++++++- .../pipe/task/subtask/connector/PipeConnectorSubtask.java | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java index 1df20db17f635..f11fad9cb9a74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.pipe.api.collector.EventCollector; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; @@ -71,7 +72,9 @@ public void collect(final Event event) { parseAndCollectEvent((PipeRawTabletInsertionEvent) event); } else if (event instanceof PipeTsFileInsertionEvent) { parseAndCollectEvent((PipeTsFileInsertionEvent) event); - } else if (event instanceof PipeSchemaRegionWritePlanEvent) { + } else if (event instanceof PipeSchemaRegionWritePlanEvent + && ((PipeSchemaRegionWritePlanEvent) event).getPlanNode().getType() + == PlanNodeType.DELETE_DATA) { parseAndCollectEvent((PipeSchemaRegionWritePlanEvent) event); } else if (!(event instanceof ProgressReportEvent)) { collectEvent(event); @@ -128,6 +131,8 @@ private void parseAndCollectEvent(final PipeTsFileInsertionEvent sourceEvent) th } private void parseAndCollectEvent(final PipeSchemaRegionWritePlanEvent deleteDataEvent) { + // No need to bind progress index here since delete data event does not have progress index + // currently IoTDBSchemaRegionExtractor.PATTERN_PARSE_VISITOR .process(deleteDataEvent.getPlanNode(), (IoTDBPipePattern) deleteDataEvent.getPipePattern()) .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java index 40ac29eaa8198..295ce42a19d5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.pipe.metric.PipeDataRegionConnectorMetrics; import org.apache.iotdb.db.pipe.metric.PipeSchemaRegionConnectorMetrics; import org.apache.iotdb.db.pipe.task.connection.PipeEventCollector; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.utils.ErrorHandlingUtils; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.event.Event; @@ -115,7 +116,10 @@ protected boolean executeOnce() { PipeDataRegionConnectorMetrics.getInstance().markTsFileEvent(taskID); } else if (event instanceof PipeSchemaRegionWritePlanEvent) { outputPipeConnector.transfer(event); - PipeSchemaRegionConnectorMetrics.getInstance().markSchemaEvent(taskID); + if (((PipeSchemaRegionWritePlanEvent) event).getPlanNode().getType() + != PlanNodeType.DELETE_DATA) { + PipeSchemaRegionConnectorMetrics.getInstance().markSchemaEvent(taskID); + } } else if (event instanceof PipeHeartbeatEvent) { transferHeartbeatEvent((PipeHeartbeatEvent) event); } else { From 74095056f0d7b3a2791218bf7573a632097c761b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 12 Jun 2024 19:27:11 +0800 Subject: [PATCH 2/2] Added logs --- .../iotdb/db/pipe/task/connection/PipeEventCollector.java | 6 ++++-- .../pipe/task/subtask/connector/PipeConnectorSubtask.java | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java index f11fad9cb9a74..3055568aadf93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/connection/PipeEventCollector.java @@ -75,6 +75,8 @@ public void collect(final Event event) { } else if (event instanceof PipeSchemaRegionWritePlanEvent && ((PipeSchemaRegionWritePlanEvent) event).getPlanNode().getType() == PlanNodeType.DELETE_DATA) { + // This is only for delete data node in data region since plan nodes in schema regions are + // already parsed in schema region extractor parseAndCollectEvent((PipeSchemaRegionWritePlanEvent) event); } else if (!(event instanceof ProgressReportEvent)) { collectEvent(event); @@ -131,8 +133,8 @@ private void parseAndCollectEvent(final PipeTsFileInsertionEvent sourceEvent) th } private void parseAndCollectEvent(final PipeSchemaRegionWritePlanEvent deleteDataEvent) { - // No need to bind progress index here since delete data event does not have progress index - // currently + // Only used by events containing delete data node, no need to bind progress index here since + // delete data event does not have progress index currently IoTDBSchemaRegionExtractor.PATTERN_PARSE_VISITOR .process(deleteDataEvent.getPlanNode(), (IoTDBPipePattern) deleteDataEvent.getPipePattern()) .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java index 295ce42a19d5e..c2f7adaf8dd4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java @@ -118,6 +118,8 @@ protected boolean executeOnce() { outputPipeConnector.transfer(event); if (((PipeSchemaRegionWritePlanEvent) event).getPlanNode().getType() != PlanNodeType.DELETE_DATA) { + // Only plan nodes in schema region will be marked, delete data node is currently not + // taken into account PipeSchemaRegionConnectorMetrics.getInstance().markSchemaEvent(taskID); } } else if (event instanceof PipeHeartbeatEvent) {