Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.Map;

public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT {

@Test
public void testAutoDropInHistoricalTransfer() throws Exception {
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
Expand Down Expand Up @@ -70,10 +71,69 @@ public void testAutoDropInHistoricalTransfer() throws Exception {
Assert.assertEquals(
TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode());

TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select count(*) from root.**",
"count(root.db.d1.s1),",
Collections.singleton("1,"));

TestUtils.assertDataEventuallyOnEnv(
senderEnv,
"show pipes",
"ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,",
Collections.emptySet());
}
}

@Test
public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception {
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);

final String receiverIp = receiverDataNode.getIp();
final int receiverPort = receiverDataNode.getPort();

try (final SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {

if (!TestUtils.tryExecuteNonQueryWithRetry(
senderEnv,
"insert into root.db.d1(time, s1) values (1000, 1), (2000, 2), (3000, 3), (4000, 4), (5000, 5)")) {
return;
}

final Map<String, String> extractorAttributes = new HashMap<>();
final Map<String, String> processorAttributes = new HashMap<>();
final Map<String, String> connectorAttributes = new HashMap<>();

extractorAttributes.put("extractor.mode", "query");
extractorAttributes.put("extractor.start-time", "1970-01-01T08:00:02+08:00");
extractorAttributes.put("extractor.end-time", "1970-01-01T08:00:04+08:00");

connectorAttributes.put("connector", "iotdb-thrift-connector");
connectorAttributes.put("connector.batch.enable", "false");
connectorAttributes.put("connector.ip", receiverIp);
connectorAttributes.put("connector.port", Integer.toString(receiverPort));

final TSStatus status =
client.createPipe(
new TCreatePipeReq("p1", connectorAttributes)
.setExtractorAttributes(extractorAttributes)
.setProcessorAttributes(processorAttributes));

Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
Assert.assertEquals(
TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode());

TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select count(*) from root.**",
"count(root.db.d1.s1),",
Collections.singleton("3,"));

TestUtils.assertDataEventuallyOnEnv(
senderEnv,
"show pipes",
"ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,",
"ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,",
Collections.emptySet());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.LockSupport;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -181,7 +182,7 @@ public void testBasicSubscribeTsFile() throws Exception {
}

// Create topic
final String topicName = "topic2";
final String topicName = "topic1";
final String host = EnvFactory.getEnv().getIP();
final int port = Integer.parseInt(EnvFactory.getEnv().getPort());
try (final SubscriptionSession session = new SubscriptionSession(host, port)) {
Expand Down Expand Up @@ -297,11 +298,12 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception {
}

// Create topic
final String topicName = "topic1";
final String host = EnvFactory.getEnv().getIP();
final int port = Integer.parseInt(EnvFactory.getEnv().getPort());
try (final SubscriptionSession session = new SubscriptionSession(host, port)) {
session.open();
session.createTopic("topic1");
session.createTopic(topicName);
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
Expand All @@ -325,7 +327,7 @@ public void testBasicPullConsumerWithCommitAsync() throws Exception {
.autoCommit(false)
.buildPullConsumer()) {
consumer.open();
consumer.subscribe("topic1");
consumer.subscribe(topicName);
while (!isClosed.get()) {
LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time
final List<SubscriptionMessage> messages =
Expand Down Expand Up @@ -370,7 +372,7 @@ public void onFailure(final Throwable e) {
}
});
}
consumer.unsubscribe("topic1");
consumer.unsubscribe(topicName);
} catch (final Exception e) {
e.printStackTrace();
// avoid fail
Expand Down Expand Up @@ -450,11 +452,12 @@ public void testBasicPushConsumer() {
}

// Create topic
final String topicName = "topic1";
final String host = EnvFactory.getEnv().getIP();
final int port = Integer.parseInt(EnvFactory.getEnv().getPort());
try (final SubscriptionSession session = new SubscriptionSession(host, port)) {
session.open();
session.createTopic("topic1");
session.createTopic(topicName);
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
Expand All @@ -480,7 +483,7 @@ public void testBasicPushConsumer() {
.buildPushConsumer()) {

consumer.open();
consumer.subscribe("topic1");
consumer.subscribe(topicName);

// The push consumer should automatically poll 10 rows of data by 1 onReceive()
Awaitility.await()
Expand Down Expand Up @@ -537,11 +540,112 @@ public void testBasicPushConsumer() {
Assert.assertTrue(onReceiveCount.get() > lastOnReceiveCount.get());
});

consumer.unsubscribe("topic1");
consumer.unsubscribe(topicName);
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void testPollUnsubscribedTopics() throws Exception {
// Insert some historical data
try (final ISession session = EnvFactory.getEnv().getSessionConnection()) {
for (int i = 0; i < 100; ++i) {
session.executeNonQueryStatement(
String.format("insert into root.db.d1(time, s1) values (%s, 1)", i));
}
for (int i = 100; i < 200; ++i) {
session.executeNonQueryStatement(
String.format("insert into root.db.d1(time, s1) values (%s, 1)", i));
}
session.executeNonQueryStatement("flush");
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
}

// Create topic
final String host = EnvFactory.getEnv().getIP();
final int port = Integer.parseInt(EnvFactory.getEnv().getPort());
try (final SubscriptionSession session = new SubscriptionSession(host, port)) {
session.open();
{
final Properties properties = new Properties();
properties.put(TopicConstant.END_TIME_KEY, 99);
session.createTopic("topic1", properties);
}
{
final Properties properties = new Properties();
properties.put(TopicConstant.START_TIME_KEY, 100);
session.createTopic("topic2", properties);
}
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
}

// Subscription
final AtomicInteger rowCount = new AtomicInteger();
final AtomicLong timestampSum = new AtomicLong();
final AtomicBoolean isClosed = new AtomicBoolean(false);
final Thread thread =
new Thread(
() -> {
try (final SubscriptionPullConsumer consumer =
new SubscriptionPullConsumer.Builder()
.host(host)
.port(port)
.consumerId("c1")
.consumerGroupId("cg1")
.autoCommit(false)
.buildPullConsumer()) {
consumer.open();
consumer.subscribe("topic2"); // only subscribe topic2
while (!isClosed.get()) {
LockSupport.parkNanos(IoTDBSubscriptionITConstant.SLEEP_NS); // wait some time
final List<SubscriptionMessage> messages =
consumer.poll(IoTDBSubscriptionITConstant.POLL_TIMEOUT_MS);
for (final SubscriptionMessage message : messages) {
for (final SubscriptionSessionDataSet dataSet :
message.getSessionDataSetsHandler()) {
while (dataSet.hasNext()) {
timestampSum.getAndAdd(dataSet.next().getTimestamp());
rowCount.addAndGet(1);
}
}
}
consumer.commitSync(messages);
}
// automatically unsubscribe topics when closing
} catch (final Exception e) {
e.printStackTrace();
// Avoid failure
} finally {
LOGGER.info("consumer exiting...");
}
});
thread.start();

// Check row count
try {
// Keep retrying if there are execution failures
Awaitility.await()
.pollDelay(IoTDBSubscriptionITConstant.AWAITILITY_POLL_DELAY_SECOND, TimeUnit.SECONDS)
.pollInterval(
IoTDBSubscriptionITConstant.AWAITILITY_POLL_INTERVAL_SECOND, TimeUnit.SECONDS)
.atMost(IoTDBSubscriptionITConstant.AWAITILITY_AT_MOST_SECOND, TimeUnit.SECONDS)
.untilAsserted(
() -> {
Assert.assertEquals(100, rowCount.get());
Assert.assertEquals((100 + 199) * 100 / 2, timestampSum.get());
});
} catch (final Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
isClosed.set(true);
thread.join();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;

import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -101,6 +102,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa

private boolean sloppyTimeRange; // true to disable time range filter after extraction

private Pair<Boolean, Boolean> listeningOptionPair;
private boolean shouldExtractInsertion;
private boolean shouldTransferModFile; // Whether to transfer mods

Expand All @@ -113,6 +115,14 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
public void validate(final PipeParameterValidator validator) {
final PipeParameters parameters = validator.getParameters();

try {
listeningOptionPair =
DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters);
} catch (final Exception e) {
// compatible with the current validation framework
throw new PipeParameterNotValidException(e.getMessage());
}

if (parameters.hasAnyAttributes(
SOURCE_START_TIME_KEY,
EXTRACTOR_START_TIME_KEY,
Expand Down Expand Up @@ -140,11 +150,13 @@ public void validate(final PipeParameterValidator validator) {
SOURCE_END_TIME_KEY,
EXTRACTOR_END_TIME_KEY));
}
return;
} catch (final Exception e) {
// compatible with the current validation framework
throw new PipeParameterNotValidException(e.getMessage());
}

// return here
return;
}

// Historical data extraction is enabled in the following cases:
Expand Down Expand Up @@ -185,23 +197,6 @@ public void validate(final PipeParameterValidator validator) {
EXTRACTOR_HISTORY_END_TIME_KEY,
SOURCE_HISTORY_END_TIME_KEY));
}

shouldTransferModFile =
parameters.getBooleanOrDefault(
Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY),
EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE
|| // Should extract deletion
DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters)
.getRight());

shouldTerminatePipeOnAllHistoricalEventsConsumed =
parameters
.getStringOrDefault(
Arrays.asList(
PipeExtractorConstant.EXTRACTOR_MODE_KEY,
PipeExtractorConstant.SOURCE_MODE_KEY),
PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE)
.equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE);
} catch (final Exception e) {
// Compatible with the current validation framework
throw new PipeParameterNotValidException(e.getMessage());
Expand All @@ -212,8 +207,7 @@ public void validate(final PipeParameterValidator validator) {
public void customize(
final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration)
throws IllegalPathException {
shouldExtractInsertion =
DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters).getLeft();
shouldExtractInsertion = listeningOptionPair.getLeft();
// Do nothing if only extract deletion
if (!shouldExtractInsertion) {
return;
Expand Down Expand Up @@ -295,15 +289,33 @@ public void customize(
.collect(Collectors.toSet())
.contains("time");

shouldTransferModFile =
parameters.getBooleanOrDefault(
Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY),
EXTRACTOR_MODS_ENABLE_DEFAULT_VALUE
|| // Should extract deletion
listeningOptionPair.getRight());

shouldTerminatePipeOnAllHistoricalEventsConsumed =
parameters
.getStringOrDefault(
Arrays.asList(
PipeExtractorConstant.EXTRACTOR_MODE_KEY,
PipeExtractorConstant.SOURCE_MODE_KEY),
PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE)
.equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE);

LOGGER.info(
"Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}",
"Pipe {}@{}: historical data extraction time range, start time {}({}), end time {}({}), sloppy time range {}, should transfer mod file {}, should terminate pipe on all historical events consumed {}",
pipeName,
dataRegionId,
DateTimeUtils.convertLongToDate(historicalDataExtractionStartTime),
historicalDataExtractionStartTime,
DateTimeUtils.convertLongToDate(historicalDataExtractionEndTime),
historicalDataExtractionEndTime,
sloppyTimeRange);
sloppyTimeRange,
shouldTransferModFile,
shouldTerminatePipeOnAllHistoricalEventsConsumed);
}

private void flushDataRegionAllTsFiles() {
Expand Down
Loading