Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
new ConcurrentHashMap<>();

private final Set<CommitterKey> droppedPipeTaskKeys = ConcurrentHashMap.newKeySet();
private final Map<String, ReceiverTemporaryUnavailableBackoff> receiverBackoffMap =
new ConcurrentHashMap<>();
private final Map<String, ReceiverRetryBackoff> receiverBackoffMap = new ConcurrentHashMap<>();

private boolean enableSendTsFileLimit;
private volatile boolean isConnectionException;
Expand Down Expand Up @@ -740,13 +739,13 @@ public boolean isEnableSendTsFileLimit() {
return enableSendTsFileLimit;
}

public void waitIfReceiverTemporarilyUnavailable(final TEndPoint endPoint) {
public void waitIfReceiverRetryIsBackedOff(final TEndPoint endPoint) {
final String endPointKey = format(endPoint);
if (Objects.isNull(endPointKey)) {
return;
}

final ReceiverTemporaryUnavailableBackoff backoff = receiverBackoffMap.get(endPointKey);
final ReceiverRetryBackoff backoff = receiverBackoffMap.get(endPointKey);
if (Objects.isNull(backoff)) {
return;
}
Expand Down Expand Up @@ -782,8 +781,7 @@ public void waitIfReceiverTemporarilyUnavailable(final TEndPoint endPoint) {
}

private void throwIfReceiverProbeIsDelayed() {
for (final Map.Entry<String, ReceiverTemporaryUnavailableBackoff> entry :
receiverBackoffMap.entrySet()) {
for (final Map.Entry<String, ReceiverRetryBackoff> entry : receiverBackoffMap.entrySet()) {
final long probeDelayInMs = entry.getValue().getRemainingProbeDelayInMs();
if (probeDelayInMs <= 0) {
continue;
Expand All @@ -795,10 +793,10 @@ private void throwIfReceiverProbeIsDelayed() {

private static PipeRuntimeSinkNonReportTimeConfigurableException
createReceiverProbeDelayException(
final String endPointKey, final ReceiverTemporaryUnavailableBackoff backoff) {
final String endPointKey, final ReceiverRetryBackoff backoff) {
return new PipeRuntimeSinkNonReportTimeConfigurableException(
String.format(
"Receiver %s remained temporarily unavailable for more than %d ms, pause regular retries and probe every %d ms.",
"Receiver %s has required retries for more than %d ms, pause regular retries and probe every %d ms.",
endPointKey, backoff.getRetryMaxDurationInMs(), backoff.getRetryProbeIntervalInMs()),
Long.MAX_VALUE);
}
Expand All @@ -809,14 +807,14 @@ public void recordReceiverStatus(final TEndPoint endPoint, final TSStatus status
return;
}

if (isReceiverTemporarilyUnavailable(status)) {
if (isReceiverRetryNeeded(status)) {
final long backoffTimeInMs =
receiverBackoffMap
.computeIfAbsent(endPointKey, key -> new ReceiverTemporaryUnavailableBackoff())
.markTemporarilyUnavailable();
.computeIfAbsent(endPointKey, key -> new ReceiverRetryBackoff())
.markRetryNeeded();
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"Receiver {} is temporarily unavailable, throttle requests for {} ms. Status: {}",
"Receiver {} requires a retry, throttle requests for {} ms. Status: {}",
endPointKey,
backoffTimeInMs,
status);
Expand All @@ -834,20 +832,17 @@ public void recordReceiverStatus(final TEndPoint endPoint, final TSStatus status
}
}

private static boolean isReceiverTemporarilyUnavailable(final TSStatus status) {
private static boolean isReceiverRetryNeeded(final TSStatus status) {
if (Objects.isNull(status)) {
return false;
}

final int statusCode = status.getCode();
if (statusCode == TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode()
|| statusCode == TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode()) {
if (!isSuccess(status)) {
return true;
}

return status.isSetSubStatus()
&& status.getSubStatus().stream()
.anyMatch(IoTDBDataRegionAsyncSink::isReceiverTemporarilyUnavailable);
&& status.getSubStatus().stream().anyMatch(IoTDBDataRegionAsyncSink::isReceiverRetryNeeded);
}

private static boolean isSuccess(final TSStatus status) {
Expand Down Expand Up @@ -1036,7 +1031,7 @@ public void setBatchEventSizeHistogram(Histogram eventSizeHistogram) {
}
}

private static class ReceiverTemporaryUnavailableBackoff {
private static class ReceiverRetryBackoff {

private final long maxBackoffTimeInMs =
Math.max(0, PipeConfig.getInstance().getPipeSinkSubtaskSleepIntervalMaxMs());
Expand All @@ -1050,17 +1045,17 @@ private static class ReceiverTemporaryUnavailableBackoff {
Math.max(1, PipeConfig.getInstance().getPipeAsyncSinkRetryProbeIntervalMs());

private boolean active = false;
private long firstUnavailableTimeInMs = 0;
private long firstRetryTimeInMs = 0;
private long currentBackoffTimeInMs = initialBackoffTimeInMs;
private long failureBackoffUntilInMs = 0;
private long nextReservedRetryTimeInMs = 0;
private long nextProbeTimeInMs = 0;

private synchronized long markTemporarilyUnavailable() {
private synchronized long markRetryNeeded() {
final long currentTimeInMs = System.currentTimeMillis();
if (!active) {
active = true;
firstUnavailableTimeInMs = currentTimeInMs;
firstRetryTimeInMs = currentTimeInMs;
currentBackoffTimeInMs = initialBackoffTimeInMs;
failureBackoffUntilInMs = 0;
nextReservedRetryTimeInMs = 0;
Expand All @@ -1082,7 +1077,7 @@ private synchronized boolean isActive() {
private synchronized boolean isRetryMaxDurationExceeded() {
return active
&& retryMaxDurationInMs >= 0
&& System.currentTimeMillis() - firstUnavailableTimeInMs >= retryMaxDurationInMs;
&& System.currentTimeMillis() - firstRetryTimeInMs >= retryMaxDurationInMs;
}

private synchronized long reserveNextRetryTimeInMs() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected boolean tryTransfer(
return false;
}
try {
sink.waitIfReceiverTemporarilyUnavailable(client.getEndPoint());
sink.waitIfReceiverRetryIsBackedOff(client.getEndPoint());
} catch (final PipeRuntimeSinkNonReportTimeConfigurableException e) {
returnClientToPool(client);
onError(e);
Expand Down Expand Up @@ -293,7 +293,7 @@ private void fallbackToWholeRequest(

try {
client.setShouldReturnSelf(shouldReturnSelf);
sink.waitIfReceiverTemporarilyUnavailable(client.getEndPoint());
sink.waitIfReceiverRetryIsBackedOff(client.getEndPoint());
if (returnFalseIfSinkIsClosed(client)) {
return;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public void testTransferWaitsForReceiverBackoffAndRecordsStatus() throws Excepti
handler.transfer(client, createReq(1));

final InOrder inOrder = Mockito.inOrder(sink, client);
inOrder.verify(sink).waitIfReceiverTemporarilyUnavailable(endPoint);
inOrder.verify(sink).waitIfReceiverRetryIsBackedOff(endPoint);
inOrder.verify(client).pipeTransfer(Mockito.any(TPipeTransferReq.class), Mockito.any());
Mockito.verify(sink).recordReceiverStatus(endPoint, status);
}
Expand All @@ -208,7 +208,7 @@ public void testClientIsReturnedWhenReceiverProbeIsDelayed() throws Exception {
final PipeRuntimeSinkNonReportTimeConfigurableException exception =
new PipeRuntimeSinkNonReportTimeConfigurableException("probe delayed", Long.MAX_VALUE);
Mockito.when(client.getEndPoint()).thenReturn(endPoint);
Mockito.doThrow(exception).when(sink).waitIfReceiverTemporarilyUnavailable(endPoint);
Mockito.doThrow(exception).when(sink).waitIfReceiverRetryIsBackedOff(endPoint);

final TestPipeTransferTrackableHandler handler = new TestPipeTransferTrackableHandler(sink);

Expand All @@ -221,18 +221,19 @@ public void testClientIsReturnedWhenReceiverProbeIsDelayed() throws Exception {
}

@Test
public void testReceiverRetriesAreSerialized() {
public void testReceiverRetriesAreSerializedForAnyFailureStatus() {
commonConfig.setPipeSinkSubtaskSleepIntervalInitMs(40);
commonConfig.setPipeSinkSubtaskSleepIntervalMaxMs(40);
commonConfig.setPipeAsyncSinkRetryMaxDurationMs(5000);

final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
final TEndPoint endPoint = new TEndPoint("127.0.0.1", 6667);
sink.recordReceiverStatus(endPoint, temporarilyUnavailableStatus());
sink.recordReceiverStatus(
endPoint, new TSStatus().setCode(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()));

final long startTimeInMs = System.currentTimeMillis();
sink.waitIfReceiverTemporarilyUnavailable(endPoint);
sink.waitIfReceiverTemporarilyUnavailable(endPoint);
sink.waitIfReceiverRetryIsBackedOff(endPoint);
sink.waitIfReceiverRetryIsBackedOff(endPoint);

Assert.assertTrue(System.currentTimeMillis() - startTimeInMs >= 60);
}
Expand All @@ -246,13 +247,14 @@ public void testReceiverRetryFallsBackToSingleProbeAfterMaxDuration() {
final TEndPoint endPoint = new TEndPoint("127.0.0.1", 6667);
sink.recordReceiverStatus(endPoint, temporarilyUnavailableStatus());

sink.waitIfReceiverTemporarilyUnavailable(endPoint);
sink.waitIfReceiverRetryIsBackedOff(endPoint);
Assert.assertThrows(
PipeRuntimeSinkNonReportTimeConfigurableException.class,
() -> sink.waitIfReceiverTemporarilyUnavailable(endPoint));
() -> sink.waitIfReceiverRetryIsBackedOff(endPoint));

sink.recordReceiverStatus(
endPoint, new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
sink.waitIfReceiverTemporarilyUnavailable(endPoint);
sink.waitIfReceiverRetryIsBackedOff(endPoint);
}

private static TSStatus temporarilyUnavailableStatus() {
Expand Down
Loading