From 7dd334b337e9599553ab36befd6bcf564c87eb4a Mon Sep 17 00:00:00 2001 From: Karsten Schnitter Date: Mon, 18 Mar 2024 15:40:07 +0100 Subject: [PATCH 01/18] WIP Send RetryInfo on OTel Timeouts DataPrepper is sending `RESOURCE_EXHAUSTED` gRPC responses whenever a buffer is full or a circuit breaker is active. These statuses do not contain a retry info. In the OpenTelemetry protocol, this implies a non-retryable error, that will lead to message drops, e.g. in the OTel collector. To apply proper back pressure in these scenarios a retry info is added to the status. Signed-off-by: Karsten Schnitter --- .../GrpcRequestExceptionHandler.java | 45 ++++++++++++------- .../GrpcRequestExceptionHandlerTest.java | 26 +++++++++++ .../source/otellogs/OTelLogsSource.java | 2 +- .../source/otellogs/OTelLogsSourceTest.java | 8 ++-- .../source/otelmetrics/OTelMetricsSource.java | 3 +- .../otelmetrics/OTelMetricsSourceTest.java | 5 ++- .../source/oteltrace/OTelTraceSource.java | 2 +- .../source/oteltrace/OTelTraceSourceTest.java | 5 ++- 8 files changed, 68 insertions(+), 28 deletions(-) diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java index 048172bea1..109868f698 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java @@ -5,9 +5,12 @@ package org.opensearch.dataprepper; +import com.google.protobuf.Any; +import com.google.protobuf.Duration; +import com.google.rpc.RetryInfo; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.grpc.GrpcStatusFunction; +import com.linecorp.armeria.common.grpc.GoogleGrpcExceptionHandlerFunction; import com.linecorp.armeria.server.RequestTimeoutException; import io.grpc.Metadata; import io.grpc.Status; @@ -24,7 +27,7 @@ import java.util.concurrent.TimeoutException; -public class GrpcRequestExceptionHandler implements GrpcStatusFunction { +public class GrpcRequestExceptionHandler implements GoogleGrpcExceptionHandlerFunction { private static final Logger LOG = LoggerFactory.getLogger(GrpcRequestExceptionHandler.class); static final String ARMERIA_REQUEST_TIMEOUT_MESSAGE = "Timeout waiting for request to be served. This is usually due to the buffer being full."; @@ -46,44 +49,52 @@ public GrpcRequestExceptionHandler(final PluginMetrics pluginMetrics) { } @Override - public @Nullable Status apply(final RequestContext context, final Throwable exception, final Metadata metadata) { - final Throwable exceptionCause = exception instanceof BufferWriteException ? exception.getCause() : exception; - + public com.google.rpc.@Nullable Status applyStatusProto(RequestContext ctx, Throwable throwable, + Metadata metadata) { + final Throwable exceptionCause = throwable instanceof BufferWriteException ? throwable.getCause() : throwable; return handleExceptions(exceptionCause); } - private Status handleExceptions(final Throwable e) { + private com.google.rpc.Status handleExceptions(final Throwable e) { String message = e.getMessage(); if (e instanceof RequestTimeoutException || e instanceof TimeoutException) { requestTimeoutsCounter.increment(); - return createStatus(e, Status.RESOURCE_EXHAUSTED); + return createStatus(e, Status.Code.RESOURCE_EXHAUSTED); } else if (e instanceof SizeOverflowException) { requestsTooLargeCounter.increment(); - return createStatus(e, Status.RESOURCE_EXHAUSTED); + return createStatus(e, Status.Code.RESOURCE_EXHAUSTED); } else if (e instanceof BadRequestException) { badRequestsCounter.increment(); - return createStatus(e, Status.INVALID_ARGUMENT); + return createStatus(e, Status.Code.INVALID_ARGUMENT); } else if ((e instanceof StatusRuntimeException) && (message.contains("Invalid protobuf byte sequence") || message.contains("Can't decode compressed frame"))) { badRequestsCounter.increment(); - return createStatus(e, Status.INVALID_ARGUMENT); + return createStatus(e, Status.Code.INVALID_ARGUMENT); } else if (e instanceof RequestCancelledException) { requestTimeoutsCounter.increment(); - return createStatus(e, Status.CANCELLED); + return createStatus(e, Status.Code.CANCELLED); } internalServerErrorCounter.increment(); LOG.error("Unexpected exception handling gRPC request", e); - return createStatus(e, Status.INTERNAL); + return createStatus(e, Status.Code.INTERNAL); } - private Status createStatus(final Throwable e, final Status status) { - final String message; + private com.google.rpc.Status createStatus(final Throwable e, final Status.Code code) { + com.google.rpc.Status.Builder builder = com.google.rpc.Status.newBuilder().setCode(code.value()); if (e instanceof RequestTimeoutException) { - message = ARMERIA_REQUEST_TIMEOUT_MESSAGE; + builder.setMessage(ARMERIA_REQUEST_TIMEOUT_MESSAGE); } else { - message = e.getMessage() == null ? status.getCode().name() : e.getMessage(); + builder.setMessage(e.getMessage() == null ? code.name() :e.getMessage()); + } + if (code == Status.Code.RESOURCE_EXHAUSTED) { + builder.addDetails(Any.pack(createRetryInfo())); } + return builder.build(); + } - return status.withDescription(message); + // TODO: Implement logic for the response retry delay to be sent with the retry info + private RetryInfo createRetryInfo() { + Duration.Builder duration = Duration.newBuilder().setSeconds(0L).setNanos(100_000_000); + return RetryInfo.newBuilder().setRetryDelay(duration).build(); } } diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java index 7100891d3a..a23c8f3a20 100644 --- a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java @@ -5,6 +5,8 @@ package org.opensearch.dataprepper; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.RetryInfo; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.server.RequestTimeoutException; import io.grpc.Metadata; @@ -13,6 +15,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.exceptions.BadRequestException; @@ -22,9 +27,11 @@ import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import java.io.IOException; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; +import static com.linecorp.armeria.internal.common.grpc.MetadataUtil.GRPC_STATUS_DETAILS_BIN_KEY; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.times; @@ -55,6 +62,9 @@ public class GrpcRequestExceptionHandlerTest { @Mock private Metadata metadata; + @Captor + private ArgumentCaptor status; + private GrpcRequestExceptionHandler grpcRequestExceptionHandler; @BeforeEach @@ -99,6 +109,22 @@ public void testHandleTimeoutException() { assertThat(messageStatus.getDescription(), equalTo(exceptionMessage)); verify(requestTimeoutsCounter, times(2)).increment(); + + // TODO: Adjust to retry delay logic + verify(metadata, times(2)).put(ArgumentMatchers.eq(GRPC_STATUS_DETAILS_BIN_KEY), status.capture()); + assertThat(status.getValue().getDetailsCount(), equalTo(1)); + + status.getAllValues().stream().map(com.google.rpc.Status::getDetailsList).flatMap(List::stream).map(e -> { + try { + return e.unpack( + RetryInfo.class); + } catch (InvalidProtocolBufferException ex) { + throw new AssertionError("unxepected status detail item",ex); + } + }).forEach(info -> { + assertThat(info.getRetryDelay().getSeconds(), equalTo(0L)); + assertThat(info.getRetryDelay().getNanos(), equalTo(100_000_000)); + }); } @Test diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index 5f5428489f..b37406e773 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -110,7 +110,7 @@ public void start(Buffer> buffer) { .builder() .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true) - .exceptionMapping(requestExceptionHandler); + .exceptionHandler(requestExceptionHandler); final MethodDescriptor methodDescriptor = LogsServiceGrpc.getExportMethod(); final String oTelLogsSourcePath = oTelLogsSourceConfig.getPath(); diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java index 29d3536259..2ce4daba91 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceTest.java @@ -19,7 +19,6 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.common.grpc.GrpcStatusFunction; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -55,6 +54,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.MetricNames; @@ -521,7 +521,8 @@ void start_with_Health_configured_includes_HealthCheck_service() throws IOExcept grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.exceptionMapping(any(GrpcStatusFunction.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); when(server.stop()).thenReturn(completableFuture); final Path certFilePath = Path.of("data/certificate/test_cert.crt"); @@ -563,7 +564,8 @@ void start_without_Health_configured_does_not_include_HealthCheck_service() thro grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.exceptionMapping(any(GrpcStatusFunction.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); when(server.stop()).thenReturn(completableFuture); final Path certFilePath = Path.of("data/certificate/test_cert.crt"); diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index 43e2e1f92d..200d4e87f2 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -112,8 +112,7 @@ public void start(Buffer> buffer) { final GrpcServiceBuilder grpcServiceBuilder = GrpcService .builder() .useClientTimeoutHeader(false) - .useBlockingTaskExecutor(true) - .exceptionMapping(requestExceptionHandler); + .useBlockingTaskExecutor(true).exceptionHandler(requestExceptionHandler); final MethodDescriptor methodDescriptor = MetricsServiceGrpc.getExportMethod(); final String oTelMetricsSourcePath = oTelMetricsSourceConfig.getPath(); diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java index d03f84f07a..9972a81de8 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java @@ -20,7 +20,6 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.common.grpc.GrpcStatusFunction; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -61,6 +60,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.MetricNames; @@ -197,7 +197,8 @@ public void beforeEach() { lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.exceptionMapping(any(GrpcStatusFunction.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); MetricsTestUtil.initMetrics(); pluginMetrics = PluginMetrics.fromNames("otel_metrics", "pipeline"); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 077e4bc879..a4f3c1f8fb 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -113,7 +113,7 @@ public void start(Buffer> buffer) { .builder() .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true) - .exceptionMapping(requestExceptionHandler); + .exceptionHandler(requestExceptionHandler); final MethodDescriptor methodDescriptor = TraceServiceGrpc.getExportMethod(); final String oTelTraceSourcePath = oTelTraceSourceConfig.getPath(); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 9873cc6611..3dfd938657 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -20,7 +20,6 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.common.grpc.GrpcStatusFunction; import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; @@ -55,6 +54,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.MetricNames; @@ -202,7 +202,8 @@ void beforeEach() { lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.exceptionMapping(any(GrpcStatusFunction.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); From 85a7a18d6fc1f075251191f218fb7e2fe1e76ca1 Mon Sep 17 00:00:00 2001 From: Karsten Schnitter Date: Wed, 20 Mar 2024 10:51:44 +0100 Subject: [PATCH 02/18] WIP Add RequestInfo Delay Calculator Implementation of exponential backoff. Idea is to start with a minimum delay on the first time-out or circuit breaker activation. If the next such event happens within twice the last delay after the previous event, double the delay until a maximum delay is reached. Use the maximum delay from then on, until a sufficiently long period (maximum delay) without an event happens. Then the delay is reset to minimum. TODO: Make minimum and maximum delay configurable. Signed-off-by: Karsten Schnitter --- .../GrpcRequestExceptionHandler.java | 13 ++-- .../dataprepper/GrpcRetryInfoCalculator.java | 48 +++++++++++++++ .../GrpcRequestExceptionHandlerTest.java | 23 +++---- .../GrpcRetryInfoCalculatorTest.java | 60 +++++++++++++++++++ 4 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java create mode 100644 data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java index 109868f698..bec2fecc68 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java @@ -6,8 +6,6 @@ package org.opensearch.dataprepper; import com.google.protobuf.Any; -import com.google.protobuf.Duration; -import com.google.rpc.RetryInfo; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.grpc.GoogleGrpcExceptionHandlerFunction; @@ -25,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.util.concurrent.TimeoutException; public class GrpcRequestExceptionHandler implements GoogleGrpcExceptionHandlerFunction { @@ -40,12 +39,14 @@ public class GrpcRequestExceptionHandler implements GoogleGrpcExceptionHandlerFu private final Counter badRequestsCounter; private final Counter requestsTooLargeCounter; private final Counter internalServerErrorCounter; + private final GrpcRetryInfoCalculator retryInfoCalculator; public GrpcRequestExceptionHandler(final PluginMetrics pluginMetrics) { requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); + retryInfoCalculator = new GrpcRetryInfoCalculator(Duration.ofMillis(100), Duration.ofSeconds(2)); } @Override @@ -87,14 +88,8 @@ private com.google.rpc.Status createStatus(final Throwable e, final Status.Code builder.setMessage(e.getMessage() == null ? code.name() :e.getMessage()); } if (code == Status.Code.RESOURCE_EXHAUSTED) { - builder.addDetails(Any.pack(createRetryInfo())); + builder.addDetails(Any.pack(retryInfoCalculator.createRetryInfo())); } return builder.build(); } - - // TODO: Implement logic for the response retry delay to be sent with the retry info - private RetryInfo createRetryInfo() { - Duration.Builder duration = Duration.newBuilder().setSeconds(0L).setNanos(100_000_000); - return RetryInfo.newBuilder().setRetryDelay(duration).build(); - } } diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java new file mode 100644 index 0000000000..1211e5e8ba --- /dev/null +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java @@ -0,0 +1,48 @@ +package org.opensearch.dataprepper; + +import com.google.rpc.RetryInfo; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; + +class GrpcRetryInfoCalculator { + + private final Duration minimumDelay; + private final Duration maximumDelay; + + private final AtomicReference lastTimeCalled; + private final AtomicReference nextDelay; + + GrpcRetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { + this.minimumDelay = minimumDelay; + this.maximumDelay = maximumDelay; + this.lastTimeCalled = new AtomicReference<>(Instant.now()); + this.nextDelay = new AtomicReference<>(minimumDelay); + } + + private static RetryInfo createProtoResult(Duration delay) { + return RetryInfo.newBuilder().setRetryDelay(mapDuration(delay)).build(); + } + + private static Duration minDuration(Duration left, Duration right) { + return left.compareTo(right) <= 0 ? left : right; + } + + private static com.google.protobuf.Duration.Builder mapDuration(Duration duration) { + return com.google.protobuf.Duration.newBuilder().setSeconds(duration.getSeconds()).setNanos(duration.getNano()); + } + + RetryInfo createRetryInfo() { + Instant now = Instant.now(); + // Is the last time we got called longer ago than the next delay? + if (lastTimeCalled.getAndSet(now).isBefore(now.minus(nextDelay.get()))) { + // Use minimum delay and reset the saved delay + nextDelay.set(minimumDelay); + return createProtoResult(minimumDelay); + } + Duration delay = nextDelay.getAndUpdate(d -> minDuration(maximumDelay, d.multipliedBy(2))); + return createProtoResult(delay); + } + +} diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java index a23c8f3a20..2dc3310196 100644 --- a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java @@ -5,7 +5,7 @@ package org.opensearch.dataprepper; -import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Any; import com.google.rpc.RetryInfo; import com.linecorp.armeria.common.RequestContext; import com.linecorp.armeria.server.RequestTimeoutException; @@ -27,13 +27,14 @@ import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import java.io.IOException; -import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeoutException; import static com.linecorp.armeria.internal.common.grpc.MetadataUtil.GRPC_STATUS_DETAILS_BIN_KEY; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -110,21 +111,11 @@ public void testHandleTimeoutException() { verify(requestTimeoutsCounter, times(2)).increment(); - // TODO: Adjust to retry delay logic verify(metadata, times(2)).put(ArgumentMatchers.eq(GRPC_STATUS_DETAILS_BIN_KEY), status.capture()); - assertThat(status.getValue().getDetailsCount(), equalTo(1)); - - status.getAllValues().stream().map(com.google.rpc.Status::getDetailsList).flatMap(List::stream).map(e -> { - try { - return e.unpack( - RetryInfo.class); - } catch (InvalidProtocolBufferException ex) { - throw new AssertionError("unxepected status detail item",ex); - } - }).forEach(info -> { - assertThat(info.getRetryDelay().getSeconds(), equalTo(0L)); - assertThat(info.getRetryDelay().getNanos(), equalTo(100_000_000)); - }); + for (com.google.rpc.Status currentStatus: status.getAllValues()) { + Optional retryInfo = currentStatus.getDetailsList().stream().filter(d -> d.is(RetryInfo.class)).findFirst(); + assertTrue(retryInfo.isPresent(), "No RetryInfo at status:\n" + currentStatus.toString()); + } } @Test diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java new file mode 100644 index 0000000000..5848330f08 --- /dev/null +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java @@ -0,0 +1,60 @@ +package org.opensearch.dataprepper; + +import com.google.rpc.RetryInfo; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class GrpcRetryInfoCalculatorTest { + + @Test + public void testMinimumDelayOnFirstCall() { + RetryInfo retryInfo = new GrpcRetryInfoCalculator(Duration.ofMillis(100), Duration.ofSeconds(1)).createRetryInfo(); + + assertThat(retryInfo.getRetryDelay().getNanos(), equalTo(100_000_000)); + assertThat(retryInfo.getRetryDelay().getSeconds(), equalTo(0L)); + } + + @Test + public void testExponentialBackoff() { + GrpcRetryInfoCalculator calculator = + new GrpcRetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(10)); + RetryInfo retryInfo1 = calculator.createRetryInfo(); + RetryInfo retryInfo2 = calculator.createRetryInfo(); + RetryInfo retryInfo3 = calculator.createRetryInfo(); + + assertThat(retryInfo1.getRetryDelay().getSeconds(), equalTo(1L)); + assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(2L)); + assertThat(retryInfo3.getRetryDelay().getSeconds(), equalTo(4L)); + } + + @Test + public void testUsesMaximumAsLongestDelay() { + GrpcRetryInfoCalculator calculator = + new GrpcRetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(2)); + RetryInfo retryInfo1 = calculator.createRetryInfo(); + RetryInfo retryInfo2 = calculator.createRetryInfo(); + RetryInfo retryInfo3 = calculator.createRetryInfo(); + + assertThat(retryInfo1.getRetryDelay().getSeconds(), equalTo(1L)); + assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(2L)); + assertThat(retryInfo3.getRetryDelay().getSeconds(), equalTo(2L)); + } + + @Test + public void testResetAfterDelayWearsOff() throws InterruptedException { + GrpcRetryInfoCalculator calculator = + new GrpcRetryInfoCalculator(Duration.ofNanos(1_000_000), Duration.ofSeconds(1)); + RetryInfo retryInfo1 = calculator.createRetryInfo(); + RetryInfo retryInfo2 = calculator.createRetryInfo(); + Thread.sleep(5); + RetryInfo retryInfo3 = calculator.createRetryInfo(); + + assertThat(retryInfo1.getRetryDelay().getNanos(), equalTo(1_000_000)); + assertThat(retryInfo2.getRetryDelay().getNanos(), equalTo(2_000_000)); + assertThat(retryInfo3.getRetryDelay().getNanos(), equalTo(1_000_000)); + } +} From d6119b691b330d9134c6fa990721fe8996deb9b9 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 9 Sep 2024 12:47:43 +0200 Subject: [PATCH 03/18] Make backoff calculation configurable (cherry picked from commit 0d45f77a1794f13974a6ddbc2f6e85a440b08dc3) Signed-off-by: Tomas Longo --- .../GrpcRequestExceptionHandler.java | 4 +-- .../GrpcRequestExceptionHandlerTest.java | 3 +- .../source/otellogs/OTelLogsSource.java | 4 ++- .../source/otelmetrics/OTelMetricsSource.java | 4 ++- .../source/oteltrace/OTelTraceSource.java | 12 +++++-- .../oteltrace/OTelTraceSourceConfig.java | 8 +++++ .../plugins/source/oteltrace/RetryInfo.java | 36 +++++++++++++++++++ .../source/oteltrace/OTelTraceSourceTest.java | 6 +++- .../oteltrace/OtelTraceSourceConfigTests.java | 23 ++++++++++++ 9 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java index bec2fecc68..1b7f591f24 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java @@ -41,12 +41,12 @@ public class GrpcRequestExceptionHandler implements GoogleGrpcExceptionHandlerFu private final Counter internalServerErrorCounter; private final GrpcRetryInfoCalculator retryInfoCalculator; - public GrpcRequestExceptionHandler(final PluginMetrics pluginMetrics) { + public GrpcRequestExceptionHandler(final PluginMetrics pluginMetrics, Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); - retryInfoCalculator = new GrpcRetryInfoCalculator(Duration.ofMillis(100), Duration.ofSeconds(2)); + retryInfoCalculator = new GrpcRetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); } @Override diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java index 2dc3310196..031861c50e 100644 --- a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRequestExceptionHandlerTest.java @@ -27,6 +27,7 @@ import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import java.io.IOException; +import java.time.Duration; import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeoutException; @@ -75,7 +76,7 @@ public void setUp() { when(pluginMetrics.counter(HttpRequestExceptionHandler.REQUESTS_TOO_LARGE)).thenReturn(requestsTooLargeCounter); when(pluginMetrics.counter(HttpRequestExceptionHandler.INTERNAL_SERVER_ERROR)).thenReturn(internalServerErrorCounter); - grpcRequestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics); + grpcRequestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); } @Test diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index b37406e773..2d8bddfb7c 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -43,6 +43,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; @@ -80,7 +81,8 @@ public OTelLogsSource(final OTelLogsSourceConfig oTelLogsSourceConfig, this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); this.authenticationProvider = createAuthenticationProvider(pluginFactory); - this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics); + // TODO tlongo read from config + this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); this.byteDecoder = new OTelLogsDecoder(); } diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index 200d4e87f2..4b53bd5181 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -45,6 +45,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -84,7 +85,8 @@ public OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); this.authenticationProvider = createAuthenticationProvider(pluginFactory); - this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics); + // TODO tlongo read from config + this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); this.byteDecoder = new OTelMetricDecoder(); } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index a4f3c1f8fb..8b8f6d3298 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; import com.linecorp.armeria.common.util.BlockingTaskExecutor; import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; @@ -45,6 +46,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -63,7 +65,6 @@ public class OTelTraceSource implements Source> { private final PluginMetrics pluginMetrics; private final GrpcAuthenticationProvider authenticationProvider; private final CertificateProviderFactory certificateProviderFactory; - private final GrpcRequestExceptionHandler requestExceptionHandler; private final String pipelineName; private Server server; private final ByteDecoder byteDecoder; @@ -83,7 +84,6 @@ public OTelTraceSource(final OTelTraceSourceConfig oTelTraceSourceConfig, final this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); this.authenticationProvider = createAuthenticationProvider(pluginFactory); - this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics); this.byteDecoder = new OTelTraceDecoder(); } @@ -113,7 +113,7 @@ public void start(Buffer> buffer) { .builder() .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true) - .exceptionHandler(requestExceptionHandler); + .exceptionHandler(createGrpExceptionHandler()); final MethodDescriptor methodDescriptor = TraceServiceGrpc.getExportMethod(); final String oTelTraceSourcePath = oTelTraceSourceConfig.getPath(); @@ -208,6 +208,12 @@ public void start(Buffer> buffer) { LOG.info("Started otel_trace_source on port " + oTelTraceSourceConfig.getPort() + "..."); } + private GrpcExceptionHandlerFunction createGrpExceptionHandler() { + RetryInfo retryInfo = oTelTraceSourceConfig.getRetryInfo() != null ? oTelTraceSourceConfig.getRetryInfo() : new RetryInfo(100, 2000); + + return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + } + @Override public void stop() { if (server != null) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index 4760da34a4..61a9c3f205 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -32,6 +32,7 @@ public class OTelTraceSourceConfig { static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; static final String UNAUTHENTICATED_HEALTH_CHECK = "unauthenticated_health_check"; static final String COMPRESSION = "compression"; + static final String RETRY_INFO = "retry_info"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21890; static final int DEFAULT_THREAD_COUNT = 200; @@ -107,6 +108,9 @@ public class OTelTraceSourceConfig { @JsonProperty("max_request_length") private ByteCount maxRequestLength; + @JsonProperty(RETRY_INFO) + private RetryInfo retryInfo; + @AssertTrue(message = "path should start with /") boolean isPathValid() { return path == null || path.startsWith("/"); @@ -228,4 +232,8 @@ public CompressionOption getCompression() { public ByteCount getMaxRequestLength() { return maxRequestLength; } + + public RetryInfo getRetryInfo() { + return retryInfo; + } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java new file mode 100644 index 0000000000..be2092f4aa --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java @@ -0,0 +1,36 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RetryInfo { + + @JsonProperty("min_delay") + private Integer minDelay; + + @JsonProperty("max_delay") + private Integer maxDelay; + + // Jackson needs this constructor + public RetryInfo() {} + + public RetryInfo(int minDelay, int maxDelay) { + this.minDelay = minDelay; + this.maxDelay = maxDelay; + } + + public int getMinDelay() { + return minDelay; + } + + public void setMinDelay(Integer minDelay) { + this.minDelay = minDelay; + } + + public int getMaxDelay() { + return maxDelay; + } + + public void setMaxDelay(Integer maxDelay) { + this.maxDelay = maxDelay; + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 3dfd938657..8e7944fd0f 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -136,6 +136,7 @@ class OTelTraceSourceTest { private static final String TEST_PATH = "${pipelineName}/v1/traces"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfo TEST_RETRY_INFO = new RetryInfo(100, 2000); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() @@ -214,6 +215,7 @@ void beforeEach() { when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(authenticationProvider); @@ -851,7 +853,9 @@ void testRunAnotherSourceWithSamePort() { // starting server SOURCE.start(buffer); - testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); + + Map settingsMap = Map.of("retry_info", TEST_RETRY_INFO, SSL, false); + testPluginSetting = new PluginSetting(null, settingsMap); testPluginSetting.setPipelineName("pipeline"); oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index ea5ce66b91..8b2595a605 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -302,6 +302,28 @@ void testInValidConfigWithCustomPath() { assertThat(otelTraceSourceConfig.isPathValid(), equalTo(false)); } + @Test + void testRetryInfoConfig() { + final PluginSetting customPathPluginSetting = completePluginSettingForOtelTraceSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + null, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelTraceSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelTraceSourceConfig.class); + + + assertThat(otelTraceSourceConfig.getRetryInfo().getMaxDelay(), equalTo(100)); + assertThat(otelTraceSourceConfig.getRetryInfo().getMinDelay(), equalTo(50)); + } + private PluginSetting completePluginSettingForOtelTraceSource(final int requestTimeoutInMillis, final int port, final String path, @@ -325,6 +347,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); settings.put(OTelTraceSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelTraceSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); + settings.put(OTelTraceSourceConfig.RETRY_INFO, new RetryInfo(50, 100)); return new PluginSetting(PLUGIN_NAME, settings); } } From 52d2488abad439fbabf25fc24e2476b1df30a386 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Wed, 11 Sep 2024 08:15:46 +0200 Subject: [PATCH 04/18] Rename RetryInfo -> RetryInfoConfig (cherry picked from commit f8ac48edf39c8c169d45e1a47212e555c9d163f7) Signed-off-by: Tomas Longo --- .../plugins/source/oteltrace/OTelTraceSource.java | 2 +- .../plugins/source/oteltrace/OTelTraceSourceConfig.java | 4 ++-- .../oteltrace/{RetryInfo.java => RetryInfoConfig.java} | 6 +++--- .../plugins/source/oteltrace/OTelTraceSourceTest.java | 2 +- .../source/oteltrace/OtelTraceSourceConfigTests.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/{RetryInfo.java => RetryInfoConfig.java} (84%) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 8b8f6d3298..db2b466551 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -209,7 +209,7 @@ public void start(Buffer> buffer) { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - RetryInfo retryInfo = oTelTraceSourceConfig.getRetryInfo() != null ? oTelTraceSourceConfig.getRetryInfo() : new RetryInfo(100, 2000); + RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null ? oTelTraceSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index 61a9c3f205..c558cd9dbe 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -109,7 +109,7 @@ public class OTelTraceSourceConfig { private ByteCount maxRequestLength; @JsonProperty(RETRY_INFO) - private RetryInfo retryInfo; + private RetryInfoConfig retryInfo; @AssertTrue(message = "path should start with /") boolean isPathValid() { @@ -233,7 +233,7 @@ public ByteCount getMaxRequestLength() { return maxRequestLength; } - public RetryInfo getRetryInfo() { + public RetryInfoConfig getRetryInfo() { return retryInfo; } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java similarity index 84% rename from data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java rename to data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java index be2092f4aa..a38553cbe7 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfo.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; -public class RetryInfo { +public class RetryInfoConfig { @JsonProperty("min_delay") private Integer minDelay; @@ -11,9 +11,9 @@ public class RetryInfo { private Integer maxDelay; // Jackson needs this constructor - public RetryInfo() {} + public RetryInfoConfig() {} - public RetryInfo(int minDelay, int maxDelay) { + public RetryInfoConfig(int minDelay, int maxDelay) { this.minDelay = minDelay; this.maxDelay = maxDelay; } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 8e7944fd0f..38cf9fc232 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -136,7 +136,7 @@ class OTelTraceSourceTest { private static final String TEST_PATH = "${pipelineName}/v1/traces"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfo TEST_RETRY_INFO = new RetryInfo(100, 2000); + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index 8b2595a605..ebe5023896 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -347,7 +347,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); settings.put(OTelTraceSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelTraceSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); - settings.put(OTelTraceSourceConfig.RETRY_INFO, new RetryInfo(50, 100)); + settings.put(OTelTraceSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); return new PluginSetting(PLUGIN_NAME, settings); } } From 8c667a71033077619f2175010ee4eb76fe5f2c82 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 13 Sep 2024 15:20:43 +0200 Subject: [PATCH 05/18] Create test for back-off calculation (cherry picked from commit ff675dc34fc01e45a0d66131ba87740d6aa3ae9f) Signed-off-by: Tomas Longo --- .../OTelTraceSourceRetryInfoTest.java | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java new file mode 100644 index 0000000000..6881bb8a81 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.oteltrace; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; + +import java.time.Duration; +import java.util.UUID; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.RetryInfo; +import com.linecorp.armeria.client.Clients; + +import io.grpc.Metadata; +import io.grpc.StatusRuntimeException; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.Span; + +@ExtendWith(MockitoExtension.class) +class OTelTraceSourceRetryInfoTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelTraceSourceConfig oTelTraceSourceConfig; + + @Mock + private Buffer> buffer; + + private PipelineDescription pipelineDescription; + private OTelTraceSource SOURCE; + + @BeforeEach + void beforeEach() throws Exception { + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); + + when(oTelTraceSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelTraceSourceConfig.isSsl()).thenReturn(false); + when(oTelTraceSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + + configureObjectUnderTest(); + SOURCE.start(buffer); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + @Test + public void gRPC_failed_request_returns_minimal_delay_in_status() throws Exception { + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + + RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void gRPC_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callService3TimesAndReturnRetryInfo(); + + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); + return RetryInfo.parseFrom(status.getDetails(0).getValue()); + } + + /** + * The back off is calculated with the second call, and returned with the third + */ + private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + } + + return extracRetryInfoFromStatusRuntimeException(e); + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } +} From a42ccce6c6485f01c4dfbbaca38c309fcfd55673 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 16 Sep 2024 08:32:54 +0200 Subject: [PATCH 06/18] Return retry info in metrics source (cherry picked from commit 43ba7ee73f599b0bed6cbb7fc7f684339e2ac572) Signed-off-by: Tomas Longo --- .../source/otelmetrics/OTelMetricsSource.java | 13 +- .../otelmetrics/OTelMetricsSourceConfig.java | 12 ++ .../source/otelmetrics/RetryInfoConfig.java | 36 ++++ .../OTelMetricsSourceRetryInfoTest.java | 182 ++++++++++++++++++ .../OtelMetricsSourceConfigTests.java | 24 +++ 5 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java create mode 100644 data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index 4b53bd5181..419351ddcf 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source.otelmetrics; +import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; @@ -66,7 +67,6 @@ public class OTelMetricsSource implements Source> { private final PluginMetrics pluginMetrics; private final GrpcAuthenticationProvider authenticationProvider; private final CertificateProviderFactory certificateProviderFactory; - private final GrpcRequestExceptionHandler requestExceptionHandler; private Server server; private final ByteDecoder byteDecoder; @@ -85,8 +85,6 @@ public OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); this.authenticationProvider = createAuthenticationProvider(pluginFactory); - // TODO tlongo read from config - this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); this.byteDecoder = new OTelMetricDecoder(); } @@ -114,7 +112,8 @@ public void start(Buffer> buffer) { final GrpcServiceBuilder grpcServiceBuilder = GrpcService .builder() .useClientTimeoutHeader(false) - .useBlockingTaskExecutor(true).exceptionHandler(requestExceptionHandler); + .useBlockingTaskExecutor(true) + .exceptionHandler(createGrpExceptionHandler()); final MethodDescriptor methodDescriptor = MetricsServiceGrpc.getExportMethod(); final String oTelMetricsSourcePath = oTelMetricsSourceConfig.getPath(); @@ -226,6 +225,12 @@ public void stop() { LOG.info("Stopped otel_metrics_source."); } + private GrpcExceptionHandlerFunction createGrpExceptionHandler() { + RetryInfoConfig retryInfo = oTelMetricsSourceConfig.getRetryInfo() != null ? oTelMetricsSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); + + return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + } + private List getAuthenticationInterceptor() { final ServerInterceptor authenticationInterceptor = authenticationProvider.getAuthenticationInterceptor(); if (authenticationInterceptor == null) { diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java index ea590fd80b..248ecf2154 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java @@ -31,6 +31,7 @@ public class OTelMetricsSourceConfig { static final String MAX_CONNECTION_COUNT = "max_connection_count"; static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; static final String COMPRESSION = "compression"; + static final String RETRY_INFO = "retry_info"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21891; static final int DEFAULT_THREAD_COUNT = 200; @@ -107,6 +108,9 @@ public class OTelMetricsSourceConfig { @JsonProperty("max_request_length") private ByteCount maxRequestLength; + @JsonProperty(RETRY_INFO) + private RetryInfoConfig retryInfo; + @AssertTrue(message = "path should start with /") boolean isPathValid() { return path == null || path.startsWith("/"); @@ -228,5 +232,13 @@ public CompressionOption getCompression() { public ByteCount getMaxRequestLength() { return maxRequestLength; } + + public RetryInfoConfig getRetryInfo() { + return retryInfo; + } + + public void setRetryInfo(RetryInfoConfig retryInfo) { + this.retryInfo = retryInfo; + } } diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java new file mode 100644 index 0000000000..7068597e98 --- /dev/null +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java @@ -0,0 +1,36 @@ +package org.opensearch.dataprepper.plugins.source.otelmetrics; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RetryInfoConfig { + + @JsonProperty("min_delay") + private Integer minDelay; + + @JsonProperty("max_delay") + private Integer maxDelay; + + // Jackson needs this constructor + public RetryInfoConfig() {} + + public RetryInfoConfig(int minDelay, int maxDelay) { + this.minDelay = minDelay; + this.maxDelay = maxDelay; + } + + public int getMinDelay() { + return minDelay; + } + + public void setMinDelay(Integer minDelay) { + this.minDelay = minDelay; + } + + public int getMaxDelay() { + return maxDelay; + } + + public void setMaxDelay(Integer maxDelay) { + this.maxDelay = maxDelay; + } +} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java new file mode 100644 index 0000000000..024eff7ce1 --- /dev/null +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java @@ -0,0 +1,182 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otelmetrics; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; + +import java.time.Duration; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.metric.Metric; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.RetryInfo; +import com.linecorp.armeria.client.Clients; + +import io.grpc.Metadata; +import io.grpc.StatusRuntimeException; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.InstrumentationLibrary; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.Gauge; +import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.resource.v1.Resource; + +@ExtendWith(MockitoExtension.class) +class OTelMetricsSourceRetryInfoTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelMetricsSourceConfig oTelMetricsSourceConfig; + + @Mock + private Buffer> buffer; + + private PipelineDescription pipelineDescription; + private OTelMetricsSource SOURCE; + + @BeforeEach + void beforeEach() throws Exception { + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); + + when(oTelMetricsSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelMetricsSourceConfig.isSsl()).thenReturn(false); + when(oTelMetricsSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelMetricsSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelMetricsSourceConfig.getThreadCount()).thenReturn(5); + when(oTelMetricsSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelMetricsSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + + configureObjectUnderTest(); + SOURCE.start(buffer); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + @Test + public void gRPC_failed_request_returns_minimal_delay_in_status() throws Exception { + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + + RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void gRPC_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callService3TimesAndReturnRetryInfo(); + + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); + return RetryInfo.parseFrom(status.getDetails(0).getValue()); + } + + /** + * The back off is calculated with the second call, and returned with the third + */ + private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + } + + return extracRetryInfoFromStatusRuntimeException(e); + } + + private ExportMetricsServiceRequest createExportMetricsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build()) + ).build(); + NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); + Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); + + io.opentelemetry.proto.metrics.v1.Metric.Builder metric = io.opentelemetry.proto.metrics.v1.Metric.newBuilder() + .setGauge(gauge) + .setUnit("seconds") + .setName("name") + .setDescription("description"); + InstrumentationLibraryMetrics isntLib = InstrumentationLibraryMetrics.newBuilder() + .addMetrics(metric) + .setInstrumentationLibrary(InstrumentationLibrary.newBuilder() + .setName("ilname") + .setVersion("ilversion") + .build()) + .build(); + + + final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() + .setResource(resource) + .addInstrumentationLibraryMetrics(isntLib) + .build(); + + return ExportMetricsServiceRequest.newBuilder() + .addResourceMetrics(resourceMetrics).build(); + } +} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index 60193484c9..b2053e90ec 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -303,6 +303,29 @@ void testInValidConfigWithCustomPath() { assertThat(oTelMetricsSourceConfig.isPathValid(), equalTo(false)); } + @Test + void testRetryInfoConfig() { + final PluginSetting customPathPluginSetting = completePluginSettingForOtelMetricsSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + null, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelMetricsSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + + + RetryInfoConfig retryInfo = otelTraceSourceConfig.getRetryInfo(); + assertThat(retryInfo.getMaxDelay(), equalTo(100)); + assertThat(retryInfo.getMinDelay(), equalTo(50)); + } + private PluginSetting completePluginSettingForOtelMetricsSource(final int requestTimeoutInMillis, final int port, final String path, @@ -326,6 +349,7 @@ private PluginSetting completePluginSettingForOtelMetricsSource(final int reques settings.put(OTelMetricsSourceConfig.SSL_KEY_FILE, sslKeyFile); settings.put(OTelMetricsSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelMetricsSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); + settings.put(OTelMetricsSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); return new PluginSetting(PLUGIN_NAME, settings); } } From 603f4fbdbaeb6464f95f7abb10fb2a8cbe587321 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 16 Sep 2024 08:50:16 +0200 Subject: [PATCH 07/18] Return retry info in logs source (cherry picked from commit 1f90615278a560ae78990013db590efcb57bdeb5) Signed-off-by: Tomas Longo --- .../source/otellogs/OTelLogsSource.java | 63 ++++--- .../source/otellogs/OTelLogsSourceConfig.java | 12 ++ .../source/otellogs/RetryInfoConfig.java | 36 ++++ .../otellogs/OtelLogsSourceConfigTests.java | 24 +++ .../otellogs/OtelLogsSourceRetryInfoTest.java | 165 ++++++++++++++++++ .../OTelMetricsSourceRetryInfoTest.java | 6 +- 6 files changed, 272 insertions(+), 34 deletions(-) create mode 100644 data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java create mode 100644 data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index 2d8bddfb7c..c3e70dea1e 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -5,28 +5,21 @@ package org.opensearch.dataprepper.plugins.source.otellogs; -import com.linecorp.armeria.server.encoding.DecodingService; -import org.opensearch.dataprepper.GrpcRequestExceptionHandler; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; -import org.opensearch.dataprepper.plugins.source.otellogs.certificate.CertificateProviderFactory; -import com.linecorp.armeria.server.Server; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; -import io.grpc.MethodDescriptor; -import io.grpc.ServerInterceptor; -import io.grpc.ServerInterceptors; -import io.grpc.protobuf.services.ProtoReflectionService; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; -import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -35,19 +28,28 @@ import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; -import org.opensearch.dataprepper.model.codec.ByteDecoder; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; import org.opensearch.dataprepper.plugins.otel.codec.OTelLogsDecoder; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; +import org.opensearch.dataprepper.plugins.source.otellogs.certificate.CertificateProviderFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; +import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.encoding.DecodingService; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; + +import io.grpc.MethodDescriptor; +import io.grpc.ServerInterceptor; +import io.grpc.ServerInterceptors; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; @DataPrepperPlugin(name = "otel_logs_source", pluginType = Source.class, pluginConfigurationType = OTelLogsSourceConfig.class) public class OTelLogsSource implements Source> { @@ -60,7 +62,6 @@ public class OTelLogsSource implements Source> { private final PluginMetrics pluginMetrics; private final GrpcAuthenticationProvider authenticationProvider; private final CertificateProviderFactory certificateProviderFactory; - private final GrpcRequestExceptionHandler requestExceptionHandler; private Server server; private ByteDecoder byteDecoder; @@ -81,8 +82,6 @@ public OTelLogsSource(final OTelLogsSourceConfig oTelLogsSourceConfig, this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); this.authenticationProvider = createAuthenticationProvider(pluginFactory); - // TODO tlongo read from config - this.requestExceptionHandler = new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); this.byteDecoder = new OTelLogsDecoder(); } @@ -112,7 +111,7 @@ public void start(Buffer> buffer) { .builder() .useClientTimeoutHeader(false) .useBlockingTaskExecutor(true) - .exceptionHandler(requestExceptionHandler); + .exceptionHandler(createGrpExceptionHandler()); final MethodDescriptor methodDescriptor = LogsServiceGrpc.getExportMethod(); final String oTelLogsSourcePath = oTelLogsSourceConfig.getPath(); @@ -207,6 +206,12 @@ public void stop() { LOG.info("Stopped otel_logs_source."); } + private GrpcExceptionHandlerFunction createGrpExceptionHandler() { + RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo() != null ? oTelLogsSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); + + return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + } + private List getAuthenticationInterceptor() { final ServerInterceptor authenticationInterceptor = authenticationProvider.getAuthenticationInterceptor(); if (authenticationInterceptor == null) { diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java index bfca665d76..24662ec802 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSourceConfig.java @@ -32,6 +32,7 @@ public class OTelLogsSourceConfig { static final String MAX_CONNECTION_COUNT = "max_connection_count"; static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; static final String COMPRESSION = "compression"; + static final String RETRY_INFO = "retry_info"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21892; static final int DEFAULT_THREAD_COUNT = 200; @@ -104,6 +105,9 @@ public class OTelLogsSourceConfig { @JsonProperty("max_request_length") private ByteCount maxRequestLength; + @JsonProperty(RETRY_INFO) + private RetryInfoConfig retryInfo; + @AssertTrue(message = "path should start with /") boolean isPathValid() { return path == null || path.startsWith("/"); @@ -217,5 +221,13 @@ public CompressionOption getCompression() { public ByteCount getMaxRequestLength() { return maxRequestLength; } + + public RetryInfoConfig getRetryInfo() { + return retryInfo; + } + + public void setRetryInfo(RetryInfoConfig retryInfo) { + this.retryInfo = retryInfo; + } } diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java new file mode 100644 index 0000000000..7ab544caca --- /dev/null +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java @@ -0,0 +1,36 @@ +package org.opensearch.dataprepper.plugins.source.otellogs; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RetryInfoConfig { + + @JsonProperty("min_delay") + private Integer minDelay; + + @JsonProperty("max_delay") + private Integer maxDelay; + + // Jackson needs this constructor + public RetryInfoConfig() {} + + public RetryInfoConfig(int minDelay, int maxDelay) { + this.minDelay = minDelay; + this.maxDelay = maxDelay; + } + + public int getMinDelay() { + return minDelay; + } + + public void setMinDelay(Integer minDelay) { + this.minDelay = minDelay; + } + + public int getMaxDelay() { + return maxDelay; + } + + public void setMaxDelay(Integer maxDelay) { + this.maxDelay = maxDelay; + } +} diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java index 69f92e5b1e..963035edc4 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java @@ -276,6 +276,29 @@ void testInValidConfigWithCustomPath() { assertThat(oTelLogsSourceConfig.isPathValid(), equalTo(false)); } + @Test + void testRetryInfoConfig() { + final PluginSetting customPathPluginSetting = completePluginSettingForOtelLogsSource( + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_PORT, + null, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + DEFAULT_THREAD_COUNT, + DEFAULT_MAX_CONNECTION_COUNT); + + final OTelLogsSourceConfig oTelLogsSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelLogsSourceConfig.class); + + + RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo(); + assertThat(retryInfo.getMaxDelay(), equalTo(100)); + assertThat(retryInfo.getMinDelay(), equalTo(50)); + } + private PluginSetting completePluginSettingForOtelLogsSource(final int requestTimeoutInMillis, final int port, final String path, @@ -299,6 +322,7 @@ private PluginSetting completePluginSettingForOtelLogsSource(final int requestTi settings.put(SSL_KEY_FILE, sslKeyFile); settings.put(THREAD_COUNT, threadCount); settings.put(MAX_CONNECTION_COUNT, maxConnectionCount); + settings.put(OTelLogsSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); return new PluginSetting(PLUGIN_NAME, settings); } } diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java new file mode 100644 index 0000000000..ef1b500ad2 --- /dev/null +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java @@ -0,0 +1,165 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otellogs; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; + +import java.time.Duration; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.otel.codec.OTelLogsDecoder; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.RetryInfo; +import com.linecorp.armeria.client.Clients; + +import io.grpc.Metadata; +import io.grpc.StatusRuntimeException; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.resource.v1.Resource; + +@ExtendWith(MockitoExtension.class) +class OtelLogsSourceRetryInfoTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21892/"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelLogsSourceConfig oTelLogsSourceConfig; + + @Mock + private Buffer> buffer; + + private OTelLogsSource SOURCE; + + @BeforeEach + void beforeEach() throws Exception { + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); + + when(oTelLogsSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelLogsSourceConfig.isSsl()).thenReturn(false); + when(oTelLogsSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelLogsSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelLogsSourceConfig.getThreadCount()).thenReturn(5); + when(oTelLogsSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelLogsSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + + configureObjectUnderTest(); + SOURCE.start(buffer); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_logs", "pipeline"); + PipelineDescription pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + + SOURCE = new OTelLogsSource(oTelLogsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + assertTrue(SOURCE.getDecoder() instanceof OTelLogsDecoder); + } + + @Test + public void gRPC_failed_request_returns_minimal_delay_in_status() throws Exception { + final LogsServiceGrpc.LogsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); + + RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void gRPC_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callService3TimesAndReturnRetryInfo(); + + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); + return RetryInfo.parseFrom(status.getDetails(0).getValue()); + } + + /** + * The back off is calculated with the second call, and returned with the third + */ + private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final LogsServiceGrpc.LogsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); + } + + return extracRetryInfoFromStatusRuntimeException(e); + } + + private ExportLogsServiceRequest createExportLogsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build()) + ).build(); + + final ResourceLogs resourceLogs = ResourceLogs.newBuilder() + .addScopeLogs(ScopeLogs.newBuilder() + .addLogRecords(LogRecord.newBuilder().setSeverityNumberValue(1)) + .build()) + .setResource(resource) + .build(); + + return ExportLogsServiceRequest.newBuilder() + .addResourceLogs(resourceLogs) + .build(); + } +} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java index 024eff7ce1..0595b4c029 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java @@ -73,7 +73,6 @@ class OTelMetricsSourceRetryInfoTest { @Mock private Buffer> buffer; - private PipelineDescription pipelineDescription; private OTelMetricsSource SOURCE; @BeforeEach @@ -91,9 +90,6 @@ void beforeEach() throws Exception { when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(authenticationProvider); - configureObjectUnderTest(); - pipelineDescription = mock(PipelineDescription.class); - lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); configureObjectUnderTest(); SOURCE.start(buffer); @@ -107,7 +103,7 @@ void afterEach() { private void configureObjectUnderTest() { PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); - pipelineDescription = mock(PipelineDescription.class); + PipelineDescription pipelineDescription = mock(PipelineDescription.class); when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } From 7c6bb9a2cfeb3edeb62d7d7523a6bc3c0ff06f6b Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 16 Sep 2024 08:52:05 +0200 Subject: [PATCH 08/18] Get rid of a few warnings (cherry picked from commit 2977c1f2e8ac629d91a0a5fa808e431d3300dc54) Signed-off-by: Tomas Longo --- .../dataprepper/plugins/source/otellogs/OTelLogsSource.java | 2 +- .../plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index c3e70dea1e..5e46acc18b 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -62,8 +62,8 @@ public class OTelLogsSource implements Source> { private final PluginMetrics pluginMetrics; private final GrpcAuthenticationProvider authenticationProvider; private final CertificateProviderFactory certificateProviderFactory; + private final ByteDecoder byteDecoder; private Server server; - private ByteDecoder byteDecoder; @DataPrepperPluginConstructor public OTelLogsSource(final OTelLogsSourceConfig oTelLogsSourceConfig, diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java index ef1b500ad2..41de6302ba 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java @@ -7,6 +7,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -105,7 +106,7 @@ private void configureObjectUnderTest() { lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); SOURCE = new OTelLogsSource(oTelLogsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - assertTrue(SOURCE.getDecoder() instanceof OTelLogsDecoder); + assertInstanceOf(OTelLogsDecoder.class, SOURCE.getDecoder()); } @Test From c2bc33f6bce7039b5865535f2de86dd54d479b19 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 23 Sep 2024 14:33:58 +0200 Subject: [PATCH 09/18] Use Duration for delays in RetryInfoConfig (cherry picked from commit 473db0e0c3b2ef0186bad685b853b2a9efab4f97) Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 9 +++++++-- .../source/oteltrace/RetryInfoConfig.java | 16 +++++++++------- .../oteltrace/OTelTraceSourceRetryInfoTest.java | 2 +- .../source/oteltrace/OTelTraceSourceTest.java | 3 ++- .../oteltrace/OtelTraceSourceConfigTests.java | 3 ++- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index db2b466551..16d1eba2b0 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -209,9 +209,14 @@ public void start(Buffer> buffer) { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null ? oTelTraceSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); + Duration defaultMinDelay = Duration.ofMillis(100); + Duration defaultMaxDelay = Duration.ofMillis(2000); - return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null + ? oTelTraceSourceConfig.getRetryInfo() + : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + + return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } @Override diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java index a38553cbe7..84f95c82bf 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java @@ -1,36 +1,38 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import java.time.Duration; + import com.fasterxml.jackson.annotation.JsonProperty; public class RetryInfoConfig { @JsonProperty("min_delay") - private Integer minDelay; + private Duration minDelay; @JsonProperty("max_delay") - private Integer maxDelay; + private Duration maxDelay; // Jackson needs this constructor public RetryInfoConfig() {} - public RetryInfoConfig(int minDelay, int maxDelay) { + public RetryInfoConfig(Duration minDelay, Duration maxDelay) { this.minDelay = minDelay; this.maxDelay = maxDelay; } - public int getMinDelay() { + public Duration getMinDelay() { return minDelay; } - public void setMinDelay(Integer minDelay) { + public void setMinDelay(Duration minDelay) { this.minDelay = minDelay; } - public int getMaxDelay() { + public Duration getMaxDelay() { return maxDelay; } - public void setMaxDelay(Integer maxDelay) { + public void setMaxDelay(Duration maxDelay) { this.maxDelay = maxDelay; } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java index 6881bb8a81..a00faa33b9 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java @@ -55,7 +55,7 @@ class OTelTraceSourceRetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); @Mock private PluginFactory pluginFactory; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 38cf9fc232..8c91518959 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -81,6 +81,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.Base64; import java.util.Collection; import java.util.Collections; @@ -136,7 +137,7 @@ class OTelTraceSourceTest { private static final String TEST_PATH = "${pipelineName}/v1/traces"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index ebe5023896..a5a7e885c1 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -347,7 +348,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); settings.put(OTelTraceSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelTraceSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); - settings.put(OTelTraceSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); + settings.put(OTelTraceSourceConfig.RETRY_INFO, new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); return new PluginSetting(PLUGIN_NAME, settings); } } From da5b668b51b1a4dada9842db23d2c283df5de105 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 23 Sep 2024 14:38:03 +0200 Subject: [PATCH 10/18] Use Duration for delays in RetryInfoConfig for OTelLogsSource (cherry picked from commit 6ef9b7e36dc1970e1e87dceb30bcc5204d082818) Signed-off-by: Tomas Longo --- .../plugins/source/otellogs/OTelLogsSource.java | 9 +++++++-- .../plugins/source/otellogs/RetryInfoConfig.java | 16 +++++++++------- .../otellogs/OtelLogsSourceConfigTests.java | 3 ++- .../otellogs/OtelLogsSourceRetryInfoTest.java | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index 5e46acc18b..737f791fb3 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -207,9 +207,14 @@ public void stop() { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo() != null ? oTelLogsSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); + Duration defaultMinDelay = Duration.ofMillis(100); + Duration defaultMaxDelay = Duration.ofMillis(2000); - return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo() != null + ? oTelLogsSourceConfig.getRetryInfo() + : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + + return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } private List getAuthenticationInterceptor() { diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java index 7ab544caca..e3dbf028d4 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java @@ -1,36 +1,38 @@ package org.opensearch.dataprepper.plugins.source.otellogs; +import java.time.Duration; + import com.fasterxml.jackson.annotation.JsonProperty; public class RetryInfoConfig { @JsonProperty("min_delay") - private Integer minDelay; + private Duration minDelay; @JsonProperty("max_delay") - private Integer maxDelay; + private Duration maxDelay; // Jackson needs this constructor public RetryInfoConfig() {} - public RetryInfoConfig(int minDelay, int maxDelay) { + public RetryInfoConfig(Duration minDelay, Duration maxDelay) { this.minDelay = minDelay; this.maxDelay = maxDelay; } - public int getMinDelay() { + public Duration getMinDelay() { return minDelay; } - public void setMinDelay(Integer minDelay) { + public void setMinDelay(Duration minDelay) { this.minDelay = minDelay; } - public int getMaxDelay() { + public Duration getMaxDelay() { return maxDelay; } - public void setMaxDelay(Integer maxDelay) { + public void setMaxDelay(Duration maxDelay) { this.maxDelay = maxDelay; } } diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java index 963035edc4..f5965b3431 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java @@ -13,6 +13,7 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -322,7 +323,7 @@ private PluginSetting completePluginSettingForOtelLogsSource(final int requestTi settings.put(SSL_KEY_FILE, sslKeyFile); settings.put(THREAD_COUNT, threadCount); settings.put(MAX_CONNECTION_COUNT, maxConnectionCount); - settings.put(OTelLogsSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); + settings.put(OTelLogsSourceConfig.RETRY_INFO, new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); return new PluginSetting(PLUGIN_NAME, settings); } } diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java index 41de6302ba..a2aab9259b 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java @@ -59,7 +59,7 @@ class OtelLogsSourceRetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21892/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); @Mock private PluginFactory pluginFactory; From 9c6d808d64153b6443e00d8212fdebfeee679ba9 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 23 Sep 2024 14:41:07 +0200 Subject: [PATCH 11/18] Use Duration for delays in RetryInfoConfig for OTelMetricsSource (cherry picked from commit 091e9a6b0062b267e2bd1a19ce8807cef759f76d) Signed-off-by: Tomas Longo --- .../otellogs/OtelLogsSourceRetryInfoTest.java | 1 - .../source/otelmetrics/OTelMetricsSource.java | 9 +++++++-- .../source/otelmetrics/RetryInfoConfig.java | 16 +++++++++------- .../OTelMetricsSourceRetryInfoTest.java | 2 +- .../OtelMetricsSourceConfigTests.java | 3 ++- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java index a2aab9259b..86c02356eb 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java @@ -9,7 +9,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index 419351ddcf..5ae9c413a6 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -226,9 +226,14 @@ public void stop() { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - RetryInfoConfig retryInfo = oTelMetricsSourceConfig.getRetryInfo() != null ? oTelMetricsSourceConfig.getRetryInfo() : new RetryInfoConfig(100, 2000); + Duration defaultMinDelay = Duration.ofMillis(100); + Duration defaultMaxDelay = Duration.ofMillis(2000); - return new GrpcRequestExceptionHandler(pluginMetrics, Duration.ofMillis(retryInfo.getMinDelay()), Duration.ofMillis(retryInfo.getMaxDelay())); + RetryInfoConfig retryInfo = oTelMetricsSourceConfig.getRetryInfo() != null + ? oTelMetricsSourceConfig.getRetryInfo() + : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + + return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } private List getAuthenticationInterceptor() { diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java index 7068597e98..c8ba7ed2f7 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java @@ -1,36 +1,38 @@ package org.opensearch.dataprepper.plugins.source.otelmetrics; +import java.time.Duration; + import com.fasterxml.jackson.annotation.JsonProperty; public class RetryInfoConfig { @JsonProperty("min_delay") - private Integer minDelay; + private Duration minDelay; @JsonProperty("max_delay") - private Integer maxDelay; + private Duration maxDelay; // Jackson needs this constructor public RetryInfoConfig() {} - public RetryInfoConfig(int minDelay, int maxDelay) { + public RetryInfoConfig(Duration minDelay, Duration maxDelay) { this.minDelay = minDelay; this.maxDelay = maxDelay; } - public int getMinDelay() { + public Duration getMinDelay() { return minDelay; } - public void setMinDelay(Integer minDelay) { + public void setMinDelay(Duration minDelay) { this.minDelay = minDelay; } - public int getMaxDelay() { + public Duration getMaxDelay() { return maxDelay; } - public void setMaxDelay(Integer maxDelay) { + public void setMaxDelay(Duration maxDelay) { this.maxDelay = maxDelay; } } diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java index 0595b4c029..1f1465b5da 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java @@ -59,7 +59,7 @@ class OTelMetricsSourceRetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(100, 2000); + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); @Mock private PluginFactory pluginFactory; diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index b2053e90ec..acfe7f0258 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -13,6 +13,7 @@ import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.model.configuration.PluginSetting; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -349,7 +350,7 @@ private PluginSetting completePluginSettingForOtelMetricsSource(final int reques settings.put(OTelMetricsSourceConfig.SSL_KEY_FILE, sslKeyFile); settings.put(OTelMetricsSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelMetricsSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); - settings.put(OTelMetricsSourceConfig.RETRY_INFO, new RetryInfoConfig(50, 100)); + settings.put(OTelMetricsSourceConfig.RETRY_INFO, new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); return new PluginSetting(PLUGIN_NAME, settings); } } From ccbfc9bb7df09655b6c23495a668725ed9e83b51 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 23 Sep 2024 14:53:29 +0200 Subject: [PATCH 12/18] Add documentation for retry information (cherry picked from commit a588b09ba792585ad34314f1527d6a1f4850fc3a) Signed-off-by: Tomas Longo --- data-prepper-plugins/otel-logs-source/README.md | 13 +++++++++++++ data-prepper-plugins/otel-metrics-source/README.md | 12 ++++++++++++ data-prepper-plugins/otel-trace-source/README.md | 13 +++++++++++++ 3 files changed, 38 insertions(+) diff --git a/data-prepper-plugins/otel-logs-source/README.md b/data-prepper-plugins/otel-logs-source/README.md index 547e65527b..20dde2008c 100644 --- a/data-prepper-plugins/otel-logs-source/README.md +++ b/data-prepper-plugins/otel-logs-source/README.md @@ -26,6 +26,19 @@ source: * `none`: no compression * `gzip`: apply GZip de-compression on the incoming request. +### Retry Information + +Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. + +```yaml +source: + otel_trace_source: + retry_info: + min_delay: 1000ms # defaults to 100ms + max_delay: 5s # defaults to 2s + +``` + ### SSL * ssl(Optional) => A boolean enables TLS/SSL. Default is ```true```. diff --git a/data-prepper-plugins/otel-metrics-source/README.md b/data-prepper-plugins/otel-metrics-source/README.md index 750520d5f3..05a21381e5 100644 --- a/data-prepper-plugins/otel-metrics-source/README.md +++ b/data-prepper-plugins/otel-metrics-source/README.md @@ -26,6 +26,18 @@ source: * `none`: no compression * `gzip`: apply GZip de-compression on the incoming request. +### Retry Information + +Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. + +```yaml +source: + otel_trace_source: + retry_info: + min_delay: 1000ms # defaults to 100ms + max_delay: 5s # defaults to 2s +``` + ### SSL * ssl(Optional) => A boolean enables TLS/SSL. Default is ```true```. diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index 957998489c..11c23e2359 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -42,6 +42,19 @@ For more information on migrating from Data Prepper 1.x to Data Prepper 2.x, see * `none`: no compression * `gzip`: apply GZip de-compression on the incoming request. + +### Retry Information + +Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. + +```yaml +source: + otel_trace_source: + retry_info: + min_delay: 1000ms # defaults to 100ms + max_delay: 5s # defaults to 2s +``` + ### Authentication Configurations By default, the otel-trace-source input is unauthenticated. From 9f3a39d9edbefa622f86a39ba1a2f917baa26365 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 4 Oct 2024 13:46:49 +0200 Subject: [PATCH 13/18] Add review findings Signed-off-by: Tomas Longo --- .../otel-logs-source/README.md | 3 +- .../source/otellogs/OTelLogsSource.java | 61 +++++++++---------- .../otellogs/OtelLogsSourceConfigTests.java | 3 +- .../otellogs/OtelLogsSourceRetryInfoTest.java | 6 +- .../otel-metrics-source/README.md | 2 +- .../source/otelmetrics/OTelMetricsSource.java | 8 +-- .../OTelMetricsSourceRetryInfoTest.java | 6 +- .../OtelMetricsSourceConfigTests.java | 1 - .../otel-trace-source/README.md | 2 +- .../source/oteltrace/OTelTraceSource.java | 8 +-- .../OTelTraceSourceRetryInfoTest.java | 6 +- 11 files changed, 52 insertions(+), 54 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/README.md b/data-prepper-plugins/otel-logs-source/README.md index 20dde2008c..33e142fd44 100644 --- a/data-prepper-plugins/otel-logs-source/README.md +++ b/data-prepper-plugins/otel-logs-source/README.md @@ -28,7 +28,7 @@ source: ### Retry Information -Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. +Data Prepper replies with a `RetryInfo` specifying how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. ```yaml source: @@ -36,7 +36,6 @@ source: retry_info: min_delay: 1000ms # defaults to 100ms max_delay: 5s # defaults to 2s - ``` ### SSL diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java index 737f791fb3..bbe06ae06e 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsSource.java @@ -5,21 +5,29 @@ package org.opensearch.dataprepper.plugins.source.otellogs; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; - +import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; +import com.linecorp.armeria.server.encoding.DecodingService; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.source.otellogs.certificate.CertificateProviderFactory; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import io.grpc.MethodDescriptor; +import io.grpc.ServerInterceptor; +import io.grpc.ServerInterceptors; +import io.grpc.protobuf.services.ProtoReflectionService; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -28,28 +36,19 @@ import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; -import org.opensearch.dataprepper.plugins.otel.codec.OTelLogsDecoder; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; -import org.opensearch.dataprepper.plugins.source.otellogs.certificate.CertificateProviderFactory; +import org.opensearch.dataprepper.model.codec.ByteDecoder; +import org.opensearch.dataprepper.plugins.otel.codec.OTelLogsDecoder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; -import com.linecorp.armeria.server.Server; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.encoding.DecodingService; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; - -import io.grpc.MethodDescriptor; -import io.grpc.ServerInterceptor; -import io.grpc.ServerInterceptors; -import io.grpc.protobuf.services.ProtoReflectionService; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; -import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; @DataPrepperPlugin(name = "otel_logs_source", pluginType = Source.class, pluginConfigurationType = OTelLogsSourceConfig.class) public class OTelLogsSource implements Source> { @@ -57,6 +56,9 @@ public class OTelLogsSource implements Source> { private static final String PIPELINE_NAME_PLACEHOLDER = "${pipelineName}"; static final String SERVER_CONNECTIONS = "serverConnections"; + // Default RetryInfo with minimum 100ms and maximum 2s + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + private final OTelLogsSourceConfig oTelLogsSourceConfig; private final String pipelineName; private final PluginMetrics pluginMetrics; @@ -207,12 +209,9 @@ public void stop() { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - Duration defaultMinDelay = Duration.ofMillis(100); - Duration defaultMaxDelay = Duration.ofMillis(2000); - RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo() != null ? oTelLogsSourceConfig.getRetryInfo() - : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + : DEFAULT_RETRY_INFO; return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java index f5965b3431..6c6bacf25b 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java @@ -35,6 +35,7 @@ import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.PORT; import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.PROTO_REFLECTION_SERVICE; import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.REQUEST_TIMEOUT; +import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.RETRY_INFO; import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.SSL; import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.SSL_KEY_CERT_FILE; import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.SSL_KEY_FILE; @@ -323,7 +324,7 @@ private PluginSetting completePluginSettingForOtelLogsSource(final int requestTi settings.put(SSL_KEY_FILE, sslKeyFile); settings.put(THREAD_COUNT, threadCount); settings.put(MAX_CONNECTION_COUNT, maxConnectionCount); - settings.put(OTelLogsSourceConfig.RETRY_INFO, new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); + settings.put(RETRY_INFO, new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); return new PluginSetting(PLUGIN_NAME, settings); } } diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java index 86c02356eb..ad18b7cec8 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java @@ -114,7 +114,7 @@ public void gRPC_failed_request_returns_minimal_delay_in_status() throws Excepti .build(LogsServiceGrpc.LogsServiceBlockingStub.class); final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); - RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); } @@ -125,7 +125,7 @@ public void gRPC_failed_request_returns_extended_delay_in_status() throws Except assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); } - private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + private RetryInfo extractRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); return RetryInfo.parseFrom(status.getDetails(0).getValue()); } @@ -141,7 +141,7 @@ private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); } - return extracRetryInfoFromStatusRuntimeException(e); + return extractRetryInfoFromStatusRuntimeException(e); } private ExportLogsServiceRequest createExportLogsRequest() { diff --git a/data-prepper-plugins/otel-metrics-source/README.md b/data-prepper-plugins/otel-metrics-source/README.md index 05a21381e5..93cb456cea 100644 --- a/data-prepper-plugins/otel-metrics-source/README.md +++ b/data-prepper-plugins/otel-metrics-source/README.md @@ -28,7 +28,7 @@ source: ### Retry Information -Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. +Data Prepper replies with a `RetryInfo` specifying how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. ```yaml source: diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index 5ae9c413a6..8b168ba4dc 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -62,6 +62,9 @@ public class OTelMetricsSource implements Source> { private static final String PIPELINE_NAME_PLACEHOLDER = "${pipelineName}"; static final String SERVER_CONNECTIONS = "serverConnections"; + // Default RetryInfo with minimum 100ms and maximum 2s + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + private final OTelMetricsSourceConfig oTelMetricsSourceConfig; private final String pipelineName; private final PluginMetrics pluginMetrics; @@ -226,12 +229,9 @@ public void stop() { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - Duration defaultMinDelay = Duration.ofMillis(100); - Duration defaultMaxDelay = Duration.ofMillis(2000); - RetryInfoConfig retryInfo = oTelMetricsSourceConfig.getRetryInfo() != null ? oTelMetricsSourceConfig.getRetryInfo() - : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + : DEFAULT_RETRY_INFO; return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java index 1f1465b5da..cd151bd553 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java @@ -114,7 +114,7 @@ public void gRPC_failed_request_returns_minimal_delay_in_status() throws Excepti .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); - RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); } @@ -125,7 +125,7 @@ public void gRPC_failed_request_returns_extended_delay_in_status() throws Except assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); } - private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + private RetryInfo extractRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); return RetryInfo.parseFrom(status.getDetails(0).getValue()); } @@ -141,7 +141,7 @@ private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); } - return extracRetryInfoFromStatusRuntimeException(e); + return extractRetryInfoFromStatusRuntimeException(e); } private ExportMetricsServiceRequest createExportMetricsRequest() { diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index acfe7f0258..cb7b8026d5 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -321,7 +321,6 @@ void testRetryInfoConfig() { final OTelMetricsSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); - RetryInfoConfig retryInfo = otelTraceSourceConfig.getRetryInfo(); assertThat(retryInfo.getMaxDelay(), equalTo(100)); assertThat(retryInfo.getMinDelay(), equalTo(50)); diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index 11c23e2359..6e5172b1fc 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -45,7 +45,7 @@ For more information on migrating from Data Prepper 1.x to Data Prepper 2.x, see ### Retry Information -Data Prepper gives clients a hint on how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. +Data Prepper replies with a `RetryInfo` specifying how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. ```yaml source: diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 16d1eba2b0..311abc3e7b 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -61,6 +61,9 @@ public class OTelTraceSource implements Source> { static final String SERVER_CONNECTIONS = "serverConnections"; private static final String PIPELINE_NAME_PLACEHOLDER = "${pipelineName}"; + // Default RetryInfo with minimum 100ms and maximum 2s + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + private final OTelTraceSourceConfig oTelTraceSourceConfig; private final PluginMetrics pluginMetrics; private final GrpcAuthenticationProvider authenticationProvider; @@ -209,12 +212,9 @@ public void start(Buffer> buffer) { } private GrpcExceptionHandlerFunction createGrpExceptionHandler() { - Duration defaultMinDelay = Duration.ofMillis(100); - Duration defaultMaxDelay = Duration.ofMillis(2000); - RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null ? oTelTraceSourceConfig.getRetryInfo() - : new RetryInfoConfig(defaultMinDelay, defaultMaxDelay); + : DEFAULT_RETRY_INFO; return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java index a00faa33b9..14979adc03 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java @@ -114,7 +114,7 @@ public void gRPC_failed_request_returns_minimal_delay_in_status() throws Excepti .build(TraceServiceGrpc.TraceServiceBlockingStub.class); final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); - RetryInfo retryInfo = extracRetryInfoFromStatusRuntimeException(statusRuntimeException); + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); } @@ -125,7 +125,7 @@ public void gRPC_failed_request_returns_extended_delay_in_status() throws Except assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); } - private RetryInfo extracRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { + private RetryInfo extractRetryInfoFromStatusRuntimeException(StatusRuntimeException e) throws InvalidProtocolBufferException { com.google.rpc.Status status = com.google.rpc.Status.parseFrom(e.getTrailers().get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); return RetryInfo.parseFrom(status.getDetails(0).getValue()); } @@ -141,7 +141,7 @@ private RetryInfo callService3TimesAndReturnRetryInfo() throws Exception { e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); } - return extracRetryInfoFromStatusRuntimeException(e); + return extractRetryInfoFromStatusRuntimeException(e); } private ExportTraceServiceRequest createExportTraceRequest() { From b3fe06d4de8044c6e85fd731ad36f7f90e24aef4 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Tue, 8 Oct 2024 16:01:11 +0200 Subject: [PATCH 14/18] Add java time module to tests Signed-off-by: Tomas Longo --- .../source/otellogs/OtelLogsSourceConfigTests.java | 8 +++++--- .../source/otelmetrics/OtelMetricsSourceConfigTests.java | 8 +++++--- .../plugins/source/oteltrace/OTelTraceSourceTest.java | 3 ++- .../source/oteltrace/OtelTraceSourceConfigTests.java | 8 +++++--- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java index 6c6bacf25b..abf259da33 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigTests.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.source.otellogs; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -44,7 +46,7 @@ import static org.hamcrest.Matchers.equalTo; class OtelLogsSourceConfigTests { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String PLUGIN_NAME = "otel_logs_source"; private static final String TEST_KEY_CERT = "test.crt"; private static final String TEST_KEY = "test.key"; @@ -297,8 +299,8 @@ void testRetryInfoConfig() { RetryInfoConfig retryInfo = oTelLogsSourceConfig.getRetryInfo(); - assertThat(retryInfo.getMaxDelay(), equalTo(100)); - assertThat(retryInfo.getMinDelay(), equalTo(50)); + assertThat(retryInfo.getMaxDelay(), equalTo(Duration.ofMillis(100))); + assertThat(retryInfo.getMinDelay(), equalTo(Duration.ofMillis(50))); } private PluginSetting completePluginSettingForOtelLogsSource(final int requestTimeoutInMillis, diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index cb7b8026d5..3ee075cc03 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.source.otelmetrics; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -31,7 +33,7 @@ import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_THREAD_COUNT; class OtelMetricsSourceConfigTests { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String PLUGIN_NAME = "otel_metrics_source"; private static final String TEST_KEY_CERT = "test.crt"; private static final String TEST_KEY = "test.key"; @@ -322,8 +324,8 @@ void testRetryInfoConfig() { final OTelMetricsSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); RetryInfoConfig retryInfo = otelTraceSourceConfig.getRetryInfo(); - assertThat(retryInfo.getMaxDelay(), equalTo(100)); - assertThat(retryInfo.getMinDelay(), equalTo(50)); + assertThat(retryInfo.getMaxDelay(), equalTo(Duration.ofMillis(100))); + assertThat(retryInfo.getMinDelay(), equalTo(Duration.ofMillis(50))); } private PluginSetting completePluginSettingForOtelMetricsSource(final int requestTimeoutInMillis, diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 8c91518959..f52f2379dd 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; @@ -135,7 +136,7 @@ class OTelTraceSourceTest { private static final String USERNAME = "test_user"; private static final String PASSWORD = "test_password"; private static final String TEST_PATH = "${pipelineName}/v1/traces"; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index a5a7e885c1..f67a637220 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -11,6 +11,8 @@ import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.model.configuration.PluginSetting; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + import org.junit.jupiter.api.Test; import java.time.Duration; @@ -31,7 +33,7 @@ import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_THREAD_COUNT; class OtelTraceSourceConfigTests { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String PLUGIN_NAME = "otel_trace_source"; private static final String TEST_KEY_CERT = "test.crt"; private static final String TEST_KEY = "test.key"; @@ -321,8 +323,8 @@ void testRetryInfoConfig() { final OTelTraceSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelTraceSourceConfig.class); - assertThat(otelTraceSourceConfig.getRetryInfo().getMaxDelay(), equalTo(100)); - assertThat(otelTraceSourceConfig.getRetryInfo().getMinDelay(), equalTo(50)); + assertThat(otelTraceSourceConfig.getRetryInfo().getMaxDelay(), equalTo(Duration.ofMillis(100))); + assertThat(otelTraceSourceConfig.getRetryInfo().getMinDelay(), equalTo(Duration.ofMillis(50))); } private PluginSetting completePluginSettingForOtelTraceSource(final int requestTimeoutInMillis, From b29246aaf7a0044b66ded84b671ced2aaf0e1a95 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 11 Oct 2024 16:37:59 +0200 Subject: [PATCH 15/18] 4119 Initialize with a buffer to prevent pre-mature backoff calculation Signed-off-by: Tomas Longo --- .../dataprepper/GrpcRetryInfoCalculator.java | 3 +- .../GrpcRetryInfoCalculatorTest.java | 39 +++++++++++++++---- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java index 1211e5e8ba..2b74b0b4bc 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java @@ -17,7 +17,8 @@ class GrpcRetryInfoCalculator { GrpcRetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { this.minimumDelay = minimumDelay; this.maximumDelay = maximumDelay; - this.lastTimeCalled = new AtomicReference<>(Instant.now()); + // Create a cushion so that the calculator treats a first quick exception (after prepper startup) as normal request (e.g. does not calculate a backoff) + this.lastTimeCalled = new AtomicReference<>(Instant.now().minus(maximumDelay)); this.nextDelay = new AtomicReference<>(minimumDelay); } diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java index 5848330f08..5611826ef7 100644 --- a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java @@ -25,10 +25,12 @@ public void testExponentialBackoff() { RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); RetryInfo retryInfo3 = calculator.createRetryInfo(); + RetryInfo retryInfo4 = calculator.createRetryInfo(); assertThat(retryInfo1.getRetryDelay().getSeconds(), equalTo(1L)); - assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(2L)); - assertThat(retryInfo3.getRetryDelay().getSeconds(), equalTo(4L)); + assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(1L)); + assertThat(retryInfo3.getRetryDelay().getSeconds(), equalTo(2L)); + assertThat(retryInfo4.getRetryDelay().getSeconds(), equalTo(4L)); } @Test @@ -40,21 +42,42 @@ public void testUsesMaximumAsLongestDelay() { RetryInfo retryInfo3 = calculator.createRetryInfo(); assertThat(retryInfo1.getRetryDelay().getSeconds(), equalTo(1L)); - assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(2L)); + assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(1L)); assertThat(retryInfo3.getRetryDelay().getSeconds(), equalTo(2L)); } @Test public void testResetAfterDelayWearsOff() throws InterruptedException { + int minDelayNanos = 1_000_000; GrpcRetryInfoCalculator calculator = - new GrpcRetryInfoCalculator(Duration.ofNanos(1_000_000), Duration.ofSeconds(1)); + new GrpcRetryInfoCalculator(Duration.ofNanos(minDelayNanos), Duration.ofSeconds(1)); + RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); - Thread.sleep(5); RetryInfo retryInfo3 = calculator.createRetryInfo(); + sleep(retryInfo3); + RetryInfo retryInfo4 = calculator.createRetryInfo(); + + assertThat(retryInfo1.getRetryDelay().getNanos(), equalTo(minDelayNanos)); + assertThat(retryInfo2.getRetryDelay().getNanos(), equalTo(minDelayNanos)); + assertThat(retryInfo3.getRetryDelay().getNanos(), equalTo(minDelayNanos * 2)); + assertThat(retryInfo4.getRetryDelay().getNanos(), equalTo(minDelayNanos)); + } + + @Test + public void testQuickFirstExceptionDoesNotTriggerBackoffCalculationEvenWithLongMinDelay() throws InterruptedException { + GrpcRetryInfoCalculator calculator = + new GrpcRetryInfoCalculator(Duration.ofSeconds(10), Duration.ofSeconds(20)); + + RetryInfo retryInfo1 = calculator.createRetryInfo(); + RetryInfo retryInfo2 = calculator.createRetryInfo(); + + assertThat(retryInfo1.getRetryDelay().getSeconds(), equalTo(10L)); + assertThat(retryInfo2.getRetryDelay().getSeconds(), equalTo(10L)); + } - assertThat(retryInfo1.getRetryDelay().getNanos(), equalTo(1_000_000)); - assertThat(retryInfo2.getRetryDelay().getNanos(), equalTo(2_000_000)); - assertThat(retryInfo3.getRetryDelay().getNanos(), equalTo(1_000_000)); + private void sleep(RetryInfo retryInfo) throws InterruptedException { + // make sure we let enough time pass by adding a few milliseconds on top + Thread.sleep((retryInfo.getRetryDelay().getNanos() / 1_000_000) + 200 ); } } From 3feda83d88d1e05657de8b1923864f3eac3cd405 Mon Sep 17 00:00:00 2001 From: Karsten Schnitter Date: Fri, 1 Nov 2024 21:17:16 +0100 Subject: [PATCH 16/18] Apply suggestions from code review Co-authored-by: David Venable Signed-off-by: Karsten Schnitter --- .../dataprepper/plugins/source/otellogs/RetryInfoConfig.java | 4 ++-- .../dataprepper/plugins/source/oteltrace/RetryInfoConfig.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java index e3dbf028d4..1ae026202e 100644 --- a/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-logs-source/src/main/java/org/opensearch/dataprepper/plugins/source/otellogs/RetryInfoConfig.java @@ -6,10 +6,10 @@ public class RetryInfoConfig { - @JsonProperty("min_delay") + @JsonProperty(value = "min_delay", defaultValue = "100ms") private Duration minDelay; - @JsonProperty("max_delay") + @JsonProperty(value = "max_delay", defaultValue = "2s") private Duration maxDelay; // Jackson needs this constructor diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java index 84f95c82bf..7418a909f3 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/RetryInfoConfig.java @@ -6,10 +6,10 @@ public class RetryInfoConfig { - @JsonProperty("min_delay") + @JsonProperty(value = "min_delay", defaultValue = "100ms") private Duration minDelay; - @JsonProperty("max_delay") + @JsonProperty(value = "max_delay", defaultValue = "2s") private Duration maxDelay; // Jackson needs this constructor From 9e49340fa1f7f79fa4e9f27f2a5971d2c08b26be Mon Sep 17 00:00:00 2001 From: Karsten Schnitter Date: Fri, 1 Nov 2024 21:20:17 +0100 Subject: [PATCH 17/18] Apply suggestions from code review Co-authored-by: David Venable Signed-off-by: Karsten Schnitter --- .../plugins/source/otelmetrics/RetryInfoConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java index c8ba7ed2f7..5fd80133c6 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/RetryInfoConfig.java @@ -6,10 +6,10 @@ public class RetryInfoConfig { - @JsonProperty("min_delay") + @JsonProperty(value = "min_delay", defaultValue = "100ms") private Duration minDelay; - @JsonProperty("max_delay") + @JsonProperty(value = "max_delay", defaultValue = "2s") private Duration maxDelay; // Jackson needs this constructor From 36deb0cad1c265df3f87d53098133a3704b1a582 Mon Sep 17 00:00:00 2001 From: Karsten Schnitter Date: Fri, 1 Nov 2024 21:39:50 +0100 Subject: [PATCH 18/18] Rename Test-Cases to Adhere to Convention Signed-off-by: Karsten Schnitter --- ...urceRetryInfoTest.java => OtelLogsSource_RetryInfoTest.java} | 2 +- ...eRetryInfoTest.java => OTelMetricsSource_RetryInfoTest.java} | 2 +- ...rceRetryInfoTest.java => OTelTraceSource_RetryInfoTest.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/{OtelLogsSourceRetryInfoTest.java => OtelLogsSource_RetryInfoTest.java} (99%) rename data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/{OTelMetricsSourceRetryInfoTest.java => OTelMetricsSource_RetryInfoTest.java} (99%) rename data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/{OTelTraceSourceRetryInfoTest.java => OTelTraceSource_RetryInfoTest.java} (99%) diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSource_RetryInfoTest.java similarity index 99% rename from data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java rename to data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSource_RetryInfoTest.java index ad18b7cec8..40cdaa1091 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSource_RetryInfoTest.java @@ -55,7 +55,7 @@ import io.opentelemetry.proto.resource.v1.Resource; @ExtendWith(MockitoExtension.class) -class OtelLogsSourceRetryInfoTest { +class OtelLogsSource_RetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21892/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java similarity index 99% rename from data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java rename to data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java index cd151bd553..403efee46b 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java @@ -56,7 +56,7 @@ import io.opentelemetry.proto.resource.v1.Resource; @ExtendWith(MockitoExtension.class) -class OTelMetricsSourceRetryInfoTest { +class OTelMetricsSource_RetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java similarity index 99% rename from data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java rename to data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java index 14979adc03..b3f2cb6de8 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java @@ -52,7 +52,7 @@ import io.opentelemetry.proto.trace.v1.Span; @ExtendWith(MockitoExtension.class) -class OTelTraceSourceRetryInfoTest { +class OTelTraceSource_RetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000));