diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 891c18a25..ea66023e4 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -19,7 +19,7 @@ dependencies { implementation project(':TrafficCapture:captureProtobufs') implementation project(':coreUtilities') implementation project(':awsUtilities') - + implementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') implementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') runtimeOnly project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') runtimeOnly project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java index 4b1cf2286..ca9d38815 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java @@ -13,6 +13,7 @@ import org.opensearch.migrations.replay.datatypes.UniqueSourceRequestKey; import org.opensearch.migrations.transform.IJsonTransformer; +import org.opensearch.migrations.transform.TransformationLoader; import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.ByteBuf; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index cf79ef66a..80793b457 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -29,6 +29,7 @@ import org.opensearch.migrations.transform.RemovingAuthTransformerFactory; import org.opensearch.migrations.transform.SigV4AuthTransformerFactory; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import org.opensearch.migrations.utils.ProcessHelpers; import com.beust.jcommander.JCommander; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java index dbb5b8c21..d17ea202d 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java @@ -17,6 +17,7 @@ import org.opensearch.migrations.tracing.InstrumentationTest; import org.opensearch.migrations.tracing.TestContext; import org.opensearch.migrations.transform.IJsonTransformer; +import org.opensearch.migrations.transform.TransformationLoader; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index eaa3e7572..461b97141 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -31,6 +31,7 @@ import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.protos.WriteObservation; +import org.opensearch.migrations.transform.TransformationLoader; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index 6876ffb93..fc28e9d74 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -23,7 +23,6 @@ import org.opensearch.migrations.replay.ReplayUtils; import org.opensearch.migrations.replay.RequestTransformerAndSender; import org.opensearch.migrations.replay.TimeShifter; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; import org.opensearch.migrations.replay.http.retries.NoRetryEvaluatorFactory; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; @@ -35,6 +34,7 @@ import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.tracing.InstrumentationTest; import org.opensearch.migrations.tracing.TestContext; +import org.opensearch.migrations.transform.TransformationLoader; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.handler.codec.http.HttpHeaderNames; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index 54c813bff..7e106b433 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -15,7 +15,6 @@ import org.opensearch.migrations.replay.AggregatedRawResponse; import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler; import org.opensearch.migrations.replay.TestUtils; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.util.TrackedFuture; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; @@ -24,6 +23,7 @@ import org.opensearch.migrations.transform.JsonCompositeTransformer; import org.opensearch.migrations.transform.JsonKeysForHttpMessage; import org.opensearch.migrations.transform.RemovingAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullReplayerWithTracingChecksTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullReplayerWithTracingChecksTest.java index b239ead59..251734239 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullReplayerWithTracingChecksTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullReplayerWithTracingChecksTest.java @@ -13,7 +13,6 @@ import org.opensearch.migrations.replay.RootReplayerConstructorExtensions; import org.opensearch.migrations.replay.TestHttpServerContext; import org.opensearch.migrations.replay.TimeShifter; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.traffic.source.ArrayCursorTrafficCaptureSource; import org.opensearch.migrations.replay.traffic.source.ArrayCursorTrafficSourceContext; import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; @@ -28,6 +27,7 @@ import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.protos.WriteObservation; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullTrafficReplayerTest.java index c2c7dbf5e..ff3c46c43 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/FullTrafficReplayerTest.java @@ -22,7 +22,6 @@ import org.opensearch.migrations.replay.TestHttpServerContext; import org.opensearch.migrations.replay.TimeShifter; import org.opensearch.migrations.replay.TrafficReplayerTopLevel; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamAndKey; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKeyAndContext; @@ -46,6 +45,7 @@ import org.opensearch.migrations.transform.IAuthTransformerFactory; import org.opensearch.migrations.transform.IJsonTransformer; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import lombok.Lombok; import lombok.SneakyThrows; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/SlowAndExpiredTrafficStreamBecomesTwoTargetChannelsTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/SlowAndExpiredTrafficStreamBecomesTwoTargetChannelsTest.java index de3103d03..c313e523c 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/SlowAndExpiredTrafficStreamBecomesTwoTargetChannelsTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/SlowAndExpiredTrafficStreamBecomesTwoTargetChannelsTest.java @@ -17,7 +17,6 @@ import org.opensearch.migrations.replay.RootReplayerConstructorExtensions; import org.opensearch.migrations.replay.TimeShifter; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.traffic.source.ArrayCursorTrafficCaptureSource; import org.opensearch.migrations.replay.traffic.source.ArrayCursorTrafficSourceContext; import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; @@ -31,6 +30,7 @@ import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/TrafficReplayerRunner.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/TrafficReplayerRunner.java index 657601749..79885d1f7 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/TrafficReplayerRunner.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/e2etests/TrafficReplayerRunner.java @@ -22,13 +22,13 @@ import org.opensearch.migrations.replay.TimeShifter; import org.opensearch.migrations.replay.TrafficReplayer; import org.opensearch.migrations.replay.TrafficReplayerTopLevel; -import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.tracing.IRootReplayerContext; import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; import org.opensearch.migrations.tracing.TestContext; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import com.google.common.base.Strings; import lombok.AllArgsConstructor; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/transform/HeaderTransformerTest.java similarity index 94% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java rename to TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/transform/HeaderTransformerTest.java index 9b5c2ce35..2ac621929 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/transform/HeaderTransformerTest.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.replay.transform; import java.time.Duration; import java.util.AbstractMap; @@ -10,12 +10,16 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.opensearch.migrations.replay.AggregatedRawResponse; +import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler; +import org.opensearch.migrations.replay.TestUtils; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.util.TrackedFuture; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.tracing.InstrumentationTest; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.opensearch.migrations.transform.TransformationLoader; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java new file mode 100644 index 000000000..0e7ff4b46 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java @@ -0,0 +1,24 @@ +package org.opensearch.migrations.transform; + +import java.util.Map; + +import io.burt.jmespath.BaseRuntime; +import io.burt.jmespath.Expression; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JsonJMESPathPredicate implements IJsonPredicate { + + Expression expression; + + public JsonJMESPathPredicate(BaseRuntime runtime, String script) { + this.expression = runtime.compile(script); + } + + @Override + public boolean test(Map incomingJson) { + var output = expression.search(incomingJson); + log.atDebug().setMessage("output={}").addArgument(output).log(); + return (Boolean) output; + } +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java new file mode 100644 index 000000000..35d4b85f8 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java @@ -0,0 +1,45 @@ +package org.opensearch.migrations.transform; + +import java.util.Map; + +import io.burt.jmespath.BaseRuntime; +import io.burt.jmespath.jcf.JcfRuntime; + +public class JsonJMESPathPredicateProvider implements IJsonPredicateProvider { + + public static final String SCRIPT_KEY = "script"; + private BaseRuntime adapterRuntime; + + public JsonJMESPathPredicateProvider() { + this.adapterRuntime = new JcfRuntime(); + } + + @Override + public IJsonPredicate createPredicate(Object jsonConfig) { + try { + if (jsonConfig instanceof Map) { + @SuppressWarnings("unchecked") + var jsonConfigMap = (Map) jsonConfig; + if (jsonConfigMap.size() != 1) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + var scriptValue = jsonConfigMap.get(SCRIPT_KEY); + if (!(scriptValue instanceof String)) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + return new JsonJMESPathPredicate(adapterRuntime, (String) scriptValue); + } + throw new IllegalArgumentException(getConfigUsageStr()); + } catch (ClassCastException e) { + throw new IllegalArgumentException(getConfigUsageStr(), e); + } + } + + private String getConfigUsageStr() { + return this.getClass().getName() + + " expects the incoming configuration " + + "to be a Map. " + + "Each of the Maps should have one key-value of \"script\": \"...\". " + + "Script values should be a fully-formed inlined JsonPath queries."; + } +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonPredicateProvider b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonPredicateProvider new file mode 100644 index 000000000..ea0585544 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonPredicateProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonJMESPathPredicateProvider diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java deleted file mode 100644 index d037c69d7..000000000 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.opensearch.migrations.replay; - -import java.io.IOException; -import java.io.InputStream; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonJoltTransformBuilder; -import org.opensearch.migrations.transform.JsonJoltTransformer; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -@Slf4j -@WrapWithNettyLeakDetection(disableLeakChecks = true) -class JsonTransformerTest { - public static final String DUMMY_HOSTNAME_TEST_STRING = "THIS_IS_A_TEST_STRING_THAT_ONLY_EXISTS_IN_ONE_PLACE"; - ObjectMapper mapper = new ObjectMapper(); - static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>() { - }; - - public JsonTransformerTest() { - mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); - } - - private Map parseStringAsJson(String jsonStr) throws JsonProcessingException { - return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE); - } - - @SneakyThrows - private Map parseSampleRequestFromResource(String path) { - try (InputStream inputStream = JsonTransformerTest.class.getResourceAsStream("/requests/" + path)) { - return mapper.readValue(inputStream, TYPE_REFERENCE_FOR_MAP_TYPE); - } - } - - private String emitJson(Object transformedDocument) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); // optional - return mapper.writeValueAsString(transformedDocument); - } - - @Test - public void testSimpleTransform() throws JsonProcessingException { - final String TEST_DOCUMENT = "{\"Hello\":\"world\"}"; - var documentJson = parseStringAsJson(TEST_DOCUMENT); - var transformer = JsonJoltTransformer.newBuilder() - .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.PASS_THRU) - .build(); - var transformedDocument = transformer.transformJson(documentJson); - var finalOutputStr = emitJson(transformedDocument); - - Assertions.assertEquals(TEST_DOCUMENT, finalOutputStr); - } - - @Test - public void testHttpTransform() throws IOException { - var testResourceName = "parsed/post_formUrlEncoded_withFixedLength.json"; - final var documentJson = parseSampleRequestFromResource(testResourceName); - var transformer = JsonJoltTransformer.newBuilder().addHostSwitchOperation(DUMMY_HOSTNAME_TEST_STRING).build(); - var transformedDocument = transformer.transformJson(documentJson); - String transformedJsonOutputStr = emitJson(transformedDocument); - log.info("transformed json document: " + transformedJsonOutputStr); - Assertions.assertTrue(transformedJsonOutputStr.contains(DUMMY_HOSTNAME_TEST_STRING)); - } -} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/AddCompressionEncodingTest.java similarity index 95% rename from TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/AddCompressionEncodingTest.java index 6f13e6c6b..e9b3d359b 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/AddCompressionEncodingTest.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -12,6 +12,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; +import org.opensearch.migrations.replay.AggregatedRawResponse; +import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler; +import org.opensearch.migrations.replay.Utils; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.tracing.InstrumentationTest; diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/PayloadRepackingTest.java similarity index 98% rename from TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/PayloadRepackingTest.java index 5fce12ac1..0a84d67d6 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/transform/replay/PayloadRepackingTest.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -8,6 +8,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import org.opensearch.migrations.replay.TestUtils; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.tracing.InstrumentationTest; import org.opensearch.migrations.transform.JsonJoltTransformBuilder; diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java new file mode 100644 index 000000000..eae81f022 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.transform; + +import java.util.Map; +import java.util.function.Predicate; + +public interface IJsonPredicate extends Predicate> { +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicateProvider.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicateProvider.java new file mode 100644 index 000000000..4ad01f9ac --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicateProvider.java @@ -0,0 +1,23 @@ +package org.opensearch.migrations.transform; + +import lombok.NonNull; + +public interface IJsonPredicateProvider { + /** + * Create a new Predicate from the given configuration. This Predicate + * will be used repeatedly and concurrently from different threads against + * messages. + * @param jsonConfig is a List, Map, String, or null that should be used to configure the + * IJsonPredicate that is being created + * @return + */ + IJsonPredicate createPredicate(Object jsonConfig); + + /** + * Friendly name that can be used as a key to identify Predicate providers. + * @return + */ + default @NonNull String getName() { + return this.getClass().getSimpleName(); + } +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java new file mode 100644 index 000000000..816281985 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java @@ -0,0 +1,21 @@ +package org.opensearch.migrations.transform; + +import java.util.Map; + +public class JsonConditionalTransformer implements IJsonTransformer { + IJsonPredicate jsonPredicate; + IJsonTransformer jsonTransformer; + + public JsonConditionalTransformer(IJsonPredicate jsonPredicate, IJsonTransformer jsonTransformer) { + this.jsonPredicate = jsonPredicate; + this.jsonTransformer = jsonTransformer; + } + + @Override + public Map transformJson(Map incomingJson) { + if (jsonPredicate.test(incomingJson)) { + return jsonTransformer.transformJson(incomingJson); + } + return incomingJson; + } +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle new file mode 100644 index 000000000..1e77f7e18 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + +plugins { + id 'org.opensearch.migrations.java-library-conventions' + id 'org.owasp.dependencycheck' version '8.2.1' + id 'io.freefair.lombok' +} + +dependencies { + implementation group: 'org.slf4j', name: 'slf4j-api' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + + implementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + + testImplementation testFixtures(project(path: ':testHelperFixtures')) + testImplementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') + testImplementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') + testImplementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + testImplementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') + testImplementation project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformer') + + testRuntimeOnly project(':TrafficCapture:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + testImplementation group: 'io.burt', name: 'jmespath-core' +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java new file mode 100644 index 000000000..c410b9595 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java @@ -0,0 +1,47 @@ +package org.opensearch.migrations.transform; + +import java.util.List; +import java.util.stream.Collectors; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JsonConditionalTransformerProvider implements IJsonTransformerProvider { + + public JsonConditionalTransformerProvider() { + } + + @Override + @SneakyThrows + public IJsonTransformer createTransformer(Object jsonConfig) { + if(jsonConfig instanceof List) { + @SuppressWarnings("unchecked") + var configs = (List) jsonConfig; + if (configs.size() != 2) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + var predicateConfig = configs.get(0); + var transformerConfig = configs.get(1); + IJsonPredicate predicate = new PredicateLoader() + .getPredicateFactoryFromServiceLoaderParsed(predicateConfig); + @SuppressWarnings("unchecked") + List transformer = new TransformationLoader() + .getTransformerFactoryFromServiceLoaderParsed((List) transformerConfig) + .collect(Collectors.toList()); + + return new JsonConditionalTransformer( + predicate, + new JsonCompositeTransformer(transformer.toArray(IJsonTransformer[]::new))); + } + throw new IllegalArgumentException(getConfigUsageStr()); + } + + private String getConfigUsageStr() { + return this.getClass().getName() + + " expects the incoming configuration " + + "to be a List with length 2. " + + "Script values should be a fully-formed inlined JsonPath queries encoded as a json value. " + + "All of the values within a configuration will be concatenated into one chained transformation."; + } +} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java new file mode 100644 index 000000000..0b8fa3866 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java @@ -0,0 +1,101 @@ +package org.opensearch.migrations.transform; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PredicateLoader { + public static final String WRONG_JSON_STRUCTURE_MESSAGE = + "Must specify the top-level configuration map" + + " that only has one key, where the key is the name of the Predicate to be configured."; + public static final Pattern CLASS_NAME_PATTERN = Pattern.compile("^[^{}]*$"); + private final List providers; + ObjectMapper objMapper = new ObjectMapper(); + + public PredicateLoader() { + ServiceLoader predicateProviders = ServiceLoader.load( + IJsonPredicateProvider.class + ); + var inProgressProviders = new ArrayList(); + for (var provider : predicateProviders) { + log.info("Adding IJsonPredicateProvider: " + provider); + inProgressProviders.add(provider); + } + providers = Collections.unmodifiableList(inProgressProviders); + log.atInfo() + .setMessage( + () -> "IJsonPredicateProvider loaded: " + + providers.stream().map(p -> p.getClass().toString()).collect(Collectors.joining("; ")) + ) + .log(); + } + + Map parseFullConfig(String fullConfig) throws JsonProcessingException { + if (CLASS_NAME_PATTERN.matcher(fullConfig).matches()) { + return Collections.singletonMap(fullConfig, null); + } else { + return objMapper.readValue(fullConfig, new TypeReference<>() {}); + } + } + + protected IJsonPredicate getPredicateFactoryFromServiceLoader(String fullConfig) + throws JsonProcessingException { + Map configMap = fullConfig == null ? Map.of() : parseFullConfig(fullConfig); + return getPredicateFactoryFromServiceLoaderParsed(configMap); + } + + public IJsonPredicate getPredicateFactoryFromServiceLoaderParsed(Object config) { + @SuppressWarnings("unchecked") + var configMap = (Map) config; + if (configMap.isEmpty() || providers.isEmpty()) { + log.atError().setMessage("No predicate configuration found for configuration {}") + .addArgument(config).log(); + throw new IllegalArgumentException("No predicate configuration found for configuration " + config); + } else { + return configurePredicateFromConfig(configMap); + } + } + + @SneakyThrows // JsonProcessingException should be impossible since the contents are those that were just parsed + private IJsonPredicate configurePredicateFromConfig(Map c) { + var keys = c.keySet(); + if (keys.size() != 1) { + throw new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE); + } + var key = keys.stream() + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE)); + for (var p : providers) { + var predicateName = p.getName(); + if (predicateName.equals(key)) { + var configuration = c.get(key); + log.atInfo() + .setMessage( + () -> "Creating a Predicate through provider=" + p + " with configuration=" + configuration + ) + .log(); + return p.createPredicate(configuration); + } + } + throw new IllegalArgumentException("Could not find a provider named: " + key); + } + + public IJsonPredicate getPredicateFactoryLoader(String fullConfig) { + try { + return getPredicateFactoryFromServiceLoader(fullConfig); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Could not parse the Predicate configuration as a json list", e); + } + } +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java similarity index 91% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java index 277b8be96..ffcd6b767 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform; import java.util.ArrayList; import java.util.Collections; @@ -10,11 +10,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.opensearch.migrations.transform.IJsonTransformer; -import org.opensearch.migrations.transform.IJsonTransformerProvider; -import org.opensearch.migrations.transform.JsonCompositeTransformer; -import org.opensearch.migrations.transform.JsonKeysForHttpMessage; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -58,7 +53,7 @@ List> parseFullConfig(String fullConfig) throws JsonProcessi } } - protected Stream getTransformerFactoryFromServiceLoader(String fullConfig) + public Stream getTransformerFactoryFromServiceLoader(String fullConfig) throws JsonProcessingException { var configList = fullConfig == null ? List.of() : parseFullConfig(fullConfig); if (configList.isEmpty() || providers.isEmpty()) { @@ -69,6 +64,15 @@ protected Stream getTransformerFactoryFromServiceLoader(String } } + public Stream getTransformerFactoryFromServiceLoaderParsed(List configList) { + if (configList.isEmpty() || providers.isEmpty()) { + log.warn("No transformer configuration specified. No custom transformations will be performed"); + return Stream.of(); + } else { + return configList.stream().map(c -> configureTransformerFromConfig((Map) c)); + } + } + @SneakyThrows // JsonProcessingException should be impossible since the contents are those that were just parsed private IJsonTransformer configureTransformerFromConfig(Map c) { var keys = c.keySet(); diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 000000000..74eb7c9ac --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonConditionalTransformerProvider diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java new file mode 100644 index 000000000..6bb93b1bf --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java @@ -0,0 +1,154 @@ +package org.opensearch.migrations.transform.replay; + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.transform.JsonJMESPathTransformer; +import org.opensearch.migrations.transform.JsonJoltTransformBuilder; +import org.opensearch.migrations.transform.JsonJoltTransformer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.burt.jmespath.jcf.JcfRuntime; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +@Slf4j +@WrapWithNettyLeakDetection(disableLeakChecks = true) +class JsonTransformerTest { + public static final String DUMMY_HOSTNAME_TEST_STRING = "THIS_IS_A_TEST_STRING_THAT_ONLY_EXISTS_IN_ONE_PLACE"; + ObjectMapper mapper = new ObjectMapper(); + static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>() { + }; + + public JsonTransformerTest() { + mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); + } + + private Map parseStringAsJson(String jsonStr) throws JsonProcessingException { + return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE); + } + + @SneakyThrows + private Map parseSampleRequestFromResource(String path) { + try (InputStream inputStream = JsonTransformerTest.class.getResourceAsStream("/requests/" + path)) { + return mapper.readValue(inputStream, TYPE_REFERENCE_FOR_MAP_TYPE); + } + } + + private String emitJson(Object transformedDocument) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); // optional + return mapper.writeValueAsString(transformedDocument); + } + + @Test + public void testSimpleTransform() throws JsonProcessingException { + final String TEST_DOCUMENT = "{\"Hello\":\"world\"}"; + var documentJson = parseStringAsJson(TEST_DOCUMENT); + var transformer = JsonJoltTransformer.newBuilder() + .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.PASS_THRU) + .build(); + var transformedDocument = transformer.transformJson(documentJson); + var finalOutputStr = emitJson(transformedDocument); + + Assertions.assertEquals(TEST_DOCUMENT, finalOutputStr); + } + + @Test + public void testHttpTransform() throws IOException { + var testResourceName = "parsed/post_formUrlEncoded_withFixedLength.json"; + final var documentJson = parseSampleRequestFromResource(testResourceName); + var transformer = JsonJoltTransformer.newBuilder().addHostSwitchOperation(DUMMY_HOSTNAME_TEST_STRING).build(); + var transformedDocument = transformer.transformJson(documentJson); + String transformedJsonOutputStr = emitJson(transformedDocument); + log.info("transformed json document: " + transformedJsonOutputStr); + Assertions.assertTrue(transformedJsonOutputStr.contains(DUMMY_HOSTNAME_TEST_STRING)); + } + + static final String TEST_INPUT_REQUEST = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/oldStyleIndex\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": {\n" + + " \"oldType\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + + static final String EXCISE_TYPE_EXPRESSION_STRING = "{\n" + + " \"method\": method,\n" + + " \"URI\": URI,\n" + + " \"headers\": headers,\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": payload.inlinedJsonBody.mappings.oldType\n" + + " }\n" + + " }\n" + + "}"; + + static Map parseStringAsJson(ObjectMapper mapper, String jsonStr) throws JsonProcessingException { + return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE); + } + + static String normalize(ObjectMapper mapper, String input) throws JsonProcessingException { + return mapper.writeValueAsString(mapper.readTree(input)); + } + + static String emitJson(ObjectMapper mapper, Object transformedDocument) throws JsonProcessingException { + mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); // optional + return mapper.writeValueAsString(transformedDocument); + } + + @Test + public void testSimpleTransformJMES() throws JsonProcessingException { + var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); + var transformer = new JsonJMESPathTransformer(new JcfRuntime(), EXCISE_TYPE_EXPRESSION_STRING); + var transformedDocument = transformer.transformJson(documentJson); + var outputStr = emitJson(mapper, transformedDocument); + + final String TEST_OUTPUT_REQUEST = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/oldStyleIndex\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Assertions.assertEquals(normalize(mapper, TEST_OUTPUT_REQUEST), normalize(mapper, outputStr)); + } + } diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJMESPathScriptsTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java similarity index 95% rename from TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJMESPathScriptsTest.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java index 51b3fcc3f..088015583 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJMESPathScriptsTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java @@ -1,8 +1,10 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.util.Map; import java.util.StringJoiner; +import org.opensearch.migrations.transform.TransformationLoader; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Assertions; diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java similarity index 93% rename from TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java index 094fd324e..a346370b4 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java @@ -1,8 +1,9 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.util.Map; import org.opensearch.migrations.transform.JsonKeysForHttpMessage; +import org.opensearch.migrations.transform.TransformationLoader; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -36,7 +37,10 @@ public void testAddGzip() throws Exception { @Test public void testAddGzipAndCustom() throws Exception { final var addGzip = "[" - + "{\"JsonJoltTransformerProvider\": { \"canned\": \"ADD_GZIP\" }}," + + "{\"JsonConditionalTransformerProvider\": [" + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"" + "URI == '/testindex/_search'" + "\"}}," + + " [{\"JsonJoltTransformerProvider\": { \"canned\": \"ADD_GZIP\" }}]" + + "]}," + "{ \"JsonJoltTransformerProvider\":" + " {\"script\": \n" + " { \"operation\": \"modify-overwrite-beta\", \"spec\": " diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/SampleContents.java similarity index 91% rename from TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/SampleContents.java index b27ad9a69..abad451c1 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/SampleContents.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java similarity index 96% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java index cc67b9866..47e93b320 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java @@ -1,7 +1,9 @@ -package org.opensearch.migrations.replay; +package org.opensearch.migrations.transform.replay; import java.util.Map; +import org.opensearch.migrations.transform.TransformationLoader; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Assertions; diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/log4j2.properties b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/log4j2.properties new file mode 100644 index 000000000..6adca47b5 --- /dev/null +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/log4j2.properties @@ -0,0 +1,18 @@ +status = WARN + +property.ownedPackagesLogLevel=${sys:migrationLogLevel:-INFO} + +appender.console.type = Console +appender.console.name = Console +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = Console + +# Allow customization of owned package logs +logger.rfs.name = org.opensearch.migrations.bulkload +logger.rfs.level = ${ownedPackagesLogLevel} +logger.migration.name = org.opensearch.migrations +logger.migration.level = ${ownedPackagesLogLevel} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json similarity index 100% rename from TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/sampleRequestMessage.json similarity index 100% rename from TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json rename to TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/resources/sampleRequestMessage.json