-
Notifications
You must be signed in to change notification settings - Fork 98
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use FakeStreamingClientHandler with real snowflake implementation
- Loading branch information
1 parent
f74b2b5
commit 41bf7a4
Showing
7 changed files
with
183 additions
and
136 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 34 additions & 34 deletions
68
src/test/java/com/snowflake/kafka/connector/ProduceConsumeMessageIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,45 +1,45 @@ | ||
| package com.snowflake.kafka.connector; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.awaitility.Awaitility.await; | ||
|
|
||
| import com.snowflake.kafka.connector.fake.SnowflakeFakeSinkTask; | ||
| import java.util.List; | ||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
| import org.junit.jupiter.api.AfterEach; | ||
| import org.junit.jupiter.api.BeforeEach; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.awaitility.Awaitility.await; | ||
|
|
||
| public class ProduceConsumeMessageIT extends ConnectClusterBaseIT { | ||
|
|
||
| private static final String TEST_TOPIC = "kafka-int-test"; | ||
| private static final String TEST_CONNECTOR_NAME = "test-connector"; | ||
|
|
||
| @BeforeEach | ||
| void createConnector() { | ||
| connectCluster.kafka().createTopic(TEST_TOPIC); | ||
| connectCluster.configureConnector(TEST_CONNECTOR_NAME, defaultProperties(TEST_TOPIC, TEST_CONNECTOR_NAME)); | ||
| await().timeout(CONNECTOR_MAX_STARTUP_TIME).until(isConnectorRunning(TEST_CONNECTOR_NAME)); | ||
| } | ||
|
|
||
| @AfterEach | ||
| void deleteConnector() { | ||
| connectCluster.deleteConnector(TEST_CONNECTOR_NAME); | ||
| connectCluster.kafka().deleteTopic(TEST_TOPIC); | ||
| } | ||
|
|
||
| @Test | ||
| public void connectorShouldConsumeMessagesFromTopic() { | ||
| connectCluster.kafka().produce(TEST_TOPIC, "test1"); | ||
| connectCluster.kafka().produce(TEST_TOPIC, "test2"); | ||
|
|
||
| await() | ||
| .untilAsserted( | ||
| () -> { | ||
| List<SinkRecord> records = SnowflakeFakeSinkTask.getRecords(); | ||
| assertThat(records).hasSize(2); | ||
| assertThat(records.stream().map(SinkRecord::value)).containsExactly("test1", "test2"); | ||
| }); | ||
| } | ||
| private static final String TEST_TOPIC = "kafka-int-test"; | ||
| private static final String TEST_CONNECTOR_NAME = "test-connector"; | ||
|
|
||
| @BeforeEach | ||
| void createConnector() { | ||
| connectCluster.kafka().createTopic(TEST_TOPIC); | ||
| connectCluster.configureConnector( | ||
| TEST_CONNECTOR_NAME, defaultProperties(TEST_TOPIC, TEST_CONNECTOR_NAME)); | ||
| waitForConnectorRunning(TEST_CONNECTOR_NAME); | ||
| } | ||
|
|
||
| @AfterEach | ||
| void deleteConnector() { | ||
| connectCluster.deleteConnector(TEST_CONNECTOR_NAME); | ||
| connectCluster.kafka().deleteTopic(TEST_TOPIC); | ||
| } | ||
|
|
||
| @Test | ||
| public void connectorShouldConsumeMessagesFromTopic() { | ||
| connectCluster.kafka().produce(TEST_TOPIC, "test1"); | ||
| connectCluster.kafka().produce(TEST_TOPIC, "test2"); | ||
|
|
||
| await() | ||
| .untilAsserted( | ||
| () -> { | ||
| List<SinkRecord> records = SnowflakeFakeSinkTask.getRecords(); | ||
| assertThat(records).hasSize(2); | ||
| assertThat(records.stream().map(SinkRecord::value)).containsExactly("test1", "test2"); | ||
| }); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,74 +1,84 @@ | ||
| package com.snowflake.kafka.connector; | ||
|
|
||
| import com.snowflake.kafka.connector.fake.SnowflakeFakeSinkTask; | ||
| import org.apache.kafka.connect.json.JsonConverter; | ||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
| import org.junit.jupiter.api.AfterAll; | ||
| import org.junit.jupiter.api.BeforeAll; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.function.UnaryOperator; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.BUFFER_FLUSH_TIME_SEC; | ||
| import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.INGESTION_METHOD_OPT; | ||
| import static com.snowflake.kafka.connector.SnowflakeSinkConnectorConfig.NAME; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.TRANSFORMS_CONFIG; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; | ||
| import static org.apache.kafka.connect.sink.SinkConnector.TOPICS_CONFIG; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.awaitility.Awaitility.await; | ||
|
|
||
| public class SmtIT extends ConnectClusterBaseIT { | ||
|
|
||
| private static final String SMT_TOPIC = "smtTopic"; | ||
| private static final String SMT_CONNECTOR = "smtConnector"; | ||
|
|
||
| @BeforeAll | ||
| public void createConnector() { | ||
| connectCluster.kafka().createTopic(SMT_TOPIC); | ||
| connectCluster.configureConnector(SMT_CONNECTOR, smtProperties()); | ||
| await().timeout(CONNECTOR_MAX_STARTUP_TIME).until(isConnectorRunning(SMT_CONNECTOR)); | ||
| } | ||
|
|
||
| @AfterAll | ||
| public void deleteConnector() { | ||
| connectCluster.deleteConnector(SMT_CONNECTOR); | ||
| connectCluster.kafka().deleteTopic(SMT_TOPIC); | ||
| } | ||
|
|
||
| private Map<String, String> smtProperties() { | ||
| Map<String, String> config = defaultProperties(SMT_TOPIC, SMT_CONNECTOR); | ||
|
|
||
| config.put(NAME, SMT_CONNECTOR); | ||
| config.put(TOPICS_CONFIG, SMT_TOPIC); | ||
|
|
||
| config.put(VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); | ||
| config.put("value.converter.schemas.enable", "false"); | ||
|
|
||
| config.put(TRANSFORMS_CONFIG, "extractField"); | ||
| config.put("transforms.extractField.type", "org.apache.kafka.connect.transforms.ExtractField$Value"); | ||
| config.put("transforms.extractField.field", "message"); | ||
|
|
||
| return config; | ||
| } | ||
|
|
||
| @Test | ||
| void testIfSmtRetuningNullsIngestDataCorrectly() { | ||
| Stream.iterate(0, UnaryOperator.identity()) | ||
| .limit(10) | ||
| .flatMap(v -> Stream.of("{}", "{\"message\":\"value\"}")) | ||
| .forEach(message -> connectCluster.kafka().produce(SMT_TOPIC, message)); | ||
|
|
||
| await() | ||
| .untilAsserted( | ||
| () -> { | ||
| List<SinkRecord> records = SnowflakeFakeSinkTask.getRecords(); | ||
| assertThat(records) | ||
| .hasSize(20) | ||
| .allMatch(r -> r.originalTopic().equals(SMT_TOPIC)); | ||
| }); | ||
| import com.snowflake.kafka.connector.internal.TestUtils; | ||
| import com.snowflake.kafka.connector.internal.streaming.FakeStreamingClientHandler; | ||
| import com.snowflake.kafka.connector.internal.streaming.IngestionMethodConfig; | ||
| import com.snowflake.kafka.connector.internal.streaming.StreamingClientProvider; | ||
| import java.time.Duration; | ||
| import java.util.Map; | ||
| import java.util.function.UnaryOperator; | ||
| import java.util.stream.Stream; | ||
| import org.apache.kafka.connect.json.JsonConverter; | ||
| import org.apache.kafka.connect.storage.StringConverter; | ||
| import org.junit.jupiter.api.AfterAll; | ||
| import org.junit.jupiter.api.BeforeAll; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| } | ||
| public class SmtIT extends ConnectClusterBaseIT { | ||
|
|
||
| private static final String SMT_TOPIC = "SMT_TOPIC"; | ||
| private static final String SMT_CONNECTOR = "SMT_CONNECTOR"; | ||
| private static final FakeStreamingClientHandler fakeStreamingClientHandler = | ||
| new FakeStreamingClientHandler(); | ||
|
|
||
| @BeforeAll | ||
| public void createConnector() { | ||
| StreamingClientProvider.overrideStreamingClientHandler(fakeStreamingClientHandler); | ||
| connectCluster.kafka().createTopic(SMT_TOPIC); | ||
| connectCluster.configureConnector(SMT_CONNECTOR, smtProperties()); | ||
| waitForConnectorRunning(SMT_CONNECTOR); | ||
| } | ||
|
|
||
| @AfterAll | ||
| public void deleteConnector() { | ||
| connectCluster.deleteConnector(SMT_CONNECTOR); | ||
| connectCluster.kafka().deleteTopic(SMT_TOPIC); | ||
| } | ||
|
|
||
| private Map<String, String> smtProperties() { | ||
| Map<String, String> config = TestUtils.getConf(); | ||
|
|
||
| config.put(CONNECTOR_CLASS_CONFIG, SnowflakeSinkConnector.class.getName()); | ||
| config.put(NAME, SMT_CONNECTOR); | ||
| config.put(TOPICS_CONFIG, SMT_TOPIC); | ||
| config.put(INGESTION_METHOD_OPT, IngestionMethodConfig.SNOWPIPE_STREAMING.toString()); | ||
| config.put(Utils.SF_ROLE, "testrole_kafka"); | ||
| config.put(BUFFER_FLUSH_TIME_SEC, "1"); | ||
|
|
||
| config.put(TASKS_MAX_CONFIG, TASK_NUMBER.toString()); | ||
| config.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); | ||
| config.put(VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); | ||
| config.put("value.converter.schemas.enable", "false"); | ||
|
|
||
| config.put(TRANSFORMS_CONFIG, "extractField"); | ||
| config.put( | ||
| "transforms.extractField.type", "org.apache.kafka.connect.transforms.ExtractField$Value"); | ||
| config.put("transforms.extractField.field", "message"); | ||
|
|
||
| return config; | ||
| } | ||
|
|
||
| @Test | ||
| void testIfSmtRetuningNullsIngestDataCorrectly() { | ||
| Stream.iterate(0, UnaryOperator.identity()) | ||
| .limit(10) | ||
| .flatMap(v -> Stream.of("{}", "{\"message\":\"value\"}")) | ||
| .forEach(message -> connectCluster.kafka().produce(SMT_TOPIC, message)); | ||
|
|
||
| await() | ||
| .timeout(Duration.ofSeconds(60)) | ||
| .untilAsserted(() -> assertThat(fakeStreamingClientHandler.ingestedRows()).hasSize(20)); | ||
| } | ||
| } |
Oops, something went wrong.