Changes to interfaces (#32)

* Changes to interfaces * fix(partner-sdk): updated the go example with the new proto changes (#35) * fix(partner-sdk): updated the go example with new proto changes * fix(partner-sdk): re-formatted the code * feature(partner_sdk): fix python destination examples as per updated proto files (#34) * fix python destination example * fixed tester issue * feature(partner_sdk): fix java examples as per updated proto files (#33) * fix(partner-sdk): updated the python example with updated proto interface (#38) * fix(partner-sdk): updated the python example with updated proto interface * fix(python-example): refactored the code * fix(python-example): added field as part of kwargs * fix(partner_sdk): change proto version to v2 (#39) * feature(partner_sdk): add example for max value in destination capabilities (#41) * feature(fivetran_sdk): Add support for DataType Param (#45) * added support for decimal params * incorporate comment * add max value example for python destination (#48) * Separated proto updates files into v2 files (#47) * separated updated proto files into v2, updated build files * updated the README file * addressed requested changes * addressed requested changes * removed comment * removed naive_time from old proto * fixed python example * fixed go example * addressed comments * fix(fivetran-sdk): fixed the field number * refactor(Partner_sdk): Improvement in examples (#50) improvements in examples Co-authored-by: Satvik Patil <[email protected]> * addressed requested comments, fixed python destination example * removed note --------- Co-authored-by: SatvikPatil <[email protected]> Co-authored-by: Satvik Patil <[email protected]> --------- Co-authored-by: Manjunath Tapali <[email protected]> Co-authored-by: Abdul Salam <[email protected]> Co-authored-by: Niket Khandelwal <[email protected]> Co-authored-by: SatvikPatil <[email protected]> Co-authored-by: Satvik Patil <[email protected]>
fivetran · Jun 21, 2024 · 0a8c978 · 0a8c978
1 parent 4247c73
commit 0a8c978
Show file tree

Hide file tree

Showing 18 changed files with 1,030 additions and 345 deletions.
diff --git a/common.proto b/common.proto
@@ -53,7 +53,6 @@ message TestResponse {
   oneof response {
     bool success = 1;
     string failure = 2;
-    // potential future warning
   }
 }
 
@@ -79,14 +78,13 @@ enum DataType {
   DECIMAL = 5;
   FLOAT = 6;
   DOUBLE = 7;
-  NAIVE_TIME = 8;
-  NAIVE_DATE = 9;
-  NAIVE_DATETIME = 10;
-  UTC_DATETIME = 11;
-  BINARY = 12;
-  XML = 13;
-  STRING = 14;
-  JSON = 15;
+  NAIVE_DATE = 8;
+  NAIVE_DATETIME = 9;
+  UTC_DATETIME = 10;
+  BINARY = 11;
+  XML = 12;
+  STRING = 13;
+  JSON = 14;
 }
 
 message DecimalParams {
@@ -110,15 +108,14 @@ message ValueType {
     int64 long = 5;
     float float = 6;
     double double = 7;
-    google.protobuf.Timestamp naive_time = 8;
-    google.protobuf.Timestamp naive_date = 9;
-    google.protobuf.Timestamp naive_datetime = 10;
-    google.protobuf.Timestamp utc_datetime = 11;
-    string decimal = 12;
-    bytes binary = 13;
-    string string = 14;
-    string json = 15;
-    string xml = 16;
+    google.protobuf.Timestamp naive_date = 8;
+    google.protobuf.Timestamp naive_datetime = 9;
+    google.protobuf.Timestamp utc_datetime = 10;
+    string decimal = 11;
+    bytes binary = 12;
+    string string = 13;
+    string json = 14;
+    string xml = 15;
   }
 }
 

diff --git a/common_v2.proto b/common_v2.proto
@@ -0,0 +1,164 @@
+syntax = "proto3";
+option optimize_for = SPEED;
+option java_multiple_files = true;
+option go_package = "fivetran.com/fivetran_sdk_v2";
+package fivetran_sdk.v2;
+
+import "google/protobuf/timestamp.proto";
+
+message ConfigurationFormRequest {}
+
+message ConfigurationFormResponse {
+  bool schema_selection_supported = 1;
+  bool table_selection_supported = 2;
+  repeated FormField fields = 3;
+  repeated ConfigurationTest tests = 4;
+}
+
+message FormField {
+  oneof field {
+      Field single = 1;
+      FieldSet field_set = 2;
+  }
+}
+
+message Field {
+  string name = 1;
+  string label = 2;
+  bool required = 3;
+  optional string description = 4;
+  optional string default_value = 5;
+  optional string placeholder = 6;
+  oneof type {
+    TextField text_field = 7;
+    DropdownField dropdown_field = 8;
+    ToggleField toggle_field = 9;
+  }
+}
+
+message FieldSet {
+  repeated FormField fields = 1;
+  optional VisibilityCondition condition = 2;
+}
+
+message VisibilityCondition {
+  string field_name = 1;
+  oneof condition {
+    bool has_bool_value = 2;
+    string has_string_value = 3;
+    bool has_any_value = 4;
+  }
+}
+
+message DropdownField {
+  repeated string dropdown_field = 1;
+}
+
+message ToggleField {}
+
+enum TextField {
+  PlainText = 0;
+  Password = 1;
+  Hidden = 2;
+}
+
+message ConfigurationTest {
+  string name = 1;            // unique identifier for the test
+  string label = 2;           // A few words indicating what we are testing, e.g. 'Connecting to database'
+}
+
+message TestRequest {
+  string name = 1;
+  map<string, string> configuration = 2;
+}
+
+message TestResponse {
+  oneof response {
+    bool success = 1;
+    string failure = 2;
+  }
+}
+
+message SchemaList {
+  repeated Schema schemas = 1;
+}
+
+message TableList {
+  repeated Table tables = 1;
+}
+
+message Schema {
+  string name = 1;
+  repeated Table tables = 2;
+}
+
+enum DataType {
+  UNSPECIFIED = 0;
+  BOOLEAN = 1;
+  SHORT = 2;
+  INT = 3;
+  LONG = 4;
+  DECIMAL = 5;
+  FLOAT = 6;
+  DOUBLE = 7;
+  NAIVE_TIME = 8;
+  NAIVE_DATE = 9;
+  NAIVE_DATETIME = 10;
+  UTC_DATETIME = 11;
+  BINARY = 12;
+  XML = 13;
+  STRING = 14;
+  JSON = 15;
+}
+
+message DataTypeParams {
+  oneof params {
+    DecimalParams decimal = 1;
+    int32 string_byte_length = 2;
+  }
+}
+
+message DecimalParams {
+  uint32 precision = 1;
+  uint32 scale = 2;
+}
+
+enum RecordType {
+  UPSERT = 0;
+  UPDATE = 1;
+  DELETE = 2;
+  TRUNCATE = 3;
+}
+
+message ValueType {
+  oneof inner {
+    bool null = 1;
+    bool bool = 2;
+    int32 short = 3;
+    int32 int = 4;
+    int64 long = 5;
+    float float = 6;
+    double double = 7;
+    google.protobuf.Timestamp naive_time = 8;
+    google.protobuf.Timestamp naive_date = 9;
+    google.protobuf.Timestamp naive_datetime = 10;
+    google.protobuf.Timestamp utc_datetime = 11;
+    string decimal = 12;
+    bytes binary = 13;
+    string string = 14;
+    string json = 15;
+    string xml = 16;
+  }
+}
+
+message Table {
+  string name = 1;
+  repeated Column columns = 2;
+}
+
+message Column {
+  string name = 1;
+  DataType type = 2;
+  bool primary_key = 3;
+  optional DataTypeParams params = 4;
+}
diff --git a/connector_sdk.proto b/connector_sdk.proto
@@ -106,4 +106,4 @@ message Record {
 
 message Checkpoint {
     string state_json = 1;
-}
+}
diff --git a/connector_sdk_v2.proto b/connector_sdk_v2.proto
@@ -0,0 +1,91 @@
+syntax = "proto3";
+option optimize_for = SPEED;
+option java_multiple_files = true;
+option go_package = "fivetran.com/fivetran_sdk_v2";
+package fivetran_sdk.v2;
+
+import "common_v2.proto";
+
+// Fivetran (grpc client) <> SourceConnector (grpc server)
+service SourceConnector {
+    rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
+    rpc Test (TestRequest) returns (TestResponse) {}
+    rpc Schema (SchemaRequest) returns (SchemaResponse) {}
+    rpc Update (UpdateRequest) returns (stream UpdateResponse) {}
+}
+
+message SchemaRequest {
+    map<string, string> configuration = 1;
+}
+
+message SchemaResponse {
+    oneof response {
+        bool schema_response_not_supported = 1;
+        SchemaList with_schema = 2;
+        TableList without_schema = 3;
+    }
+    optional bool selection_not_supported  = 4;
+}
+
+message UpdateRequest {
+    map<string, string> configuration = 1;
+    optional Selection selection = 2;
+    optional string state_json = 3;
+}
+
+message Selection {
+    oneof selection {
+        TablesWithNoSchema without_schema = 1;
+        TablesWithSchema with_schema = 2;
+    }
+}
+
+message TablesWithNoSchema {
+    repeated TableSelection tables = 1;
+    bool include_new_tables = 2;
+}
+
+message TablesWithSchema {
+    repeated SchemaSelection schemas = 1;
+    bool include_new_schemas = 2;
+}
+
+message SchemaSelection {
+    bool included = 1;
+    string schema_name = 2;
+    repeated TableSelection tables = 3;
+    bool include_new_tables = 4;
+}
+
+message TableSelection {
+    bool included = 1;
+    string table_name = 2;
+    map<string, bool> columns = 3;
+    bool include_new_columns = 4;
+}
+
+message UpdateResponse {
+    oneof operation {
+        Record record = 1;
+        SchemaChange schema_change = 2;
+        Checkpoint checkpoint = 3;
+    }
+}
+
+message SchemaChange {
+    oneof change {
+        SchemaList with_schema = 1;
+        TableList without_schema = 2;
+    }
+}
+
+message Record {
+    optional string schema_name = 1;
+    string table_name = 2;
+    RecordType type = 3;
+    map<string, ValueType> data = 4;
+}
+
+message Checkpoint {
+    string state_json = 1;
+}
diff --git a/destination_sdk.proto b/destination_sdk.proto
@@ -10,6 +10,7 @@ import "common.proto";
 // Fivetran (grpc client) <> Destination (grpc server)
 service Destination {
     rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
+    rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {}
     rpc Test (TestRequest) returns (TestResponse) {}
     rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {}
     rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {}
@@ -18,6 +19,12 @@ service Destination {
     rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {}
 }
 
+message CapabilitiesRequest {}
+
+message CapabilitiesResponse {
+    BatchFileFormat batch_file_format = 1;
+}
+
 message DescribeTableRequest {
     map<string, string> configuration = 1;
     string schema_name = 2;
@@ -88,6 +95,7 @@ message WriteBatchRequest {
     repeated string delete_files = 7;
     oneof file_params {
         CsvFileParams csv = 8;
+        ParquetFileParams parquet = 9;
     }
 }
 
@@ -98,11 +106,20 @@ message CsvFileParams {
     string unmodified_string = 4;
 }
 
+message ParquetFileParams {
+    Encryption encryption = 1;
+}
+
 enum Encryption {
     NONE = 0;
     AES = 1;
 }
 
+enum BatchFileFormat {
+    CSV = 0;
+    PARQUET = 1;
+}
+
 enum Compression {
     OFF = 0;
     ZSTD = 1;
@@ -114,4 +131,4 @@ message WriteBatchResponse {
         bool success = 1;
         string failure = 2;
     }
-}
+}