feature(fivetran_sdk): Add support for DataType Param (#45)

* added support for decimal params * incorporate comment
fivetran · Jun 18, 2024 · 5d15428 · 5d15428
1 parent c284b23
commit 5d15428
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 90 deletions.
diff --git a/common.proto b/common.proto
@@ -116,6 +116,13 @@ message DecimalParams {
   uint32 scale = 2;
 }
 
+message DataTypeParams {
+  oneof params {
+    DecimalParams decimal = 1;
+    int32 string_byte_length = 2;
+  }
+}
+
 enum RecordType {
   UPSERT = 0;
   UPDATE = 1;
@@ -153,5 +160,5 @@ message Column {
   string name = 1;
   DataType type = 2;
   bool primary_key = 3;
-  optional DecimalParams decimal = 4;
+  optional DataTypeParams params = 4;
 }
diff --git a/destination_sdk.proto b/destination_sdk.proto
@@ -89,7 +89,7 @@ message SchemaDiff {
     oneof change {
         Column add_column = 1;
         ChangeType change_column_type = 2;
-        PrimaryKey new_primary_key = 3;
+        PrimaryKey updated_primary_keys = 3;
     }
 }
 
@@ -100,6 +100,7 @@ message PrimaryKey {
 message ChangeType {
     string column_name = 1;
     DataType new_type = 2;
+    optional DataTypeParams params = 3;
 }
 
 message AlterTableResponse {

diff --git a/tools/destination-tester/README.md b/tools/destination-tester/README.md
@@ -45,98 +45,113 @@ Here is an example input file named `input_1.json`:
 
 ```json
 {
-    "create_table" : {
-        "transaction": {
-            "columns": {
-                "id": "INT",
-                "amount" : "DOUBLE",
-                "desc": "STRING"
-            },
-            "primary_key": ["id"]
+  "create_table" : {
+    "transaction": {
+      "columns": {
+        "id": "INT",
+        "amount" : "DOUBLE",
+        "desc": {"type": "STRING", "string_byte_length": 256}
+      },
+      "primary_key": ["id"]
+    },
+    "campaign": {
+      "columns": {
+        "name": "STRING",
+        "num": {"type": "DECIMAL", "precision": 6, "scale": 3}
+      },
+      "primary_key": []
+    }
+  },
+  "alter_table" : {
+    "transaction":
+    {
+      "add_column": {
+        "columns": {
+          "order_id": "INT",
+          "value": {
+            "type": "DECIMAL",
+            "precision": 6,
+            "scale": 3
+          }
         },
-        "campaign": {
-            "columns": {
-                "name": "STRING",
-                "num": {"type": "DECIMAL", "precision": 6, "scale": 3}
-            },
-            "primary_key": []
-        }
+        "primary_key": [
+          "order_id"
+        ]
+      },
+      "updated_primary_keys": [
+        "id",
+        "order_id"
+      ],
+      "change_column_type": {
+        "amount": "FLOAT"
+      }
+    }
+  },
+  "describe_table" : [
+    "transaction"
+  ],
+  "ops" : [
+    {
+      "upsert": {
+        "transaction": [
+          {"id":1, "amount": 100.45, "desc": null, "order_id": 1, "value": 10.10},
+          {"id":2, "amount": 150.33, "desc": "two", "order_id": 2, "value": 10.20}
+        ],
+        "campaign": [
+          {"_fivetran_id": "abc-123-xyz", "name": "Christmas", "num": 100.23},
+          {"_fivetran_id": "vbn-543-hjk", "name": "New Year", "num": 200.56}
+        ]
+      }
+    },
+    {
+      "truncate_before": [
+        "campaign"
+      ]
     },
-    "alter_table" : {
-        "transaction": {
-            "columns": {
-                "id": "INT",
-                "amount" : "FLOAT",
-                "desc": "STRING"
-            },
-            "primary_key": ["id"]
-        }
+    {
+      "update": {
+        "transaction": [
+          {"id":1, "amount": 200}
+        ]
+      }
     },
-    "describe_table" : [
+    {
+      "soft_truncate_before": [
         "transaction"
-    ],
-    "ops" : [
-        {
-            "upsert": {
-                "transaction": [
-                    {"id":1, "amount": 100.45, "desc": null},
-                    {"id":2, "amount": 150.33, "desc": "two"}
-                ],
-                "campaign": [
-                    {"_fivetran_id": "abc-123-xyz", "name": "Christmas", "num": 100.23},
-                    {"_fivetran_id": "vbn-543-hjk", "name": "New Year", "num": 200.56}
-                ]
-            }
-        },
-        {
-            "truncate_before": [
-                "campaign"
-            ]
-        },
-        {
-            "update": {
-                "transaction": [
-                    {"id":1, "amount": 200}
-                ]
-            }
-        },
-        {
-            "soft_truncate_before": [
-                "transaction"
-            ]
-        },
-        {
-            "upsert": {
-                "transaction": [
-                    {"id":10, "amount": 100, "desc": "thee"},
-                    {"id":20, "amount": 50, "desc": "mone"}
-                ],
-                "campaign": [
-                    {"_fivetran_id": "dfg-890-lkj", "name": "Christmas 2", "num": 400.32}
-                ]
-            }
-        },
-        {
-            "delete": {
-                "transaction": [
-                    {"id":3}
-                ],
-                "campaign": [
-                    {"_fivetran_id": "abc-123-xyz"}
-                ]
-            }
-        },
-        {
-            "soft_delete": {
-                "transaction": [
-                    {"id":4}
-                ],
-                "campaign": [
-                    {"_fivetran_id": "dfg-890-lkj"}
-                ]
-            }
-        }
-    ]
+      ]
+    },
+    {
+      "upsert": {
+        "transaction": [
+          {"id":10, "amount": 100, "desc": "thee", "order_id": 3, "value": 10.30},
+          {"id":20, "amount": 50, "desc": "mone", "order_id": 4, "value": 10.40}
+        ],
+        "campaign": [
+          {"_fivetran_id": "dfg-890-lkj", "name": "Christmas 2", "num": 400.32}
+        ]
+      }
+    },
+    {
+      "delete": {
+        "transaction": [
+          {"id":3}
+        ],
+        "campaign": [
+          {"_fivetran_id": "abc-123-xyz"}
+        ]
+      }
+    },
+    {
+      "soft_delete": {
+        "transaction": [
+          {"id":4}
+        ],
+        "campaign": [
+          {"_fivetran_id": "dfg-890-lkj"}
+        ]
+      }
+    }
+  ]
 }
 
 ```