diff --git a/common.proto b/common.proto index 8daa479..1dc2507 100644 --- a/common.proto +++ b/common.proto @@ -116,6 +116,13 @@ message DecimalParams { uint32 scale = 2; } +message DataTypeParams { + oneof params { + DecimalParams decimal = 1; + int32 string_byte_length = 2; + } +} + enum RecordType { UPSERT = 0; UPDATE = 1; @@ -153,5 +160,5 @@ message Column { string name = 1; DataType type = 2; bool primary_key = 3; - optional DecimalParams decimal = 4; + optional DataTypeParams params = 4; } diff --git a/destination_sdk.proto b/destination_sdk.proto index c5b6b8d..66fb1a7 100644 --- a/destination_sdk.proto +++ b/destination_sdk.proto @@ -89,7 +89,7 @@ message SchemaDiff { oneof change { Column add_column = 1; ChangeType change_column_type = 2; - PrimaryKey new_primary_key = 3; + PrimaryKey updated_primary_keys = 3; } } @@ -100,6 +100,7 @@ message PrimaryKey { message ChangeType { string column_name = 1; DataType new_type = 2; + optional DataTypeParams params = 3; } message AlterTableResponse { diff --git a/tools/destination-tester/README.md b/tools/destination-tester/README.md index 62d5302..dc0c33f 100644 --- a/tools/destination-tester/README.md +++ b/tools/destination-tester/README.md @@ -45,98 +45,113 @@ Here is an example input file named `input_1.json`: ```json { - "create_table" : { - "transaction": { - "columns": { - "id": "INT", - "amount" : "DOUBLE", - "desc": "STRING" - }, - "primary_key": ["id"] + "create_table" : { + "transaction": { + "columns": { + "id": "INT", + "amount" : "DOUBLE", + "desc": {"type": "STRING", "string_byte_length": 256} + }, + "primary_key": ["id"] + }, + "campaign": { + "columns": { + "name": "STRING", + "num": {"type": "DECIMAL", "precision": 6, "scale": 3} + }, + "primary_key": [] + } + }, + "alter_table" : { + "transaction": + { + "add_column": { + "columns": { + "order_id": "INT", + "value": { + "type": "DECIMAL", + "precision": 6, + "scale": 3 + } }, - "campaign": { - "columns": { - "name": "STRING", - "num": {"type": "DECIMAL", "precision": 6, "scale": 3} - }, - "primary_key": [] - } + "primary_key": [ + "order_id" + ] + }, + "updated_primary_keys": [ + "id", + "order_id" + ], + "change_column_type": { + "amount": "FLOAT" + } + } + }, + "describe_table" : [ + "transaction" + ], + "ops" : [ + { + "upsert": { + "transaction": [ + {"id":1, "amount": 100.45, "desc": null, "order_id": 1, "value": 10.10}, + {"id":2, "amount": 150.33, "desc": "two", "order_id": 2, "value": 10.20} + ], + "campaign": [ + {"_fivetran_id": "abc-123-xyz", "name": "Christmas", "num": 100.23}, + {"_fivetran_id": "vbn-543-hjk", "name": "New Year", "num": 200.56} + ] + } + }, + { + "truncate_before": [ + "campaign" + ] }, - "alter_table" : { - "transaction": { - "columns": { - "id": "INT", - "amount" : "FLOAT", - "desc": "STRING" - }, - "primary_key": ["id"] - } + { + "update": { + "transaction": [ + {"id":1, "amount": 200} + ] + } }, - "describe_table" : [ + { + "soft_truncate_before": [ "transaction" - ], - "ops" : [ - { - "upsert": { - "transaction": [ - {"id":1, "amount": 100.45, "desc": null}, - {"id":2, "amount": 150.33, "desc": "two"} - ], - "campaign": [ - {"_fivetran_id": "abc-123-xyz", "name": "Christmas", "num": 100.23}, - {"_fivetran_id": "vbn-543-hjk", "name": "New Year", "num": 200.56} - ] - } - }, - { - "truncate_before": [ - "campaign" - ] - }, - { - "update": { - "transaction": [ - {"id":1, "amount": 200} - ] - } - }, - { - "soft_truncate_before": [ - "transaction" - ] - }, - { - "upsert": { - "transaction": [ - {"id":10, "amount": 100, "desc": "thee"}, - {"id":20, "amount": 50, "desc": "mone"} - ], - "campaign": [ - {"_fivetran_id": "dfg-890-lkj", "name": "Christmas 2", "num": 400.32} - ] - } - }, - { - "delete": { - "transaction": [ - {"id":3} - ], - "campaign": [ - {"_fivetran_id": "abc-123-xyz"} - ] - } - }, - { - "soft_delete": { - "transaction": [ - {"id":4} - ], - "campaign": [ - {"_fivetran_id": "dfg-890-lkj"} - ] - } - } - ] + ] + }, + { + "upsert": { + "transaction": [ + {"id":10, "amount": 100, "desc": "thee", "order_id": 3, "value": 10.30}, + {"id":20, "amount": 50, "desc": "mone", "order_id": 4, "value": 10.40} + ], + "campaign": [ + {"_fivetran_id": "dfg-890-lkj", "name": "Christmas 2", "num": 400.32} + ] + } + }, + { + "delete": { + "transaction": [ + {"id":3} + ], + "campaign": [ + {"_fivetran_id": "abc-123-xyz"} + ] + } + }, + { + "soft_delete": { + "transaction": [ + {"id":4} + ], + "campaign": [ + {"_fivetran_id": "dfg-890-lkj"} + ] + } + } + ] } ```