From 14b031ca8e18029cd4cf917fef4c645b89c35396 Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Fri, 2 Aug 2024 17:09:51 +0530 Subject: [PATCH 01/10] init --- core/pom.xml | 13 +- .../Remorph-Reconciliation.lvdash.json | 2905 ++++++++++++++++- .../reconcile/dashboards/dashboard.yml | 0 .../dashboards/failed_table_list.sql | 26 + .../dashboards/recon_details_pivot.sql | 22 + .../dashboards/recon_details_schema.sql | 31 + .../reconcile/dashboards/recon_main.sql | 31 + .../dashboards/success_table_list.sql | 20 + 8 files changed, 3044 insertions(+), 4 deletions(-) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql diff --git a/core/pom.xml b/core/pom.xml index 825038335..7efa1c8c7 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -16,7 +16,7 @@ 1.8 1.8 - 4.11.0 + 5.11.0 UTF-8 2.0.9 4.13.1 @@ -47,7 +47,7 @@ com.databricks databricks-sdk-java - 0.21.0 + 0.24.1 org.apache.spark @@ -78,7 +78,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.3.0-SNAP3 + 3.3.0-SNAP4 test @@ -230,4 +230,11 @@ + + + maven_central + Maven Central + https://repo.maven.apache.org/maven2/ + + diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json b/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json index c088b1431..80419b4f2 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json @@ -1 +1,2904 @@ -{"datasets":[{"name":"655d5caf","displayName":"recon_main","query":"select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.exception_message as exception,\nmetrics.recon_metrics.row_comparison.missing_in_source as missing_in_source,\nmetrics.recon_metrics.row_comparison.missing_in_target as missing_in_target,\nmetrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch,\nmetrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch,\nmetrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns,\nmetrics.recon_metrics.schema_comparison as schema_comparison,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name","parameters":[{"displayName":"catalog","keyword":"catalog","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"remorph"}]}}},{"displayName":"schema","keyword":"schema","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"reconcile"}]}}}]},{"name":"78e296d2","displayName":"recon_details_schema","query":"with tmp as (select \nrecon_table_id,\ninserted_ts,\nexplode(data) as schema_data\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type='schema'\n) \nselect \nmain.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nschema_data['source_column'] as source_column,\nschema_data['source_datatype'] as source_datatype,\nschema_data['databricks_column'] as databricks_column,\nschema_data['databricks_datatype'] as databricks_datatype,\nschema_data['is_valid'] as is_valid\nfrom \nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \ntmp\non main.recon_table_id = tmp.recon_table_id\norder by tmp.inserted_ts desc, main.recon_id, main.target_table\n","parameters":[{"displayName":"catalog","keyword":"catalog","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"remorph"}]}}},{"displayName":"schema","keyword":"schema","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"reconcile"}]}}}]},{"name":"d4050b68","displayName":"recon_details_pivot","query":"with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, \nrow_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type != 'schema')\nselect main.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nrecon_type, key, value, rn\nfrom tmp\ninner join\nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\non main.recon_table_id = tmp.recon_table_id\nlateral view explode(data) exploded_data AS key, value\n","parameters":[{"displayName":"catalog","keyword":"catalog","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"remorph"}]}}},{"displayName":"schema","keyword":"schema","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"reconcile"}]}}}]},{"name":"c132f0ca","displayName":"failed_table","query":"select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = false\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name","parameters":[{"displayName":"catalog","keyword":"catalog","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"remorph"}]}}},{"displayName":"schema","keyword":"schema","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"reconcile"}]}}}]},{"name":"7637e124","displayName":"success_table","query":"select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = true\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name","parameters":[{"displayName":"catalog","keyword":"catalog","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"remorph"}]}}},{"displayName":"schema","keyword":"schema","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"reconcile"}]}}}]}],"pages":[{"name":"569063d3","displayName":"New Page","layout":[{"widget":{"name":"b4a9c7fd","queries":[{"name":"main_query","query":{"datasetName":"655d5caf","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"source_type","expression":"`source_type`"},{"name":"report_type","expression":"`report_type`"},{"name":"source_table","expression":"`source_table`"},{"name":"target_table","expression":"`target_table`"},{"name":"status","expression":"`status`"},{"name":"exception","expression":"`exception`"},{"name":"missing_in_source","expression":"`missing_in_source`"},{"name":"missing_in_target","expression":"`missing_in_target`"},{"name":"absolute_mismatch","expression":"`absolute_mismatch`"},{"name":"threshold_mismatch","expression":"`threshold_mismatch`"},{"name":"mismatch_columns","expression":"`mismatch_columns`"},{"name":"schema_comparison","expression":"`schema_comparison`"},{"name":"executed_by","expression":"`executed_by`"},{"name":"start_ts","expression":"`start_ts`"},{"name":"end_ts","expression":"`end_ts`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"recon_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"recon_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"status","fn":"=","literal":"true"},"value":{"foregroundColor":"#3BD973"}},{"if":{"column":"status","fn":"=","literal":"false"},"value":{"foregroundColor":"#E92828"}}]},"displayName":"recon_id"},{"fieldName":"source_type","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"source_type","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"source_type"},{"fieldName":"report_type","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"report_type","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"report_type"},{"fieldName":"source_table","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":5,"title":"source_table","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"source_table"},{"fieldName":"target_table","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":8,"title":"target_table","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"target_table"},{"fieldName":"status","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"boolean","displayAs":"boolean","visible":true,"order":9,"title":"status","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"status"},{"fieldName":"exception","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":10,"title":"exception","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"exception"},{"fieldName":"missing_in_source","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":11,"title":"missing_in_source","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"missing_in_source"},{"fieldName":"missing_in_target","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":12,"title":"missing_in_target","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"missing_in_target"},{"fieldName":"absolute_mismatch","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":13,"title":"absolute_mismatch","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"absolute_mismatch"},{"fieldName":"threshold_mismatch","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":14,"title":"threshold_mismatch","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"threshold_mismatch"},{"fieldName":"mismatch_columns","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":15,"title":"mismatch_columns","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"mismatch_columns"},{"fieldName":"schema_comparison","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"boolean","displayAs":"boolean","visible":true,"order":16,"title":"schema_comparison","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"schema_comparison"},{"fieldName":"executed_by","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":19,"title":"executed_by","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"executed_by"},{"fieldName":"start_ts","dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":20,"title":"start_ts","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"start_ts"},{"fieldName":"end_ts","dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":21,"title":"end_ts","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"end_ts"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_catalog","type":"string","displayAs":"string","order":3,"title":"source_catalog","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_schema","type":"string","displayAs":"string","order":4,"title":"source_schema","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_catalog","type":"string","displayAs":"string","order":6,"title":"target_catalog","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_schema","type":"string","displayAs":"string","order":7,"title":"target_schema","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_table_name","type":"string","displayAs":"string","order":17,"title":"source_table_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_table_name","type":"string","displayAs":"string","order":18,"title":"target_table_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":100,"paginationSize":"default","condensed":true,"withRowNumber":true,"frame":{"showDescription":true,"description":"Summary Table","showTitle":false}}},"position":{"x":0,"y":10,"width":6,"height":7}},{"widget":{"name":"32ea02c2","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id","query":{"datasetName":"655d5caf","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"recon_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id","query":{"datasetName":"78e296d2","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"recon_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id","query":{"datasetName":"d4050b68","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"recon_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"recon_id","displayName":"recon_id","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id"},{"fieldName":"recon_id","displayName":"recon_id","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id"},{"fieldName":"recon_id","displayName":"recon_id","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]},"frame":{"showTitle":true,"showDescription":false,"title":"Recon Id","description":""}}},"position":{"x":0,"y":4,"width":2,"height":2}},{"widget":{"name":"1be6ab4a","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table","query":{"datasetName":"655d5caf","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"target_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table","query":{"datasetName":"78e296d2","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"target_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table","query":{"datasetName":"d4050b68","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"target_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"target_table","displayName":"target_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table"},{"fieldName":"target_table","displayName":"target_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table"},{"fieldName":"target_table","displayName":"target_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]},"frame":{"showTitle":true,"showDescription":false,"title":"Traget Table Name","description":""}}},"position":{"x":2,"y":4,"width":2,"height":2}},{"widget":{"name":"3ec3ec2d","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table","query":{"datasetName":"655d5caf","fields":[{"name":"source_table","expression":"`source_table`"},{"name":"source_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table","query":{"datasetName":"78e296d2","fields":[{"name":"source_table","expression":"`source_table`"},{"name":"source_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table","query":{"datasetName":"d4050b68","fields":[{"name":"source_table","expression":"`source_table`"},{"name":"source_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"source_table","displayName":"source_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table"},{"fieldName":"source_table","displayName":"source_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table"},{"fieldName":"source_table","displayName":"source_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]},"frame":{"showTitle":true,"showDescription":false,"title":"Source Table Name","description":""}}},"position":{"x":4,"y":4,"width":2,"height":2}},{"widget":{"name":"8cf05b2f","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type","query":{"datasetName":"655d5caf","fields":[{"name":"source_type","expression":"`source_type`"},{"name":"source_type_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"source_type","displayName":"source_type","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Source Type","description":"Applied only on summary table"}}},"position":{"x":0,"y":6,"width":2,"height":2}},{"widget":{"name":"21add8d7","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type","query":{"datasetName":"d4050b68","fields":[{"name":"recon_type","expression":"`recon_type`"},{"name":"recon_type_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"recon_type","displayName":"recon_type","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Category","description":"Applied only on details table"}}},"position":{"x":3,"y":27,"width":3,"height":1}},{"widget":{"name":"49e40526","queries":[{"name":"main_query","query":{"datasetName":"655d5caf","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"hourly(start_ts)","expression":"DATE_TRUNC(\"HOUR\", `start_ts`)"},{"name":"absolute_mismatch","expression":"`absolute_mismatch`"}],"disaggregated":true}}],"spec":{"version":3,"widgetType":"area","encodings":{"x":{"fieldName":"hourly(start_ts)","scale":{"type":"temporal"},"displayName":"start_ts"},"y":{"fieldName":"absolute_mismatch","scale":{"type":"quantitative"},"displayName":"absolute_mismatch"},"color":{"fieldName":"target_table","scale":{"type":"categorical"},"displayName":"target_table"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Mismatched Records","showDescription":false}}},"position":{"x":0,"y":47,"width":3,"height":6}},{"widget":{"name":"2cbc9f42","queries":[{"name":"main_query","query":{"datasetName":"655d5caf","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"hourly(start_ts)","expression":"DATE_TRUNC(\"HOUR\", `start_ts`)"},{"name":"missing_in_target","expression":"`missing_in_target`"}],"disaggregated":true}}],"spec":{"version":3,"widgetType":"area","encodings":{"x":{"fieldName":"hourly(start_ts)","scale":{"type":"temporal"},"displayName":"start_ts"},"y":{"fieldName":"missing_in_target","scale":{"type":"quantitative"},"displayName":"missing_in_target"},"color":{"fieldName":"target_table","scale":{"type":"categorical"},"displayName":"target_table"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Missing in Source"}}},"position":{"x":3,"y":53,"width":3,"height":6}},{"widget":{"name":"5ae38078","queries":[{"name":"main_query","query":{"datasetName":"655d5caf","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"hourly(start_ts)","expression":"DATE_TRUNC(\"HOUR\", `start_ts`)"},{"name":"missing_in_source","expression":"`missing_in_source`"}],"disaggregated":true}}],"spec":{"version":3,"widgetType":"area","encodings":{"x":{"fieldName":"hourly(start_ts)","scale":{"type":"temporal"},"displayName":"start_ts"},"y":{"fieldName":"missing_in_source","scale":{"type":"quantitative"},"displayName":"missing_in_source"},"color":{"fieldName":"target_table","scale":{"type":"categorical"},"displayName":"target_table"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Missing in Databricks"}}},"position":{"x":0,"y":53,"width":3,"height":6}},{"widget":{"name":"2ea234ee","queries":[{"name":"main_query","query":{"datasetName":"655d5caf","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"hourly(start_ts)","expression":"DATE_TRUNC(\"HOUR\", `start_ts`)"},{"name":"threshold_mismatch","expression":"`threshold_mismatch`"}],"disaggregated":true}}],"spec":{"version":3,"widgetType":"area","encodings":{"x":{"fieldName":"hourly(start_ts)","scale":{"type":"temporal"},"displayName":"start_ts"},"y":{"fieldName":"threshold_mismatch","scale":{"type":"quantitative"},"displayName":"threshold_mismatch"},"color":{"fieldName":"target_table","scale":{"type":"categorical"},"displayName":"target_table"}},"frame":{"showTitle":true,"title":"Threshold Mismatches"}}},"position":{"x":3,"y":47,"width":3,"height":6}},{"widget":{"name":"403919aa","queries":[{"name":"main_query","query":{"datasetName":"78e296d2","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"source_table","expression":"`source_table`"},{"name":"target_table","expression":"`target_table`"},{"name":"source_column","expression":"`source_column`"},{"name":"source_datatype","expression":"`source_datatype`"},{"name":"databricks_column","expression":"`databricks_column`"},{"name":"databricks_datatype","expression":"`databricks_datatype`"},{"name":"is_valid","expression":"`is_valid`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"recon_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"recon_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"is_valid","fn":"=","literal":"false"},"value":{"foregroundColor":"#E92828"}},{"if":{"column":"is_valid","fn":"=","literal":"true"},"value":{"foregroundColor":"#3BD973"}}]},"displayName":"recon_id"},{"fieldName":"source_table","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":4,"title":"source_table","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"source_table"},{"fieldName":"target_table","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":8,"title":"target_table","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"target_table"},{"fieldName":"source_column","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":9,"title":"source_column","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"source_column"},{"fieldName":"source_datatype","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":10,"title":"source_datatype","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"source_datatype"},{"fieldName":"databricks_column","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":11,"title":"databricks_column","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"databricks_column"},{"fieldName":"databricks_datatype","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":12,"title":"databricks_datatype","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"databricks_datatype"},{"fieldName":"is_valid","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":13,"title":"is_valid","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"is_valid"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_table_name","type":"string","displayAs":"string","order":1,"title":"source_table_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_catalog","type":"string","displayAs":"string","order":2,"title":"source_catalog","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"source_schema","type":"string","displayAs":"string","order":3,"title":"source_schema","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_catalog","type":"string","displayAs":"string","order":5,"title":"target_catalog","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_schema","type":"string","displayAs":"string","order":6,"title":"target_schema","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"target_table_name","type":"string","displayAs":"string","order":7,"title":"target_table_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":true,"withRowNumber":false,"frame":{"showTitle":true,"title":"Schema Details"}}},"position":{"x":0,"y":19,"width":6,"height":6}},{"widget":{"name":"b154de70","queries":[{"name":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog","query":{"datasetName":"655d5caf","parameters":[{"name":"catalog","keyword":"catalog"}],"disaggregated":false}},{"name":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog","query":{"datasetName":"78e296d2","parameters":[{"name":"catalog","keyword":"catalog"}],"disaggregated":false}},{"name":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog","query":{"datasetName":"d4050b68","parameters":[{"name":"catalog","keyword":"catalog"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"parameterName":"catalog","queryName":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog"},{"parameterName":"catalog","queryName":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog"},{"parameterName":"catalog","queryName":"parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Catalog","description":"’remorph' is the default catalog for the metrics table; specify a different catalog name if the metrics are stored elsewhere."}}},"position":{"x":0,"y":2,"width":3,"height":2}},{"widget":{"name":"05101c82","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type","query":{"datasetName":"655d5caf","fields":[{"name":"report_type","expression":"`report_type`"},{"name":"report_type_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"report_type","displayName":"report_type","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Report","description":"Applied only on summary table"}}},"position":{"x":2,"y":6,"width":2,"height":2}},{"widget":{"name":"081790c2","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts","query":{"datasetName":"655d5caf","fields":[{"name":"start_ts","expression":"`start_ts`"},{"name":"start_ts_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-date-range-picker","encodings":{"fields":[{"fieldName":"start_ts","displayName":"start_ts","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts"}]},"frame":{"showTitle":true,"showDescription":true,"title":"started at","description":"Applied on all tables"}}},"position":{"x":0,"y":8,"width":6,"height":2}},{"widget":{"name":"ffcb9c29","textbox_spec":"# Main Reconciliation Table\n### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records."},"position":{"x":0,"y":0,"width":6,"height":2}},{"widget":{"name":"acd1adcc","textbox_spec":"# Drill Down\n### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues."},"position":{"x":0,"y":25,"width":6,"height":2}},{"widget":{"name":"680c9fe9","textbox_spec":"# Schema Comparison Details\n### This table provides a detailed view of schema mismatches."},"position":{"x":0,"y":17,"width":6,"height":2}},{"widget":{"name":"b1a2db69","textbox_spec":"# Visualization of Missing and Mismatched Records"},"position":{"x":0,"y":45,"width":6,"height":2}},{"widget":{"name":"2199286c","queries":[{"name":"main_query","query":{"datasetName":"d4050b68","fields":[{"name":"value","expression":"`value`"},{"name":"key","expression":"`key`"},{"name":"recon_id","expression":"`recon_id`"},{"name":"source_table","expression":"`source_table`"},{"name":"target_table","expression":"`target_table`"},{"name":"recon_type","expression":"`recon_type`"},{"name":"rn","expression":"`rn`"}],"disaggregated":false,"orders":[{"direction":"ASC","expression":"`recon_id`"},{"direction":"ASC","expression":"`source_table`"},{"direction":"ASC","expression":"`target_table`"},{"direction":"ASC","expression":"`recon_type`"},{"direction":"ASC","expression":"`rn`"},{"direction":"ASC","expression":"`key`"}]}}],"spec":{"version":3,"widgetType":"pivot","encodings":{"rows":[{"fieldName":"recon_id","displayName":"recon_id"},{"fieldName":"source_table","displayName":"source_table"},{"fieldName":"target_table","displayName":"target_table"},{"fieldName":"recon_type","displayName":"recon_type"},{"fieldName":"rn","displayName":"rn"}],"columns":[{"fieldName":"key","displayName":"key"}],"cell":{"fieldName":"value","cellType":"text","displayName":"value"}}}},"position":{"x":0,"y":29,"width":6,"height":6}},{"widget":{"name":"9c0a2a5e","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table","query":{"datasetName":"d4050b68","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"target_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"target_table","displayName":"target_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]},"frame":{"showDescription":true,"showTitle":true,"title":"Target Table Name","description":"Applied only on details table"}}},"position":{"x":0,"y":28,"width":3,"height":1}},{"widget":{"name":"1e536216","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table","query":{"datasetName":"d4050b68","fields":[{"name":"source_table","expression":"`source_table`"},{"name":"source_table_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"source_table","displayName":"source_table","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Source Table Name","description":"Applied only on details table"}}},"position":{"x":3,"y":28,"width":3,"height":1}},{"widget":{"name":"8e96d5ba","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id","query":{"datasetName":"d4050b68","fields":[{"name":"recon_id","expression":"`recon_id`"},{"name":"recon_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"recon_id","displayName":"recon_id","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Recon Id","description":"Applied only on details table"}}},"position":{"x":0,"y":27,"width":3,"height":1}},{"widget":{"name":"6b586312","queries":[{"name":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by","query":{"datasetName":"655d5caf","fields":[{"name":"executed_by","expression":"`executed_by`"},{"name":"executed_by_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"executed_by","displayName":"executed_by","queryName":"dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Executed by","description":"Applied only on summary table"}}},"position":{"x":4,"y":6,"width":2,"height":2}},{"widget":{"name":"9f0e38d5","queries":[{"name":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema","query":{"datasetName":"655d5caf","parameters":[{"name":"schema","keyword":"schema"}],"disaggregated":false}},{"name":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema","query":{"datasetName":"78e296d2","parameters":[{"name":"schema","keyword":"schema"}],"disaggregated":false}},{"name":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema","query":{"datasetName":"d4050b68","parameters":[{"name":"schema","keyword":"schema"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"parameterName":"schema","queryName":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema"},{"parameterName":"schema","queryName":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema"},{"parameterName":"schema","queryName":"parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema"}]},"frame":{"showTitle":true,"showDescription":true,"title":"Schema","description":"’reconcile’ is the default schema for the metrics table; specify a different schema name if the metrics are stored elsewhere."}}},"position":{"x":3,"y":2,"width":3,"height":2}},{"widget":{"name":"482b5576","textbox_spec":"# Daily Data Validation Issues Report\n### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day."},"position":{"x":0,"y":35,"width":6,"height":2}},{"widget":{"name":"fc96dc6d","queries":[{"name":"main_query","query":{"datasetName":"c132f0ca","fields":[{"name":"target_table","expression":"`target_table`"},{"name":"countdistinct(recon_id)","expression":"COUNT(DISTINCT `recon_id`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"target_table","scale":{"type":"categorical","sort":{"by":"y-reversed"}},"displayName":"target_table"},"y":{"fieldName":"countdistinct(recon_id)","scale":{"type":"quantitative"},"displayName":"Count of Unique recon_id"},"label":{"show":true}},"frame":{"showDescription":false,"title":"Number of Distinct Recon IDs per Target Table Failed","showTitle":true}}},"position":{"x":0,"y":39,"width":5,"height":6}},{"widget":{"name":"4c17bbd2","queries":[{"name":"main_query","query":{"datasetName":"c132f0ca","fields":[{"name":"countdistinct(target_table)","expression":"COUNT(DISTINCT `target_table`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"countdistinct(target_table)","displayName":"Count of Unique target_table"}},"frame":{"title":"Unique target tables failed","showTitle":true,"showDescription":false,"description":"Unique target tables failed"}}},"position":{"x":5,"y":41,"width":1,"height":2}},{"widget":{"name":"d73ad40b","queries":[{"name":"main_query","query":{"datasetName":"c132f0ca","fields":[{"name":"countdistinct(recon_id)","expression":"COUNT(DISTINCT `recon_id`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"countdistinct(recon_id)","displayName":"Count of Unique recon_id"}},"frame":{"title":"Total number of runs failed","showTitle":true}}},"position":{"x":5,"y":39,"width":1,"height":2}},{"widget":{"name":"482ad4f3","queries":[{"name":"main_query","query":{"datasetName":"7637e124","fields":[{"name":"countdistinct(target_table)","expression":"COUNT(DISTINCT `target_table`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"countdistinct(target_table)","displayName":"Count of Unique target_table"}},"frame":{"showTitle":true,"title":"Unique target tables success"}}},"position":{"x":5,"y":43,"width":1,"height":2}},{"widget":{"name":"0aa08479","queries":[{"name":"dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date","query":{"datasetName":"c132f0ca","fields":[{"name":"start_date","expression":"`start_date`"},{"name":"start_date_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date","query":{"datasetName":"7637e124","fields":[{"name":"start_date","expression":"`start_date`"},{"name":"start_date_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-date-range-picker","encodings":{"fields":[{"fieldName":"start_date","displayName":"start_date","queryName":"dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date"},{"fieldName":"start_date","displayName":"start_date","queryName":"dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date"}]}}},"position":{"x":0,"y":37,"width":6,"height":2}}]}]} \ No newline at end of file +{ + "datasets": [ + { + "name": "655d5caf", + "displayName": "recon_main", + "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.exception_message as exception,\nmetrics.recon_metrics.row_comparison.missing_in_source as missing_in_source,\nmetrics.recon_metrics.row_comparison.missing_in_target as missing_in_target,\nmetrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch,\nmetrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch,\nmetrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns,\nmetrics.recon_metrics.schema_comparison as schema_comparison,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", + "parameters": [ + { + "displayName": "catalog", + "keyword": "catalog", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "remorph" + } + ] + } + } + }, + { + "displayName": "schema", + "keyword": "schema", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "reconcile" + } + ] + } + } + } + ] + }, + { + "name": "78e296d2", + "displayName": "recon_details_schema", + "query": "with tmp as (select \nrecon_table_id,\ninserted_ts,\nexplode(data) as schema_data\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type='schema'\n) \nselect \nmain.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nschema_data['source_column'] as source_column,\nschema_data['source_datatype'] as source_datatype,\nschema_data['databricks_column'] as databricks_column,\nschema_data['databricks_datatype'] as databricks_datatype,\nschema_data['is_valid'] as is_valid\nfrom \nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \ntmp\non main.recon_table_id = tmp.recon_table_id\norder by tmp.inserted_ts desc, main.recon_id, main.target_table\n", + "parameters": [ + { + "displayName": "catalog", + "keyword": "catalog", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "remorph" + } + ] + } + } + }, + { + "displayName": "schema", + "keyword": "schema", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "reconcile" + } + ] + } + } + } + ] + }, + { + "name": "d4050b68", + "displayName": "recon_details_pivot", + "query": "with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, \nrow_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type != 'schema')\nselect main.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nrecon_type, key, value, rn\nfrom tmp\ninner join\nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\non main.recon_table_id = tmp.recon_table_id\nlateral view explode(data) exploded_data AS key, value\n", + "parameters": [ + { + "displayName": "catalog", + "keyword": "catalog", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "remorph" + } + ] + } + } + }, + { + "displayName": "schema", + "keyword": "schema", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "reconcile" + } + ] + } + } + } + ] + }, + { + "name": "c132f0ca", + "displayName": "failed_table", + "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = false\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", + "parameters": [ + { + "displayName": "catalog", + "keyword": "catalog", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "remorph" + } + ] + } + } + }, + { + "displayName": "schema", + "keyword": "schema", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "reconcile" + } + ] + } + } + } + ] + }, + { + "name": "7637e124", + "displayName": "success_table", + "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = true\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", + "parameters": [ + { + "displayName": "catalog", + "keyword": "catalog", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "remorph" + } + ] + } + } + }, + { + "displayName": "schema", + "keyword": "schema", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "reconcile" + } + ] + } + } + } + ] + } + ], + "pages": [ + { + "name": "569063d3", + "displayName": "New Page", + "layout": [ + { + "widget": { + "name": "b4a9c7fd", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "source_type", + "expression": "`source_type`" + }, + { + "name": "report_type", + "expression": "`report_type`" + }, + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "status", + "expression": "`status`" + }, + { + "name": "exception", + "expression": "`exception`" + }, + { + "name": "missing_in_source", + "expression": "`missing_in_source`" + }, + { + "name": "missing_in_target", + "expression": "`missing_in_target`" + }, + { + "name": "absolute_mismatch", + "expression": "`absolute_mismatch`" + }, + { + "name": "threshold_mismatch", + "expression": "`threshold_mismatch`" + }, + { + "name": "mismatch_columns", + "expression": "`mismatch_columns`" + }, + { + "name": "schema_comparison", + "expression": "`schema_comparison`" + }, + { + "name": "executed_by", + "expression": "`executed_by`" + }, + { + "name": "start_ts", + "expression": "`start_ts`" + }, + { + "name": "end_ts", + "expression": "`end_ts`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "recon_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "recon_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": null + }, + "rules": [ + { + "if": { + "column": "status", + "fn": "=", + "literal": "true" + }, + "value": { + "foregroundColor": "#3BD973" + } + }, + { + "if": { + "column": "status", + "fn": "=", + "literal": "false" + }, + "value": { + "foregroundColor": "#E92828" + } + } + ] + }, + "displayName": "recon_id" + }, + { + "fieldName": "source_type", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "source_type", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "source_type" + }, + { + "fieldName": "report_type", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "report_type", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "report_type" + }, + { + "fieldName": "source_table", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 5, + "title": "source_table", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "source_table" + }, + { + "fieldName": "target_table", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 8, + "title": "target_table", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "target_table" + }, + { + "fieldName": "status", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "boolean", + "displayAs": "boolean", + "visible": true, + "order": 9, + "title": "status", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "status" + }, + { + "fieldName": "exception", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 10, + "title": "exception", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "exception" + }, + { + "fieldName": "missing_in_source", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "number", + "visible": true, + "order": 11, + "title": "missing_in_source", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "missing_in_source" + }, + { + "fieldName": "missing_in_target", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "number", + "visible": true, + "order": 12, + "title": "missing_in_target", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "missing_in_target" + }, + { + "fieldName": "absolute_mismatch", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "number", + "visible": true, + "order": 13, + "title": "absolute_mismatch", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "absolute_mismatch" + }, + { + "fieldName": "threshold_mismatch", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "number", + "visible": true, + "order": 14, + "title": "threshold_mismatch", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "threshold_mismatch" + }, + { + "fieldName": "mismatch_columns", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 15, + "title": "mismatch_columns", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "mismatch_columns" + }, + { + "fieldName": "schema_comparison", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "boolean", + "displayAs": "boolean", + "visible": true, + "order": 16, + "title": "schema_comparison", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "schema_comparison" + }, + { + "fieldName": "executed_by", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 19, + "title": "executed_by", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "executed_by" + }, + { + "fieldName": "start_ts", + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "datetime", + "displayAs": "datetime", + "visible": true, + "order": 20, + "title": "start_ts", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "start_ts" + }, + { + "fieldName": "end_ts", + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "datetime", + "displayAs": "datetime", + "visible": true, + "order": 21, + "title": "end_ts", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "end_ts" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_catalog", + "type": "string", + "displayAs": "string", + "order": 3, + "title": "source_catalog", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_schema", + "type": "string", + "displayAs": "string", + "order": 4, + "title": "source_schema", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_catalog", + "type": "string", + "displayAs": "string", + "order": 6, + "title": "target_catalog", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_schema", + "type": "string", + "displayAs": "string", + "order": 7, + "title": "target_schema", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_table_name", + "type": "string", + "displayAs": "string", + "order": 17, + "title": "source_table_name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_table_name", + "type": "string", + "displayAs": "string", + "order": 18, + "title": "target_table_name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 100, + "paginationSize": "default", + "condensed": true, + "withRowNumber": true, + "frame": { + "showDescription": true, + "description": "Summary Table", + "showTitle": false + } + } + }, + "position": { + "x": 0, + "y": 10, + "width": 6, + "height": 7 + } + }, + { + "widget": { + "name": "32ea02c2", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "recon_id_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id", + "query": { + "datasetName": "78e296d2", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "recon_id_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "recon_id_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "recon_id", + "displayName": "recon_id", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id" + }, + { + "fieldName": "recon_id", + "displayName": "recon_id", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id" + }, + { + "fieldName": "recon_id", + "displayName": "recon_id", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": false, + "title": "Recon Id", + "description": "" + } + } + }, + "position": { + "x": 0, + "y": 4, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "1be6ab4a", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "target_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table", + "query": { + "datasetName": "78e296d2", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "target_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "target_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "target_table", + "displayName": "target_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table" + }, + { + "fieldName": "target_table", + "displayName": "target_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table" + }, + { + "fieldName": "target_table", + "displayName": "target_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": false, + "title": "Traget Table Name", + "description": "" + } + } + }, + "position": { + "x": 2, + "y": 4, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "3ec3ec2d", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "source_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table", + "query": { + "datasetName": "78e296d2", + "fields": [ + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "source_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "source_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "source_table", + "displayName": "source_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table" + }, + { + "fieldName": "source_table", + "displayName": "source_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table" + }, + { + "fieldName": "source_table", + "displayName": "source_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": false, + "title": "Source Table Name", + "description": "" + } + } + }, + "position": { + "x": 4, + "y": 4, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "8cf05b2f", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "source_type", + "expression": "`source_type`" + }, + { + "name": "source_type_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "source_type", + "displayName": "source_type", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Source Type", + "description": "Applied only on summary table" + } + } + }, + "position": { + "x": 0, + "y": 6, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "21add8d7", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "recon_type", + "expression": "`recon_type`" + }, + { + "name": "recon_type_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-single-select", + "encodings": { + "fields": [ + { + "fieldName": "recon_type", + "displayName": "recon_type", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Category", + "description": "Applied only on details table" + } + } + }, + "position": { + "x": 3, + "y": 27, + "width": 3, + "height": 1 + } + }, + { + "widget": { + "name": "49e40526", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "hourly(start_ts)", + "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" + }, + { + "name": "absolute_mismatch", + "expression": "`absolute_mismatch`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": { + "fieldName": "hourly(start_ts)", + "scale": { + "type": "temporal" + }, + "displayName": "start_ts" + }, + "y": { + "fieldName": "absolute_mismatch", + "scale": { + "type": "quantitative" + }, + "displayName": "absolute_mismatch" + }, + "color": { + "fieldName": "target_table", + "scale": { + "type": "categorical" + }, + "displayName": "target_table" + }, + "label": { + "show": false + } + }, + "frame": { + "showTitle": true, + "title": "Mismatched Records", + "showDescription": false + } + } + }, + "position": { + "x": 0, + "y": 47, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "2cbc9f42", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "hourly(start_ts)", + "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" + }, + { + "name": "missing_in_target", + "expression": "`missing_in_target`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": { + "fieldName": "hourly(start_ts)", + "scale": { + "type": "temporal" + }, + "displayName": "start_ts" + }, + "y": { + "fieldName": "missing_in_target", + "scale": { + "type": "quantitative" + }, + "displayName": "missing_in_target" + }, + "color": { + "fieldName": "target_table", + "scale": { + "type": "categorical" + }, + "displayName": "target_table" + }, + "label": { + "show": false + } + }, + "frame": { + "showTitle": true, + "title": "Missing in Source" + } + } + }, + "position": { + "x": 3, + "y": 53, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "5ae38078", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "hourly(start_ts)", + "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" + }, + { + "name": "missing_in_source", + "expression": "`missing_in_source`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": { + "fieldName": "hourly(start_ts)", + "scale": { + "type": "temporal" + }, + "displayName": "start_ts" + }, + "y": { + "fieldName": "missing_in_source", + "scale": { + "type": "quantitative" + }, + "displayName": "missing_in_source" + }, + "color": { + "fieldName": "target_table", + "scale": { + "type": "categorical" + }, + "displayName": "target_table" + }, + "label": { + "show": false + } + }, + "frame": { + "showTitle": true, + "title": "Missing in Databricks" + } + } + }, + "position": { + "x": 0, + "y": 53, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "2ea234ee", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "hourly(start_ts)", + "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" + }, + { + "name": "threshold_mismatch", + "expression": "`threshold_mismatch`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": { + "fieldName": "hourly(start_ts)", + "scale": { + "type": "temporal" + }, + "displayName": "start_ts" + }, + "y": { + "fieldName": "threshold_mismatch", + "scale": { + "type": "quantitative" + }, + "displayName": "threshold_mismatch" + }, + "color": { + "fieldName": "target_table", + "scale": { + "type": "categorical" + }, + "displayName": "target_table" + } + }, + "frame": { + "showTitle": true, + "title": "Threshold Mismatches" + } + } + }, + "position": { + "x": 3, + "y": 47, + "width": 3, + "height": 6 + } + }, + { + "widget": { + "name": "403919aa", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "78e296d2", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "source_column", + "expression": "`source_column`" + }, + { + "name": "source_datatype", + "expression": "`source_datatype`" + }, + { + "name": "databricks_column", + "expression": "`databricks_column`" + }, + { + "name": "databricks_datatype", + "expression": "`databricks_datatype`" + }, + { + "name": "is_valid", + "expression": "`is_valid`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "recon_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "recon_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": null + }, + "rules": [ + { + "if": { + "column": "is_valid", + "fn": "=", + "literal": "false" + }, + "value": { + "foregroundColor": "#E92828" + } + }, + { + "if": { + "column": "is_valid", + "fn": "=", + "literal": "true" + }, + "value": { + "foregroundColor": "#3BD973" + } + } + ] + }, + "displayName": "recon_id" + }, + { + "fieldName": "source_table", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 4, + "title": "source_table", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "source_table" + }, + { + "fieldName": "target_table", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 8, + "title": "target_table", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "target_table" + }, + { + "fieldName": "source_column", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 9, + "title": "source_column", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "source_column" + }, + { + "fieldName": "source_datatype", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 10, + "title": "source_datatype", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "source_datatype" + }, + { + "fieldName": "databricks_column", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "databricks_column", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "databricks_column" + }, + { + "fieldName": "databricks_datatype", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 12, + "title": "databricks_datatype", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "databricks_datatype" + }, + { + "fieldName": "is_valid", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 13, + "title": "is_valid", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "is_valid" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_table_name", + "type": "string", + "displayAs": "string", + "order": 1, + "title": "source_table_name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_catalog", + "type": "string", + "displayAs": "string", + "order": 2, + "title": "source_catalog", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "source_schema", + "type": "string", + "displayAs": "string", + "order": 3, + "title": "source_schema", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_catalog", + "type": "string", + "displayAs": "string", + "order": 5, + "title": "target_catalog", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_schema", + "type": "string", + "displayAs": "string", + "order": 6, + "title": "target_schema", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "target_table_name", + "type": "string", + "displayAs": "string", + "order": 7, + "title": "target_table_name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": true, + "withRowNumber": false, + "frame": { + "showTitle": true, + "title": "Schema Details" + } + } + }, + "position": { + "x": 0, + "y": 19, + "width": 6, + "height": 6 + } + }, + { + "widget": { + "name": "b154de70", + "queries": [ + { + "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog", + "query": { + "datasetName": "655d5caf", + "parameters": [ + { + "name": "catalog", + "keyword": "catalog" + } + ], + "disaggregated": false + } + }, + { + "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog", + "query": { + "datasetName": "78e296d2", + "parameters": [ + { + "name": "catalog", + "keyword": "catalog" + } + ], + "disaggregated": false + } + }, + { + "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog", + "query": { + "datasetName": "d4050b68", + "parameters": [ + { + "name": "catalog", + "keyword": "catalog" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-single-select", + "encodings": { + "fields": [ + { + "parameterName": "catalog", + "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog" + }, + { + "parameterName": "catalog", + "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog" + }, + { + "parameterName": "catalog", + "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Catalog", + "description": "’remorph' is the default catalog for the metrics table; specify a different catalog name if the metrics are stored elsewhere." + } + } + }, + "position": { + "x": 0, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "widget": { + "name": "05101c82", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "report_type", + "expression": "`report_type`" + }, + { + "name": "report_type_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "report_type", + "displayName": "report_type", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Report", + "description": "Applied only on summary table" + } + } + }, + "position": { + "x": 2, + "y": 6, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "081790c2", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "start_ts", + "expression": "`start_ts`" + }, + { + "name": "start_ts_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-date-range-picker", + "encodings": { + "fields": [ + { + "fieldName": "start_ts", + "displayName": "start_ts", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "started at", + "description": "Applied on all tables" + } + } + }, + "position": { + "x": 0, + "y": 8, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "ffcb9c29", + "textbox_spec": "# Main Reconciliation Table\n### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records." + }, + "position": { + "x": 0, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "acd1adcc", + "textbox_spec": "# Drill Down\n### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues." + }, + "position": { + "x": 0, + "y": 25, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "680c9fe9", + "textbox_spec": "# Schema Comparison Details\n### This table provides a detailed view of schema mismatches." + }, + "position": { + "x": 0, + "y": 17, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "b1a2db69", + "textbox_spec": "# Visualization of Missing and Mismatched Records" + }, + "position": { + "x": 0, + "y": 45, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "2199286c", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "value", + "expression": "`value`" + }, + { + "name": "key", + "expression": "`key`" + }, + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "recon_type", + "expression": "`recon_type`" + }, + { + "name": "rn", + "expression": "`rn`" + } + ], + "disaggregated": false, + "orders": [ + { + "direction": "ASC", + "expression": "`recon_id`" + }, + { + "direction": "ASC", + "expression": "`source_table`" + }, + { + "direction": "ASC", + "expression": "`target_table`" + }, + { + "direction": "ASC", + "expression": "`recon_type`" + }, + { + "direction": "ASC", + "expression": "`rn`" + }, + { + "direction": "ASC", + "expression": "`key`" + } + ] + } + } + ], + "spec": { + "version": 3, + "widgetType": "pivot", + "encodings": { + "rows": [ + { + "fieldName": "recon_id", + "displayName": "recon_id" + }, + { + "fieldName": "source_table", + "displayName": "source_table" + }, + { + "fieldName": "target_table", + "displayName": "target_table" + }, + { + "fieldName": "recon_type", + "displayName": "recon_type" + }, + { + "fieldName": "rn", + "displayName": "rn" + } + ], + "columns": [ + { + "fieldName": "key", + "displayName": "key" + } + ], + "cell": { + "fieldName": "value", + "cellType": "text", + "displayName": "value" + } + } + } + }, + "position": { + "x": 0, + "y": 29, + "width": 6, + "height": 6 + } + }, + { + "widget": { + "name": "9c0a2a5e", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "target_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "target_table", + "displayName": "target_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table" + } + ] + }, + "frame": { + "showDescription": true, + "showTitle": true, + "title": "Target Table Name", + "description": "Applied only on details table" + } + } + }, + "position": { + "x": 0, + "y": 28, + "width": 3, + "height": 1 + } + }, + { + "widget": { + "name": "1e536216", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "source_table", + "expression": "`source_table`" + }, + { + "name": "source_table_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "source_table", + "displayName": "source_table", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Source Table Name", + "description": "Applied only on details table" + } + } + }, + "position": { + "x": 3, + "y": 28, + "width": 3, + "height": 1 + } + }, + { + "widget": { + "name": "8e96d5ba", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", + "query": { + "datasetName": "d4050b68", + "fields": [ + { + "name": "recon_id", + "expression": "`recon_id`" + }, + { + "name": "recon_id_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "recon_id", + "displayName": "recon_id", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Recon Id", + "description": "Applied only on details table" + } + } + }, + "position": { + "x": 0, + "y": 27, + "width": 3, + "height": 1 + } + }, + { + "widget": { + "name": "6b586312", + "queries": [ + { + "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by", + "query": { + "datasetName": "655d5caf", + "fields": [ + { + "name": "executed_by", + "expression": "`executed_by`" + }, + { + "name": "executed_by_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + { + "fieldName": "executed_by", + "displayName": "executed_by", + "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Executed by", + "description": "Applied only on summary table" + } + } + }, + "position": { + "x": 4, + "y": 6, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "9f0e38d5", + "queries": [ + { + "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema", + "query": { + "datasetName": "655d5caf", + "parameters": [ + { + "name": "schema", + "keyword": "schema" + } + ], + "disaggregated": false + } + }, + { + "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema", + "query": { + "datasetName": "78e296d2", + "parameters": [ + { + "name": "schema", + "keyword": "schema" + } + ], + "disaggregated": false + } + }, + { + "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema", + "query": { + "datasetName": "d4050b68", + "parameters": [ + { + "name": "schema", + "keyword": "schema" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-single-select", + "encodings": { + "fields": [ + { + "parameterName": "schema", + "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema" + }, + { + "parameterName": "schema", + "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema" + }, + { + "parameterName": "schema", + "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema" + } + ] + }, + "frame": { + "showTitle": true, + "showDescription": true, + "title": "Schema", + "description": "’reconcile’ is the default schema for the metrics table; specify a different schema name if the metrics are stored elsewhere." + } + } + }, + "position": { + "x": 3, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "widget": { + "name": "482b5576", + "textbox_spec": "# Daily Data Validation Issues Report\n### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day." + }, + "position": { + "x": 0, + "y": 35, + "width": 6, + "height": 2 + } + }, + { + "widget": { + "name": "fc96dc6d", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "c132f0ca", + "fields": [ + { + "name": "target_table", + "expression": "`target_table`" + }, + { + "name": "countdistinct(recon_id)", + "expression": "COUNT(DISTINCT `recon_id`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "target_table", + "scale": { + "type": "categorical", + "sort": { + "by": "y-reversed" + } + }, + "displayName": "target_table" + }, + "y": { + "fieldName": "countdistinct(recon_id)", + "scale": { + "type": "quantitative" + }, + "displayName": "Count of Unique recon_id" + }, + "label": { + "show": true + } + }, + "frame": { + "showDescription": false, + "title": "Number of Distinct Recon IDs per Target Table Failed", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 39, + "width": 5, + "height": 6 + } + }, + { + "widget": { + "name": "4c17bbd2", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "c132f0ca", + "fields": [ + { + "name": "countdistinct(target_table)", + "expression": "COUNT(DISTINCT `target_table`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "countdistinct(target_table)", + "displayName": "Count of Unique target_table" + } + }, + "frame": { + "title": "Unique target tables failed", + "showTitle": true, + "showDescription": false, + "description": "Unique target tables failed" + } + } + }, + "position": { + "x": 5, + "y": 41, + "width": 1, + "height": 2 + } + }, + { + "widget": { + "name": "d73ad40b", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "c132f0ca", + "fields": [ + { + "name": "countdistinct(recon_id)", + "expression": "COUNT(DISTINCT `recon_id`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "countdistinct(recon_id)", + "displayName": "Count of Unique recon_id" + } + }, + "frame": { + "title": "Total number of runs failed", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 39, + "width": 1, + "height": 2 + } + }, + { + "widget": { + "name": "482ad4f3", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "7637e124", + "fields": [ + { + "name": "countdistinct(target_table)", + "expression": "COUNT(DISTINCT `target_table`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "countdistinct(target_table)", + "displayName": "Count of Unique target_table" + } + }, + "frame": { + "showTitle": true, + "title": "Unique target tables success" + } + } + }, + "position": { + "x": 5, + "y": 43, + "width": 1, + "height": 2 + } + }, + { + "widget": { + "name": "0aa08479", + "queries": [ + { + "name": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date", + "query": { + "datasetName": "c132f0ca", + "fields": [ + { + "name": "start_date", + "expression": "`start_date`" + }, + { + "name": "start_date_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date", + "query": { + "datasetName": "7637e124", + "fields": [ + { + "name": "start_date", + "expression": "`start_date`" + }, + { + "name": "start_date_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-date-range-picker", + "encodings": { + "fields": [ + { + "fieldName": "start_date", + "displayName": "start_date", + "queryName": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date" + }, + { + "fieldName": "start_date", + "displayName": "start_date", + "queryName": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date" + } + ] + } + } + }, + "position": { + "x": 0, + "y": 37, + "width": 6, + "height": 2 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml new file mode 100644 index 000000000..e69de29bb diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql new file mode 100644 index 000000000..a8dfea4f7 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql @@ -0,0 +1,26 @@ +select +main.recon_id, +main.source_type, +main.report_type, +main.source_table.`catalog` as source_catalog, +main.source_table.`schema` as source_schema, +main.source_table.table_name as source_table_name, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +main.target_table.`catalog` as target_catalog, +main.target_table.`schema` as target_schema, +main.target_table.table_name as target_table_name, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +metrics.run_metrics.status as status, +metrics.run_metrics.run_by_user as executed_by, +main.start_ts as start_ts, +main.end_ts as end_ts, +date(main.start_ts) as start_date +from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +inner join +IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics +on main.recon_table_id = metrics.recon_table_id +where metrics.run_metrics.status = false +order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql new file mode 100644 index 000000000..430f32489 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql @@ -0,0 +1,22 @@ +with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, +row_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn +from IDENTIFIER(:catalog || '.' || :schema || '.details' ) +where recon_type != 'schema') +select main.recon_id, +main.source_table.`catalog` as source_catalog, +main.source_table.`schema` as source_schema, +main.source_table.table_name as source_table_name, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +main.target_table.`catalog` as target_catalog, +main.target_table.`schema` as target_schema, +main.target_table.table_name as target_table_name, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +recon_type, key, value, rn +from tmp +inner join +IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +on main.recon_table_id = tmp.recon_table_id +lateral view explode(data) exploded_data AS key, value diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql new file mode 100644 index 000000000..804be609f --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql @@ -0,0 +1,31 @@ +with tmp as (select +recon_table_id, +inserted_ts, +explode(data) as schema_data +from IDENTIFIER(:catalog || '.' || :schema || '.details' ) +where recon_type='schema' +) +select +main.recon_id, +main.source_table.`catalog` as source_catalog, +main.source_table.`schema` as source_schema, +main.source_table.table_name as source_table_name, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +main.target_table.`catalog` as target_catalog, +main.target_table.`schema` as target_schema, +main.target_table.table_name as target_table_name, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +schema_data['source_column'] as source_column, +schema_data['source_datatype'] as source_datatype, +schema_data['databricks_column'] as databricks_column, +schema_data['databricks_datatype'] as databricks_datatype, +schema_data['is_valid'] as is_valid +from +IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +inner join +tmp +on main.recon_table_id = tmp.recon_table_id +order by tmp.inserted_ts desc, main.recon_id, main.target_table diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql new file mode 100644 index 000000000..01842b489 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql @@ -0,0 +1,31 @@ +select +main.recon_id, +main.source_type, +main.report_type, +main.source_table.`catalog` as source_catalog, +main.source_table.`schema` as source_schema, +main.source_table.table_name as source_table_name, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +main.target_table.`catalog` as target_catalog, +main.target_table.`schema` as target_schema, +main.target_table.table_name as target_table_name, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +metrics.run_metrics.status as status, +metrics.run_metrics.exception_message as exception, +metrics.recon_metrics.row_comparison.missing_in_source as missing_in_source, +metrics.recon_metrics.row_comparison.missing_in_target as missing_in_target, +metrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch, +metrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch, +metrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns, +metrics.recon_metrics.schema_comparison as schema_comparison, +metrics.run_metrics.run_by_user as executed_by, +main.start_ts as start_ts, +main.end_ts as end_ts +from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +inner join +IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics +on main.recon_table_id = metrics.recon_table_id +order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql new file mode 100644 index 000000000..898076bdf --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql @@ -0,0 +1,20 @@ +select +main.recon_id, +main.source_type, +main.report_type, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +metrics.run_metrics.status as status, +metrics.run_metrics.run_by_user as executed_by, +main.start_ts as start_ts, +main.end_ts as end_ts, +date(main.start_ts) as start_date +from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +inner join +IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics +on main.recon_table_id = metrics.recon_table_id +where metrics.run_metrics.status = true +order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file From 20dde6bf1f923a30b1fbbfbe79445a0785d3238a Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Mon, 5 Aug 2024 10:36:17 +0530 Subject: [PATCH 02/10] fix pom --- core/pom.xml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 7efa1c8c7..825038335 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -16,7 +16,7 @@ 1.8 1.8 - 5.11.0 + 4.11.0 UTF-8 2.0.9 4.13.1 @@ -47,7 +47,7 @@ com.databricks databricks-sdk-java - 0.24.1 + 0.21.0 org.apache.spark @@ -78,7 +78,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.3.0-SNAP4 + 3.3.0-SNAP3 test @@ -230,11 +230,4 @@ - - - maven_central - Maven Central - https://repo.maven.apache.org/maven2/ - - From a8d48352bb6fc5c73a2749a937b0ee83301feeca Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Mon, 5 Aug 2024 10:42:41 +0530 Subject: [PATCH 03/10] Renamed Files --- .../reconcile/dashboards/{recon_main.sql => 0_1_recon_main.sql} | 0 .../{recon_details_schema.sql => 1_0_recon_details_schema.sql} | 0 .../{recon_details_pivot.sql => 1_1_recon_details_pivot.sql} | 0 .../{failed_table_list.sql => 2_1_failed_table_list.sql} | 0 .../{success_table_list.sql => 2_2_success_table_list.sql} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{recon_main.sql => 0_1_recon_main.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{recon_details_schema.sql => 1_0_recon_details_schema.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{recon_details_pivot.sql => 1_1_recon_details_pivot.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{failed_table_list.sql => 2_1_failed_table_list.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{success_table_list.sql => 2_2_success_table_list.sql} (100%) diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/recon_main.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_schema.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/recon_details_pivot.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/failed_table_list.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/success_table_list.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql From dffb4db04efd23952ca1ab1598def2974e6d314a Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Mon, 5 Aug 2024 20:52:48 +0530 Subject: [PATCH 04/10] Enhancement --- pyproject.toml | 2 +- .../labs/remorph/deployment/dashboard.py | 133 ++++++++++-------- .../labs/remorph/deployment/recon.py | 12 +- .../reconcile/dashboards/dashboard.yml | 1 + tests/unit/deployment/test_dashboard.py | 19 +-- 5 files changed, 87 insertions(+), 80 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1f9a39d23..e3a5d8218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "databricks-sdk~=0.29.0", "sqlglot==25.8.1", "databricks-labs-blueprint[yaml]>=0.2.3", - "databricks-labs-lsql>=0.4.3", + "databricks-labs-lsql>=0.7.5", ] [project.urls] diff --git a/src/databricks/labs/remorph/deployment/dashboard.py b/src/databricks/labs/remorph/deployment/dashboard.py index 98cdd46d6..5fd359a7e 100644 --- a/src/databricks/labs/remorph/deployment/dashboard.py +++ b/src/databricks/labs/remorph/deployment/dashboard.py @@ -1,16 +1,17 @@ -import json import logging from datetime import timedelta -from importlib.abc import Traversable -from typing import Any +from pathlib import Path from databricks.labs.blueprint.installation import Installation from databricks.labs.blueprint.installer import InstallState from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import DatabricksError -from databricks.sdk.errors import InvalidParameterValue from databricks.sdk.retries import retried -from databricks.sdk.service.dashboards import Dashboard +from databricks.sdk.errors import InvalidParameterValue, NotFound, DatabricksError + +from databricks.sdk.service.dashboards import LifecycleState +from databricks.labs.lsql.dashboards import DashboardMetadata, Dashboards + +from databricks.labs.remorph.config import ReconcileMetadataConfig logger = logging.getLogger(__name__) @@ -18,65 +19,83 @@ class DashboardDeployment: _UPLOAD_TIMEOUT = timedelta(seconds=30) - def __init__(self, ws: WorkspaceClient, installation: Installation, install_state: InstallState): + def __init__( + self, + ws: WorkspaceClient, + installation: Installation, + install_state: InstallState, + ): self._ws = ws self._installation = installation self._install_state = install_state - def deploy(self, name: str, dashboard_file: Traversable, parameters: dict[str, Any] | None = None): - logger.debug(f"Deploying dashboard {name} from {dashboard_file.name}") - dashboard_data = self._substitute_params(dashboard_file, parameters or {}) - dashboard = self._update_or_create_dashboard(name, dashboard_data, dashboard_file) - logger.info(f"Dashboard deployed with dashboard_id {dashboard.dashboard_id}") - logger.info(f"Dashboard URL: {self._ws.config.host}/sql/dashboardsv3/{dashboard.dashboard_id}") + def _handle_existing_dashboard(self, dashboard_id: str, display_name: str, parent_path: str) -> str | None: + try: + dashboard = self._ws.lakeview.get(dashboard_id) + if dashboard.lifecycle_state is None: + raise NotFound(f"Dashboard life cycle state: {display_name} ({dashboard_id})") + if dashboard.lifecycle_state == LifecycleState.TRASHED: + logger.info(f"Recreating trashed dashboard: {display_name} ({dashboard_id})") + return None # Recreate the dashboard if it is trashed (manually) + + except (NotFound, InvalidParameterValue): + logger.info(f"Recovering invalid dashboard: {display_name} ({dashboard_id})") + try: + dashboard_path = f"{parent_path}/{display_name}.lvdash.json" + self._ws.workspace.delete(dashboard_path) # Cannot recreate dashboard if file still exists + logger.debug(f"Deleted dangling dashboard {display_name} ({dashboard_id}): {dashboard_path}") + except NotFound: + pass + return None # Recreate the dashboard if it's reference is corrupted (manually) + return dashboard_id # Update the existing dashboard + + def deploy( + self, + name: str, + folder: Path, + config: ReconcileMetadataConfig, + ): + """Create a dashboard from Queries inside folder""" + logger.info(f"Deploying dashboard {name} from {folder}") + parent_path = f"{self._installation.install_folder()}/dashboards" + + metadata = DashboardMetadata.from_path(folder).replace_database( + database=f"hive_metastore.{config.schema}", + database_to_replace="inventory", + ) + + metadata.display_name = self._name_with_prefix(metadata.display_name) + + reference = f"{folder.parent.stem}_{folder.stem}".lower() + dashboard_id = self._install_state.dashboards.get(reference) + if dashboard_id is not None: + dashboard_id = self._handle_existing_dashboard(dashboard_id, metadata.display_name, parent_path) + + # dashboard_data = self._substitute_params(dashboard_file, parameters or {}) + dashboard_id = self._update_or_create_dashboard(name, dashboard_id, metadata, parent_path) + logger.info(f"Dashboard deployed with dashboard_id {dashboard_id}") + logger.info(f"Dashboard URL: {self._ws.config.host}/sql/dashboardsv3/{dashboard_id}") self._install_state.save() @retried(on=[DatabricksError], timeout=_UPLOAD_TIMEOUT) - def _update_or_create_dashboard(self, name: str, dashboard_data, dashboard_file) -> Dashboard: - if name in self._install_state.dashboards: - try: - dashboard_id = self._install_state.dashboards[name] - logger.info(f"Updating dashboard with id={dashboard_id}") - updated_dashboard = self._ws.lakeview.update( - dashboard_id, - display_name=self._name_with_prefix(name), - serialized_dashboard=dashboard_data, - ) - return updated_dashboard - except InvalidParameterValue: - del self._install_state.dashboards[name] - logger.warning(f"Dashboard {name} does not exist anymore for some reason.") - return self._update_or_create_dashboard(name, dashboard_data, dashboard_file) - logger.info(f"Creating new dashboard {name}") - new_dashboard = self._ws.lakeview.create( - display_name=self._name_with_prefix(name), - parent_path=self._install_state.install_folder(), - serialized_dashboard=dashboard_data, + def _update_or_create_dashboard( + self, + name: str, + dashboard_id: str, + metadata: DashboardMetadata, + parent_path: str, + ) -> str: + + dashboard = Dashboards(self._ws).create_dashboard( + metadata, + dashboard_id=dashboard_id, + parent_path=parent_path, + warehouse_id=self._ws.config.warehouse_id, + publish=True, ) - assert new_dashboard.dashboard_id is not None - self._install_state.dashboards[name] = new_dashboard.dashboard_id - return new_dashboard - - def _substitute_params(self, dashboard_file: Traversable, parameters: dict[str, Any]) -> str: - if not parameters: - return dashboard_file.read_text() - - with dashboard_file.open() as f: - dashboard_data = json.load(f) - - for dataset in dashboard_data.get("datasets", []): - for param in dataset.get("parameters", []): - if param["keyword"] in parameters: - param["defaultSelection"] = { - "values": { - "dataType": "STRING", - "values": [ - {"value": parameters[param["keyword"]]}, - ], - }, - } - - return json.dumps(dashboard_data) + assert dashboard.dashboard_id is not None + self._install_state.dashboards[name] = dashboard.dashboard_id + return dashboard.dashboard_id def _name_with_prefix(self, name: str) -> str: prefix = self._installation.product() diff --git a/src/databricks/labs/remorph/deployment/recon.py b/src/databricks/labs/remorph/deployment/recon.py index 5861c1d2d..809731307 100644 --- a/src/databricks/labs/remorph/deployment/recon.py +++ b/src/databricks/labs/remorph/deployment/recon.py @@ -6,7 +6,7 @@ from databricks.labs.blueprint.wheels import ProductInfo from databricks.sdk import WorkspaceClient from databricks.sdk.errors import InvalidParameterValue, NotFound - +from databricks.labs.blueprint.wheels import find_project_root import databricks.labs.remorph.resources from databricks.labs.remorph.config import ReconcileConfig from databricks.labs.remorph.deployment.dashboard import DashboardDeployment @@ -91,15 +91,9 @@ def _deploy_dashboards(self, recon_config: ReconcileConfig): continue def _deploy_recon_metrics_dashboard(self, name: str, recon_config: ReconcileConfig): - dashboard_params = { - "catalog": recon_config.metadata_config.catalog, - "schema": recon_config.metadata_config.schema, - } - - reconcile_dashboard_path = "reconcile/dashboards/Remorph-Reconciliation.lvdash.json" - dashboard_file = files(databricks.labs.remorph.resources).joinpath(reconcile_dashboard_path) + queries_folder = find_project_root(__file__) / "src/databricks/labs/remorph/resources/reconcile/dashboard" logger.info(f"Creating Reconciliation Dashboard `{name}`") - self._dashboard_deployer.deploy(name, dashboard_file, parameters=dashboard_params) + self._dashboard_deployer.deploy(name, queries_folder, recon_config.metadata_config) def _get_dashboards(self) -> list[tuple[str, str]]: return [ diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml index e69de29bb..c331d2fd8 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml @@ -0,0 +1 @@ +display_name: "Reconciliation Metrics" \ No newline at end of file diff --git a/tests/unit/deployment/test_dashboard.py b/tests/unit/deployment/test_dashboard.py index 0f4bc528e..8eb52eff0 100644 --- a/tests/unit/deployment/test_dashboard.py +++ b/tests/unit/deployment/test_dashboard.py @@ -7,6 +7,7 @@ from databricks.sdk.errors import InvalidParameterValue from databricks.sdk.service.dashboards import Dashboard +from databricks.labs.remorph.config import ReconcileMetadataConfig from databricks.labs.remorph.deployment.dashboard import DashboardDeployment @@ -22,7 +23,7 @@ def test_deploy_new_dashboard(): install_state = InstallState.from_installation(installation) name = "Remorph-Reconciliation" dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file) + dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) _, kwargs = ws.lakeview.create.call_args assert kwargs["serialized_dashboard"] == dashboard_file.read_text() assert install_state.dashboards[name] == dashboard.dashboard_id @@ -45,11 +46,7 @@ def test_deploy_new_dashboard_with_params(): install_state = InstallState.from_installation(installation) name = "Remorph-Reconciliation" dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_params = { - "catalog": "remorph1", - "schema": "reconcile1", - } - dashboard_publisher.deploy(name, dashboard_file, parameters=dashboard_params) + dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) _, kwargs = ws.lakeview.create.call_args assert kwargs["serialized_dashboard"] == substituted_dashboard_file.read_text() assert install_state.dashboards[name] == dashboard.dashboard_id @@ -68,11 +65,7 @@ def test_deploy_new_parameterless_dashboard_with_user_params(): install_state = InstallState.from_installation(installation) name = "Test_Dashboard_No_Param" dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_params = { - "catalog": "remorph1", - "schema": "reconcile1", - } - dashboard_publisher.deploy(name, dashboard_file, parameters=dashboard_params) + dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) assert install_state.dashboards[name] == dashboard.dashboard_id @@ -89,7 +82,7 @@ def test_deploy_existing_dashboard(): installation = MockInstallation({"state.json": {"resources": {"dashboards": {name: dashboard_id}}, "version": 1}}) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file) + dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) _, kwargs = ws.lakeview.update.call_args assert kwargs["serialized_dashboard"] == dashboard_file.read_text() assert install_state.dashboards[name] == dashboard.dashboard_id @@ -116,7 +109,7 @@ def test_deploy_missing_dashboard(): ) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file) + dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) _, kwargs = ws.lakeview.create.call_args assert kwargs["serialized_dashboard"] == dashboard_file.read_text() assert install_state.dashboards[name] == dashboard.dashboard_id From a28d55a609e1b486623e75b42905093d8fac8fbb Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Tue, 6 Aug 2024 19:13:36 +0530 Subject: [PATCH 05/10] intermediate commit --- .../labs/remorph/deployment/dashboard.py | 6 +++- .../labs/remorph/deployment/installation.py | 1 + .../labs/remorph/deployment/recon.py | 5 +-- ...rph-Reconciliation-Substituted.lvdash.json | 1 - .../Remorph-Reconciliation.lvdash.json | 1 - .../Test_Dashboard_No_Param.lvdash.json | 1 - .../dashboards/queries/00_description.md | 1 + .../dashboards/queries/01_queries.sql | 31 +++++++++++++++++++ .../dashboards/queries/dashboard.yml | 1 + tests/unit/deployment/test_dashboard.py | 17 ++++++---- 10 files changed, 53 insertions(+), 12 deletions(-) delete mode 100644 tests/resources/Remorph-Reconciliation-Substituted.lvdash.json delete mode 100644 tests/resources/Remorph-Reconciliation.lvdash.json delete mode 100644 tests/resources/Test_Dashboard_No_Param.lvdash.json create mode 100644 tests/resources/dashboards/queries/00_description.md create mode 100644 tests/resources/dashboards/queries/01_queries.sql create mode 100644 tests/resources/dashboards/queries/dashboard.yml diff --git a/src/databricks/labs/remorph/deployment/dashboard.py b/src/databricks/labs/remorph/deployment/dashboard.py index 5fd359a7e..53d32a1a1 100644 --- a/src/databricks/labs/remorph/deployment/dashboard.py +++ b/src/databricks/labs/remorph/deployment/dashboard.py @@ -6,7 +6,7 @@ from databricks.labs.blueprint.installer import InstallState from databricks.sdk import WorkspaceClient from databricks.sdk.retries import retried -from databricks.sdk.errors import InvalidParameterValue, NotFound, DatabricksError +from databricks.sdk.errors import InvalidParameterValue, NotFound, DatabricksError, ResourceAlreadyExists from databricks.sdk.service.dashboards import LifecycleState from databricks.labs.lsql.dashboards import DashboardMetadata, Dashboards @@ -58,6 +58,10 @@ def deploy( """Create a dashboard from Queries inside folder""" logger.info(f"Deploying dashboard {name} from {folder}") parent_path = f"{self._installation.install_folder()}/dashboards" + try: + self._ws.workspace.mkdirs(parent_path) + except ResourceAlreadyExists: + pass metadata = DashboardMetadata.from_path(folder).replace_database( database=f"hive_metastore.{config.schema}", diff --git a/src/databricks/labs/remorph/deployment/installation.py b/src/databricks/labs/remorph/deployment/installation.py index cb6cacb82..95a7228a4 100644 --- a/src/databricks/labs/remorph/deployment/installation.py +++ b/src/databricks/labs/remorph/deployment/installation.py @@ -26,6 +26,7 @@ def __init__( def install(self, config: RemorphConfigs): if config.reconcile: + logger.info("Installing Remorph reconcile Metadata components.") self._recon_deployment.install(config.reconcile) def uninstall(self, config: RemorphConfigs): diff --git a/src/databricks/labs/remorph/deployment/recon.py b/src/databricks/labs/remorph/deployment/recon.py index 809731307..8bfcfe2dd 100644 --- a/src/databricks/labs/remorph/deployment/recon.py +++ b/src/databricks/labs/remorph/deployment/recon.py @@ -40,9 +40,10 @@ def __init__( self._dashboard_deployer = dashboard_deployer def install(self, recon_config: ReconcileConfig | None): + logger.info("Installing reconcile components.") if not recon_config: + logger.warning("Recon Config is empty") return - logger.info("Installing reconcile components.") self._deploy_tables(recon_config) self._deploy_dashboards(recon_config) self._deploy_jobs(recon_config) @@ -91,7 +92,7 @@ def _deploy_dashboards(self, recon_config: ReconcileConfig): continue def _deploy_recon_metrics_dashboard(self, name: str, recon_config: ReconcileConfig): - queries_folder = find_project_root(__file__) / "src/databricks/labs/remorph/resources/reconcile/dashboard" + queries_folder = find_project_root(__file__) / "src/databricks/labs/remorph/resources/reconcile/dashboards" logger.info(f"Creating Reconciliation Dashboard `{name}`") self._dashboard_deployer.deploy(name, queries_folder, recon_config.metadata_config) diff --git a/tests/resources/Remorph-Reconciliation-Substituted.lvdash.json b/tests/resources/Remorph-Reconciliation-Substituted.lvdash.json deleted file mode 100644 index f91b46714..000000000 --- a/tests/resources/Remorph-Reconciliation-Substituted.lvdash.json +++ /dev/null @@ -1 +0,0 @@ -{"datasets": [{"name": "088adb8a", "displayName": "recon_main", "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.exception_message as exception,\nmetrics.recon_metrics.row_comparison.missing_in_source as missing_in_source,\nmetrics.recon_metrics.row_comparison.missing_in_target as missing_in_target,\nmetrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch,\nmetrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch,\nmetrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns,\nmetrics.recon_metrics.schema_comparison as schema_comparison,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph1"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile1"}]}}}]}, {"name": "07f3d7d5", "displayName": "recon_details_schema", "query": "with tmp as (select \nrecon_table_id,\ninserted_ts,\nexplode(data) as schema_data\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type='schema'\n) \nselect \nmain.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nschema_data['source_column'] as source_column,\nschema_data['source_datatype'] as source_datatype,\nschema_data['databricks_column'] as databricks_column,\nschema_data['databricks_datatype'] as databricks_datatype,\nschema_data['is_valid'] as is_valid\nfrom \nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \ntmp\non main.recon_table_id = tmp.recon_table_id\norder by tmp.inserted_ts desc, main.recon_id, main.target_table\n", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph1"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile1"}]}}}]}, {"name": "aec31225", "displayName": "recon_details_pivot", "query": "with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, \nrow_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type != 'schema')\nselect main.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nrecon_type, key, value, rn\nfrom tmp\ninner join\nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\non main.recon_table_id = tmp.recon_table_id\nlateral view explode(data) exploded_data AS key, value\n", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph1"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile1"}]}}}]}], "pages": [{"name": "1ac59528", "displayName": "New Page", "layout": [{"widget": {"name": "b3208d80", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_type", "expression": "`source_type`"}, {"name": "report_type", "expression": "`report_type`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "status", "expression": "`status`"}, {"name": "exception", "expression": "`exception`"}, {"name": "missing_in_source", "expression": "`missing_in_source`"}, {"name": "missing_in_target", "expression": "`missing_in_target`"}, {"name": "absolute_mismatch", "expression": "`absolute_mismatch`"}, {"name": "threshold_mismatch", "expression": "`threshold_mismatch`"}, {"name": "mismatch_columns", "expression": "`mismatch_columns`"}, {"name": "schema_comparison", "expression": "`schema_comparison`"}, {"name": "executed_by", "expression": "`executed_by`"}, {"name": "start_ts", "expression": "`start_ts`"}, {"name": "end_ts", "expression": "`end_ts`"}], "disaggregated": true}}], "spec": {"version": 1, "widgetType": "table", "encodings": {"columns": [{"fieldName": "recon_id", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 0, "title": "recon_id", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": {"default": {"foregroundColor": null}, "rules": [{"if": {"column": "status", "fn": "=", "literal": "true"}, "value": {"foregroundColor": "#3BD973"}}, {"if": {"column": "status", "fn": "=", "literal": "false"}, "value": {"foregroundColor": "#E92828"}}]}, "displayName": "recon_id"}, {"fieldName": "source_type", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 1, "title": "source_type", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_type"}, {"fieldName": "report_type", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 2, "title": "report_type", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "report_type"}, {"fieldName": "source_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 5, "title": "source_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_table"}, {"fieldName": "target_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 8, "title": "target_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "target_table"}, {"fieldName": "status", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "boolean", "displayAs": "boolean", "visible": true, "order": 9, "title": "status", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "status"}, {"fieldName": "exception", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 10, "title": "exception", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "exception"}, {"fieldName": "missing_in_source", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 11, "title": "missing_in_source", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "missing_in_source"}, {"fieldName": "missing_in_target", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 12, "title": "missing_in_target", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "missing_in_target"}, {"fieldName": "absolute_mismatch", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 13, "title": "absolute_mismatch", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "absolute_mismatch"}, {"fieldName": "threshold_mismatch", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 14, "title": "threshold_mismatch", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "threshold_mismatch"}, {"fieldName": "mismatch_columns", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 15, "title": "mismatch_columns", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "mismatch_columns"}, {"fieldName": "schema_comparison", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "boolean", "displayAs": "boolean", "visible": true, "order": 16, "title": "schema_comparison", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "schema_comparison"}, {"fieldName": "executed_by", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 19, "title": "executed_by", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "executed_by"}, {"fieldName": "start_ts", "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "datetime", "displayAs": "datetime", "visible": true, "order": 20, "title": "start_ts", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "start_ts"}, {"fieldName": "end_ts", "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "datetime", "displayAs": "datetime", "visible": true, "order": 21, "title": "end_ts", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "end_ts"}]}, "invisibleColumns": [{"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_catalog", "type": "string", "displayAs": "string", "order": 3, "title": "source_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_schema", "type": "string", "displayAs": "string", "order": 4, "title": "source_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_catalog", "type": "string", "displayAs": "string", "order": 6, "title": "target_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_schema", "type": "string", "displayAs": "string", "order": 7, "title": "target_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_table_name", "type": "string", "displayAs": "string", "order": 17, "title": "source_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_table_name", "type": "string", "displayAs": "string", "order": 18, "title": "target_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}], "allowHTMLByDefault": false, "itemsPerPage": 100, "paginationSize": "default", "condensed": true, "withRowNumber": true, "frame": {"showDescription": true, "description": "Summary Table", "showTitle": false}}}, "position": {"x": 0, "y": 10, "width": 6, "height": 7}}, {"widget": {"name": "c27cb052", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id", "query": {"datasetName": "088adb8a", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id"}, {"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id"}, {"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Recon Id", "description": ""}}}, "position": {"x": 0, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "3ff15994", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", "query": {"datasetName": "aec31225", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table"}, {"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table"}, {"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Traget Table Name", "description": ""}}}, "position": {"x": 2, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "bec17005", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table", "query": {"datasetName": "088adb8a", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", "query": {"datasetName": "aec31225", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table"}, {"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table"}, {"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Source Table Name", "description": ""}}}, "position": {"x": 4, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "41d78bbf", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type", "query": {"datasetName": "088adb8a", "fields": [{"name": "source_type", "expression": "`source_type`"}, {"name": "source_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_type", "displayName": "source_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Source Type", "description": "Applied only on summary table"}}}, "position": {"x": 0, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "4bcc8273", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_type", "expression": "`recon_type`"}, {"name": "recon_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"fieldName": "recon_type", "displayName": "recon_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Category", "description": "Applied only on details table"}}}, "position": {"x": 3, "y": 27, "width": 3, "height": 1}}, {"widget": {"name": "b7d8efe7", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "absolute_mismatch", "expression": "`absolute_mismatch`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "absolute_mismatch", "scale": {"type": "quantitative"}, "displayName": "absolute_mismatch"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Mismatched Records", "showDescription": false}}}, "position": {"x": 0, "y": 37, "width": 3, "height": 6}}, {"widget": {"name": "13d8f169", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "missing_in_target", "expression": "`missing_in_target`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "missing_in_target", "scale": {"type": "quantitative"}, "displayName": "missing_in_target"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Missing in Source"}}}, "position": {"x": 3, "y": 43, "width": 3, "height": 6}}, {"widget": {"name": "1b4c4556", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "missing_in_source", "expression": "`missing_in_source`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "missing_in_source", "scale": {"type": "quantitative"}, "displayName": "missing_in_source"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Missing in Databricks"}}}, "position": {"x": 0, "y": 43, "width": 3, "height": 6}}, {"widget": {"name": "aedc94de", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "threshold_mismatch", "expression": "`threshold_mismatch`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "threshold_mismatch", "scale": {"type": "quantitative"}, "displayName": "threshold_mismatch"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}}, "frame": {"showTitle": true, "title": "Threshold Mismatches"}}}, "position": {"x": 3, "y": 37, "width": 3, "height": 6}}, {"widget": {"name": "7d60be10", "queries": [{"name": "main_query", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "source_column", "expression": "`source_column`"}, {"name": "source_datatype", "expression": "`source_datatype`"}, {"name": "databricks_column", "expression": "`databricks_column`"}, {"name": "databricks_datatype", "expression": "`databricks_datatype`"}, {"name": "is_valid", "expression": "`is_valid`"}], "disaggregated": true}}], "spec": {"version": 1, "widgetType": "table", "encodings": {"columns": [{"fieldName": "recon_id", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 0, "title": "recon_id", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": {"default": {"foregroundColor": null}, "rules": [{"if": {"column": "is_valid", "fn": "=", "literal": "false"}, "value": {"foregroundColor": "#E92828"}}, {"if": {"column": "is_valid", "fn": "=", "literal": "true"}, "value": {"foregroundColor": "#3BD973"}}]}, "displayName": "recon_id"}, {"fieldName": "source_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 4, "title": "source_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_table"}, {"fieldName": "target_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 8, "title": "target_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "target_table"}, {"fieldName": "source_column", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 9, "title": "source_column", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_column"}, {"fieldName": "source_datatype", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 10, "title": "source_datatype", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_datatype"}, {"fieldName": "databricks_column", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 11, "title": "databricks_column", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "databricks_column"}, {"fieldName": "databricks_datatype", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 12, "title": "databricks_datatype", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "databricks_datatype"}, {"fieldName": "is_valid", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 13, "title": "is_valid", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "is_valid"}]}, "invisibleColumns": [{"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_table_name", "type": "string", "displayAs": "string", "order": 1, "title": "source_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_catalog", "type": "string", "displayAs": "string", "order": 2, "title": "source_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_schema", "type": "string", "displayAs": "string", "order": 3, "title": "source_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_catalog", "type": "string", "displayAs": "string", "order": 5, "title": "target_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_schema", "type": "string", "displayAs": "string", "order": 6, "title": "target_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_table_name", "type": "string", "displayAs": "string", "order": 7, "title": "target_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}], "allowHTMLByDefault": false, "itemsPerPage": 25, "paginationSize": "default", "condensed": true, "withRowNumber": false, "frame": {"showTitle": true, "title": "Schema Details"}}}, "position": {"x": 0, "y": 19, "width": 6, "height": 6}}, {"widget": {"name": "15be1fef", "queries": [{"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_catalog", "query": {"datasetName": "088adb8a", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_catalog", "query": {"datasetName": "07f3d7d5", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_catalog", "query": {"datasetName": "aec31225", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_catalog"}, {"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_catalog"}, {"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_catalog"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Catalog", "description": "\u2019remorph' is the default catalog for the metrics table; specify a different catalog name if the metrics are stored elsewhere."}}}, "position": {"x": 0, "y": 2, "width": 3, "height": 2}}, {"widget": {"name": "d1e11541", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type", "query": {"datasetName": "088adb8a", "fields": [{"name": "report_type", "expression": "`report_type`"}, {"name": "report_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "report_type", "displayName": "report_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Report", "description": "Applied only on summary table"}}}, "position": {"x": 2, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "8e9a7f6d", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts", "query": {"datasetName": "088adb8a", "fields": [{"name": "start_ts", "expression": "`start_ts`"}, {"name": "start_ts_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-date-range-picker", "encodings": {"fields": [{"fieldName": "start_ts", "displayName": "start_ts", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "started at", "description": "Applied on all tables"}}}, "position": {"x": 0, "y": 8, "width": 6, "height": 2}}, {"widget": {"name": "12568355", "textbox_spec": "# Main Reconciliation Table\n### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records."}, "position": {"x": 0, "y": 0, "width": 6, "height": 2}}, {"widget": {"name": "72f3dc17", "textbox_spec": "# Drill Down\n### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues."}, "position": {"x": 0, "y": 25, "width": 6, "height": 2}}, {"widget": {"name": "67d6c36d", "textbox_spec": "# Schema Comparison Details\n### This table provides a detailed view of schema mismatches."}, "position": {"x": 0, "y": 17, "width": 6, "height": 2}}, {"widget": {"name": "fca3d649", "textbox_spec": "# Visualization of Missing and Mismatched Records"}, "position": {"x": 0, "y": 35, "width": 6, "height": 2}}, {"widget": {"name": "71206c9b", "queries": [{"name": "main_query", "query": {"datasetName": "aec31225", "fields": [{"name": "value", "expression": "`value`"}, {"name": "key", "expression": "`key`"}, {"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "recon_type", "expression": "`recon_type`"}, {"name": "rn", "expression": "`rn`"}], "disaggregated": false, "orders": [{"direction": "ASC", "expression": "`recon_id`"}, {"direction": "ASC", "expression": "`source_table`"}, {"direction": "ASC", "expression": "`target_table`"}, {"direction": "ASC", "expression": "`recon_type`"}, {"direction": "ASC", "expression": "`rn`"}, {"direction": "ASC", "expression": "`key`"}]}}], "spec": {"version": 3, "widgetType": "pivot", "encodings": {"rows": [{"fieldName": "recon_id", "displayName": "recon_id"}, {"fieldName": "source_table", "displayName": "source_table"}, {"fieldName": "target_table", "displayName": "target_table"}, {"fieldName": "recon_type", "displayName": "recon_type"}, {"fieldName": "rn", "displayName": "rn"}], "columns": [{"fieldName": "key", "displayName": "key"}], "cell": {"fieldName": "value", "cellType": "text", "displayName": "value"}}}}, "position": {"x": 0, "y": 29, "width": 6, "height": 6}}, {"widget": {"name": "14576173", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", "query": {"datasetName": "aec31225", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]}, "frame": {"showDescription": true, "showTitle": true, "title": "Target Table Name", "description": "Applied only on details table"}}}, "position": {"x": 0, "y": 28, "width": 3, "height": 1}}, {"widget": {"name": "81426d8f", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", "query": {"datasetName": "aec31225", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Source Table Name", "description": "Applied only on details table"}}}, "position": {"x": 3, "y": 28, "width": 3, "height": 1}}, {"widget": {"name": "6f02d764", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Recon Id", "description": "Applied only on details table"}}}, "position": {"x": 0, "y": 27, "width": 3, "height": 1}}, {"widget": {"name": "832b10ed", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by", "query": {"datasetName": "088adb8a", "fields": [{"name": "executed_by", "expression": "`executed_by`"}, {"name": "executed_by_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "executed_by", "displayName": "executed_by", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Executed by", "description": "Applied only on summary table"}}}, "position": {"x": 4, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "27298c01", "queries": [{"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_schema", "query": {"datasetName": "088adb8a", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_schema", "query": {"datasetName": "07f3d7d5", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_schema", "query": {"datasetName": "aec31225", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_schema"}, {"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_schema"}, {"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_schema"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Schema", "description": "\u2019reconcile\u2019 is the default schema for the metrics table; specify a different schema name if the metrics are stored elsewhere."}}}, "position": {"x": 3, "y": 2, "width": 3, "height": 2}}]}]} \ No newline at end of file diff --git a/tests/resources/Remorph-Reconciliation.lvdash.json b/tests/resources/Remorph-Reconciliation.lvdash.json deleted file mode 100644 index fb6093e77..000000000 --- a/tests/resources/Remorph-Reconciliation.lvdash.json +++ /dev/null @@ -1 +0,0 @@ -{"datasets": [{"name": "088adb8a", "displayName": "recon_main", "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.exception_message as exception,\nmetrics.recon_metrics.row_comparison.missing_in_source as missing_in_source,\nmetrics.recon_metrics.row_comparison.missing_in_target as missing_in_target,\nmetrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch,\nmetrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch,\nmetrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns,\nmetrics.recon_metrics.schema_comparison as schema_comparison,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile"}]}}}]}, {"name": "07f3d7d5", "displayName": "recon_details_schema", "query": "with tmp as (select \nrecon_table_id,\ninserted_ts,\nexplode(data) as schema_data\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type='schema'\n) \nselect \nmain.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nschema_data['source_column'] as source_column,\nschema_data['source_datatype'] as source_datatype,\nschema_data['databricks_column'] as databricks_column,\nschema_data['databricks_datatype'] as databricks_datatype,\nschema_data['is_valid'] as is_valid\nfrom \nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \ntmp\non main.recon_table_id = tmp.recon_table_id\norder by tmp.inserted_ts desc, main.recon_id, main.target_table\n", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile"}]}}}]}, {"name": "aec31225", "displayName": "recon_details_pivot", "query": "with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, \nrow_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type != 'schema')\nselect main.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nrecon_type, key, value, rn\nfrom tmp\ninner join\nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\non main.recon_table_id = tmp.recon_table_id\nlateral view explode(data) exploded_data AS key, value\n", "parameters": [{"displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "remorph"}]}}}, {"displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": {"values": {"dataType": "STRING", "values": [{"value": "reconcile"}]}}}]}], "pages": [{"name": "1ac59528", "displayName": "New Page", "layout": [{"widget": {"name": "b3208d80", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_type", "expression": "`source_type`"}, {"name": "report_type", "expression": "`report_type`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "status", "expression": "`status`"}, {"name": "exception", "expression": "`exception`"}, {"name": "missing_in_source", "expression": "`missing_in_source`"}, {"name": "missing_in_target", "expression": "`missing_in_target`"}, {"name": "absolute_mismatch", "expression": "`absolute_mismatch`"}, {"name": "threshold_mismatch", "expression": "`threshold_mismatch`"}, {"name": "mismatch_columns", "expression": "`mismatch_columns`"}, {"name": "schema_comparison", "expression": "`schema_comparison`"}, {"name": "executed_by", "expression": "`executed_by`"}, {"name": "start_ts", "expression": "`start_ts`"}, {"name": "end_ts", "expression": "`end_ts`"}], "disaggregated": true}}], "spec": {"version": 1, "widgetType": "table", "encodings": {"columns": [{"fieldName": "recon_id", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 0, "title": "recon_id", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": {"default": {"foregroundColor": null}, "rules": [{"if": {"column": "status", "fn": "=", "literal": "true"}, "value": {"foregroundColor": "#3BD973"}}, {"if": {"column": "status", "fn": "=", "literal": "false"}, "value": {"foregroundColor": "#E92828"}}]}, "displayName": "recon_id"}, {"fieldName": "source_type", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 1, "title": "source_type", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_type"}, {"fieldName": "report_type", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 2, "title": "report_type", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "report_type"}, {"fieldName": "source_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 5, "title": "source_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_table"}, {"fieldName": "target_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 8, "title": "target_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "target_table"}, {"fieldName": "status", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "boolean", "displayAs": "boolean", "visible": true, "order": 9, "title": "status", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "status"}, {"fieldName": "exception", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 10, "title": "exception", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "exception"}, {"fieldName": "missing_in_source", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 11, "title": "missing_in_source", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "missing_in_source"}, {"fieldName": "missing_in_target", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 12, "title": "missing_in_target", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "missing_in_target"}, {"fieldName": "absolute_mismatch", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 13, "title": "absolute_mismatch", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "absolute_mismatch"}, {"fieldName": "threshold_mismatch", "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "integer", "displayAs": "number", "visible": true, "order": 14, "title": "threshold_mismatch", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "threshold_mismatch"}, {"fieldName": "mismatch_columns", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 15, "title": "mismatch_columns", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "mismatch_columns"}, {"fieldName": "schema_comparison", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "boolean", "displayAs": "boolean", "visible": true, "order": 16, "title": "schema_comparison", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "schema_comparison"}, {"fieldName": "executed_by", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 19, "title": "executed_by", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "executed_by"}, {"fieldName": "start_ts", "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "datetime", "displayAs": "datetime", "visible": true, "order": 20, "title": "start_ts", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "start_ts"}, {"fieldName": "end_ts", "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "datetime", "displayAs": "datetime", "visible": true, "order": 21, "title": "end_ts", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "end_ts"}]}, "invisibleColumns": [{"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_catalog", "type": "string", "displayAs": "string", "order": 3, "title": "source_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_schema", "type": "string", "displayAs": "string", "order": 4, "title": "source_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_catalog", "type": "string", "displayAs": "string", "order": 6, "title": "target_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_schema", "type": "string", "displayAs": "string", "order": 7, "title": "target_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_table_name", "type": "string", "displayAs": "string", "order": 17, "title": "source_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_table_name", "type": "string", "displayAs": "string", "order": 18, "title": "target_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}], "allowHTMLByDefault": false, "itemsPerPage": 100, "paginationSize": "default", "condensed": true, "withRowNumber": true, "frame": {"showDescription": true, "description": "Summary Table", "showTitle": false}}}, "position": {"x": 0, "y": 10, "width": 6, "height": 7}}, {"widget": {"name": "c27cb052", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id", "query": {"datasetName": "088adb8a", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id"}, {"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id"}, {"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Recon Id", "description": ""}}}, "position": {"x": 0, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "3ff15994", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", "query": {"datasetName": "aec31225", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table"}, {"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table"}, {"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Traget Table Name", "description": ""}}}, "position": {"x": 2, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "bec17005", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table", "query": {"datasetName": "088adb8a", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}, {"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", "query": {"datasetName": "aec31225", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table"}, {"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table"}, {"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]}, "frame": {"showTitle": true, "showDescription": false, "title": "Source Table Name", "description": ""}}}, "position": {"x": 4, "y": 4, "width": 2, "height": 2}}, {"widget": {"name": "41d78bbf", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type", "query": {"datasetName": "088adb8a", "fields": [{"name": "source_type", "expression": "`source_type`"}, {"name": "source_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_type", "displayName": "source_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Source Type", "description": "Applied only on summary table"}}}, "position": {"x": 0, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "4bcc8273", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_type", "expression": "`recon_type`"}, {"name": "recon_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"fieldName": "recon_type", "displayName": "recon_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Category", "description": "Applied only on details table"}}}, "position": {"x": 3, "y": 27, "width": 3, "height": 1}}, {"widget": {"name": "b7d8efe7", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "absolute_mismatch", "expression": "`absolute_mismatch`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "absolute_mismatch", "scale": {"type": "quantitative"}, "displayName": "absolute_mismatch"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Mismatched Records", "showDescription": false}}}, "position": {"x": 0, "y": 37, "width": 3, "height": 6}}, {"widget": {"name": "13d8f169", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "missing_in_target", "expression": "`missing_in_target`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "missing_in_target", "scale": {"type": "quantitative"}, "displayName": "missing_in_target"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Missing in Source"}}}, "position": {"x": 3, "y": 43, "width": 3, "height": 6}}, {"widget": {"name": "1b4c4556", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "missing_in_source", "expression": "`missing_in_source`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "missing_in_source", "scale": {"type": "quantitative"}, "displayName": "missing_in_source"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}, "label": {"show": false}}, "frame": {"showTitle": true, "title": "Missing in Databricks"}}}, "position": {"x": 0, "y": 43, "width": 3, "height": 6}}, {"widget": {"name": "aedc94de", "queries": [{"name": "main_query", "query": {"datasetName": "088adb8a", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "hourly(start_ts)", "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)"}, {"name": "threshold_mismatch", "expression": "`threshold_mismatch`"}], "disaggregated": true}}], "spec": {"version": 3, "widgetType": "area", "encodings": {"x": {"fieldName": "hourly(start_ts)", "scale": {"type": "temporal"}, "displayName": "start_ts"}, "y": {"fieldName": "threshold_mismatch", "scale": {"type": "quantitative"}, "displayName": "threshold_mismatch"}, "color": {"fieldName": "target_table", "scale": {"type": "categorical"}, "displayName": "target_table"}}, "frame": {"showTitle": true, "title": "Threshold Mismatches"}}}, "position": {"x": 3, "y": 37, "width": 3, "height": 6}}, {"widget": {"name": "7d60be10", "queries": [{"name": "main_query", "query": {"datasetName": "07f3d7d5", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "source_column", "expression": "`source_column`"}, {"name": "source_datatype", "expression": "`source_datatype`"}, {"name": "databricks_column", "expression": "`databricks_column`"}, {"name": "databricks_datatype", "expression": "`databricks_datatype`"}, {"name": "is_valid", "expression": "`is_valid`"}], "disaggregated": true}}], "spec": {"version": 1, "widgetType": "table", "encodings": {"columns": [{"fieldName": "recon_id", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 0, "title": "recon_id", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": {"default": {"foregroundColor": null}, "rules": [{"if": {"column": "is_valid", "fn": "=", "literal": "false"}, "value": {"foregroundColor": "#E92828"}}, {"if": {"column": "is_valid", "fn": "=", "literal": "true"}, "value": {"foregroundColor": "#3BD973"}}]}, "displayName": "recon_id"}, {"fieldName": "source_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 4, "title": "source_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_table"}, {"fieldName": "target_table", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 8, "title": "target_table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "target_table"}, {"fieldName": "source_column", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 9, "title": "source_column", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_column"}, {"fieldName": "source_datatype", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 10, "title": "source_datatype", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "source_datatype"}, {"fieldName": "databricks_column", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 11, "title": "databricks_column", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "databricks_column"}, {"fieldName": "databricks_datatype", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 12, "title": "databricks_datatype", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "databricks_datatype"}, {"fieldName": "is_valid", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 13, "title": "is_valid", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "displayName": "is_valid"}]}, "invisibleColumns": [{"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_table_name", "type": "string", "displayAs": "string", "order": 1, "title": "source_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_catalog", "type": "string", "displayAs": "string", "order": 2, "title": "source_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "source_schema", "type": "string", "displayAs": "string", "order": 3, "title": "source_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_catalog", "type": "string", "displayAs": "string", "order": 5, "title": "target_catalog", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_schema", "type": "string", "displayAs": "string", "order": 6, "title": "target_schema", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}, {"booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "target_table_name", "type": "string", "displayAs": "string", "order": 7, "title": "target_table_name", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false}], "allowHTMLByDefault": false, "itemsPerPage": 25, "paginationSize": "default", "condensed": true, "withRowNumber": false, "frame": {"showTitle": true, "title": "Schema Details"}}}, "position": {"x": 0, "y": 19, "width": 6, "height": 6}}, {"widget": {"name": "15be1fef", "queries": [{"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_catalog", "query": {"datasetName": "088adb8a", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_catalog", "query": {"datasetName": "07f3d7d5", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_catalog", "query": {"datasetName": "aec31225", "parameters": [{"name": "catalog", "keyword": "catalog"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_catalog"}, {"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_catalog"}, {"parameterName": "catalog", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_catalog"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Catalog", "description": "\u2019remorph' is the default catalog for the metrics table; specify a different catalog name if the metrics are stored elsewhere."}}}, "position": {"x": 0, "y": 2, "width": 3, "height": 2}}, {"widget": {"name": "d1e11541", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type", "query": {"datasetName": "088adb8a", "fields": [{"name": "report_type", "expression": "`report_type`"}, {"name": "report_type_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "report_type", "displayName": "report_type", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Report", "description": "Applied only on summary table"}}}, "position": {"x": 2, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "8e9a7f6d", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts", "query": {"datasetName": "088adb8a", "fields": [{"name": "start_ts", "expression": "`start_ts`"}, {"name": "start_ts_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-date-range-picker", "encodings": {"fields": [{"fieldName": "start_ts", "displayName": "start_ts", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "started at", "description": "Applied on all tables"}}}, "position": {"x": 0, "y": 8, "width": 6, "height": 2}}, {"widget": {"name": "12568355", "textbox_spec": "# Main Reconciliation Table\n### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records."}, "position": {"x": 0, "y": 0, "width": 6, "height": 2}}, {"widget": {"name": "72f3dc17", "textbox_spec": "# Drill Down\n### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues."}, "position": {"x": 0, "y": 25, "width": 6, "height": 2}}, {"widget": {"name": "67d6c36d", "textbox_spec": "# Schema Comparison Details\n### This table provides a detailed view of schema mismatches."}, "position": {"x": 0, "y": 17, "width": 6, "height": 2}}, {"widget": {"name": "fca3d649", "textbox_spec": "# Visualization of Missing and Mismatched Records"}, "position": {"x": 0, "y": 35, "width": 6, "height": 2}}, {"widget": {"name": "71206c9b", "queries": [{"name": "main_query", "query": {"datasetName": "aec31225", "fields": [{"name": "value", "expression": "`value`"}, {"name": "key", "expression": "`key`"}, {"name": "recon_id", "expression": "`recon_id`"}, {"name": "source_table", "expression": "`source_table`"}, {"name": "target_table", "expression": "`target_table`"}, {"name": "recon_type", "expression": "`recon_type`"}, {"name": "rn", "expression": "`rn`"}], "disaggregated": false, "orders": [{"direction": "ASC", "expression": "`recon_id`"}, {"direction": "ASC", "expression": "`source_table`"}, {"direction": "ASC", "expression": "`target_table`"}, {"direction": "ASC", "expression": "`recon_type`"}, {"direction": "ASC", "expression": "`rn`"}, {"direction": "ASC", "expression": "`key`"}]}}], "spec": {"version": 3, "widgetType": "pivot", "encodings": {"rows": [{"fieldName": "recon_id", "displayName": "recon_id"}, {"fieldName": "source_table", "displayName": "source_table"}, {"fieldName": "target_table", "displayName": "target_table"}, {"fieldName": "recon_type", "displayName": "recon_type"}, {"fieldName": "rn", "displayName": "rn"}], "columns": [{"fieldName": "key", "displayName": "key"}], "cell": {"fieldName": "value", "cellType": "text", "displayName": "value"}}}}, "position": {"x": 0, "y": 29, "width": 6, "height": 6}}, {"widget": {"name": "14576173", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", "query": {"datasetName": "aec31225", "fields": [{"name": "target_table", "expression": "`target_table`"}, {"name": "target_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "target_table", "displayName": "target_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table"}]}, "frame": {"showDescription": true, "showTitle": true, "title": "Target Table Name", "description": "Applied only on details table"}}}, "position": {"x": 0, "y": 28, "width": 3, "height": 1}}, {"widget": {"name": "81426d8f", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", "query": {"datasetName": "aec31225", "fields": [{"name": "source_table", "expression": "`source_table`"}, {"name": "source_table_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "source_table", "displayName": "source_table", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Source Table Name", "description": "Applied only on details table"}}}, "position": {"x": 3, "y": 28, "width": 3, "height": 1}}, {"widget": {"name": "6f02d764", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", "query": {"datasetName": "aec31225", "fields": [{"name": "recon_id", "expression": "`recon_id`"}, {"name": "recon_id_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "recon_id", "displayName": "recon_id", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Recon Id", "description": "Applied only on details table"}}}, "position": {"x": 0, "y": 27, "width": 3, "height": 1}}, {"widget": {"name": "832b10ed", "queries": [{"name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by", "query": {"datasetName": "088adb8a", "fields": [{"name": "executed_by", "expression": "`executed_by`"}, {"name": "executed_by_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-multi-select", "encodings": {"fields": [{"fieldName": "executed_by", "displayName": "executed_by", "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Executed by", "description": "Applied only on summary table"}}}, "position": {"x": 4, "y": 6, "width": 2, "height": 2}}, {"widget": {"name": "27298c01", "queries": [{"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_schema", "query": {"datasetName": "088adb8a", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_schema", "query": {"datasetName": "07f3d7d5", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}, {"name": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_schema", "query": {"datasetName": "aec31225", "parameters": [{"name": "schema", "keyword": "schema"}], "disaggregated": false}}], "spec": {"version": 2, "widgetType": "filter-single-select", "encodings": {"fields": [{"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_schema"}, {"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_schema"}, {"parameterName": "schema", "queryName": "parameter_dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_schema"}]}, "frame": {"showTitle": true, "showDescription": true, "title": "Schema", "description": "\u2019reconcile\u2019 is the default schema for the metrics table; specify a different schema name if the metrics are stored elsewhere."}}}, "position": {"x": 3, "y": 2, "width": 3, "height": 2}}]}]} \ No newline at end of file diff --git a/tests/resources/Test_Dashboard_No_Param.lvdash.json b/tests/resources/Test_Dashboard_No_Param.lvdash.json deleted file mode 100644 index 043202049..000000000 --- a/tests/resources/Test_Dashboard_No_Param.lvdash.json +++ /dev/null @@ -1 +0,0 @@ -{"datasets":[{"name":"91a1460a","displayName":"order_status","query":"SELECT * FROM dbsql_tpch_demo.tpch.order_status"}],"pages":[{"name":"7fdcfeb6","displayName":"New Page","layout":[{"widget":{"name":"ba98213a","queries":[{"name":"main_query","query":{"datasetName":"91a1460a","disaggregated":true}}],"spec":{"version":3,"widgetType":"bar","encodings":{}}},"position":{"x":0,"y":0,"width":3,"height":6}}]}]} \ No newline at end of file diff --git a/tests/resources/dashboards/queries/00_description.md b/tests/resources/dashboards/queries/00_description.md new file mode 100644 index 000000000..ffbb93f01 --- /dev/null +++ b/tests/resources/dashboards/queries/00_description.md @@ -0,0 +1 @@ +#Reconcile Dashboard Queries Test \ No newline at end of file diff --git a/tests/resources/dashboards/queries/01_queries.sql b/tests/resources/dashboards/queries/01_queries.sql new file mode 100644 index 000000000..01842b489 --- /dev/null +++ b/tests/resources/dashboards/queries/01_queries.sql @@ -0,0 +1,31 @@ +select +main.recon_id, +main.source_type, +main.report_type, +main.source_table.`catalog` as source_catalog, +main.source_table.`schema` as source_schema, +main.source_table.table_name as source_table_name, +CASE + WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) + END AS source_table, +main.target_table.`catalog` as target_catalog, +main.target_table.`schema` as target_schema, +main.target_table.table_name as target_table_name, +CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, +metrics.run_metrics.status as status, +metrics.run_metrics.exception_message as exception, +metrics.recon_metrics.row_comparison.missing_in_source as missing_in_source, +metrics.recon_metrics.row_comparison.missing_in_target as missing_in_target, +metrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch, +metrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch, +metrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns, +metrics.recon_metrics.schema_comparison as schema_comparison, +metrics.run_metrics.run_by_user as executed_by, +main.start_ts as start_ts, +main.end_ts as end_ts +from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main +inner join +IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics +on main.recon_table_id = metrics.recon_table_id +order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file diff --git a/tests/resources/dashboards/queries/dashboard.yml b/tests/resources/dashboards/queries/dashboard.yml new file mode 100644 index 000000000..6028c8ded --- /dev/null +++ b/tests/resources/dashboards/queries/dashboard.yml @@ -0,0 +1 @@ +display_name: "Reconciliation Metrics Test" \ No newline at end of file diff --git a/tests/unit/deployment/test_dashboard.py b/tests/unit/deployment/test_dashboard.py index 8eb52eff0..dce4d25e4 100644 --- a/tests/unit/deployment/test_dashboard.py +++ b/tests/unit/deployment/test_dashboard.py @@ -12,7 +12,8 @@ def test_deploy_new_dashboard(): - dashboard_file = Path(__file__).parent / Path("../../resources/Remorph-Reconciliation.lvdash.json") + dashboard_file = Path(__file__).parent / Path( + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") ws = create_autospec(WorkspaceClient) dashboard = Dashboard( dashboard_id="9c1fbf4ad3449be67d6cb64c8acc730b", @@ -30,9 +31,10 @@ def test_deploy_new_dashboard(): def test_deploy_new_dashboard_with_params(): - dashboard_file = Path(__file__).parent / Path("../../resources/Remorph-Reconciliation.lvdash.json") + dashboard_file = Path(__file__).parent / Path( + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") substituted_dashboard_file = Path(__file__).parent / Path( - '../../resources/Remorph-Reconciliation-Substituted.lvdash.json' + '../../resources/dashboards/queries/Remorph-Reconciliation-Substituted.lvdash.json' ) ws = create_autospec(WorkspaceClient) @@ -53,7 +55,8 @@ def test_deploy_new_dashboard_with_params(): def test_deploy_new_parameterless_dashboard_with_user_params(): - dashboard_file = Path(__file__).parent / Path("../../resources/Test_Dashboard_No_Param.lvdash.json") + dashboard_file = Path(__file__).parent / Path( + "../../resources/dashboards/queries/Test_Dashboard_No_Param.lvdash.json") ws = create_autospec(WorkspaceClient) dashboard = Dashboard( dashboard_id="8c1fbf4ad3449be67d6cb64c8acc730b", @@ -70,7 +73,8 @@ def test_deploy_new_parameterless_dashboard_with_user_params(): def test_deploy_existing_dashboard(): - dashboard_file = Path(__file__).parent / Path("../../resources/Remorph-Reconciliation.lvdash.json") + dashboard_file = Path(__file__).parent / Path( + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( @@ -89,7 +93,8 @@ def test_deploy_existing_dashboard(): def test_deploy_missing_dashboard(): - dashboard_file = Path(__file__).parent / Path("../../resources/Remorph-Reconciliation.lvdash.json") + dashboard_file = Path(__file__).parent / Path( + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( From bcc21b58b144878ebcf93088be9c4f2fdac6240a Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Wed, 7 Aug 2024 12:50:26 +0530 Subject: [PATCH 06/10] intermediate commit --- .../labs/remorph/deployment/dashboard.py | 9 ++- .../reconcile/dashboards/0_0_recon_main.md | 2 + .../reconcile/dashboards/0_1_recon_main.sql | 81 ++++++++++++------- .../dashboards/1_0_recon_details_schema.sql | 31 ------- .../1_0_schema_comparison_details.md | 2 + .../dashboards/1_1_recon_details_pivot.sql | 22 ----- .../dashboards/1_1_recon_details_schema.sql | 52 ++++++++++++ .../dashboards/2_0_details_drill_down.md | 2 + .../dashboards/2_1_failed_table_list.sql | 26 ------ .../dashboards/2_1_recon_details_pivot.sql | 55 +++++++++++++ .../dashboards/2_2_success_table_list.sql | 20 ----- .../dashboards/3_0_data_validation_report.md | 2 + .../dashboards/3_1_failed_table_list.sql | 45 +++++++++++ .../dashboards/3_2_success_table_list.sql | 40 +++++++++ tests/unit/deployment/test_dashboard.py | 15 ++-- 15 files changed, 267 insertions(+), 137 deletions(-) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql diff --git a/src/databricks/labs/remorph/deployment/dashboard.py b/src/databricks/labs/remorph/deployment/dashboard.py index 53d32a1a1..b3be62af1 100644 --- a/src/databricks/labs/remorph/deployment/dashboard.py +++ b/src/databricks/labs/remorph/deployment/dashboard.py @@ -64,9 +64,14 @@ def deploy( pass metadata = DashboardMetadata.from_path(folder).replace_database( - database=f"hive_metastore.{config.schema}", - database_to_replace="inventory", + catalog=config.catalog, + catalog_to_replace="remorph_catalog", + database=config.schema, + database_to_replace="remorph_schema", ) + print("**********************") + print(metadata) + print("***********************") metadata.display_name = self._name_with_prefix(metadata.display_name) diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md new file mode 100644 index 000000000..e60f5369c --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md @@ -0,0 +1,2 @@ +# Main Reconciliation Table +## This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql index 01842b489..9bce7a142 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql @@ -1,31 +1,50 @@ -select -main.recon_id, -main.source_type, -main.report_type, -main.source_table.`catalog` as source_catalog, -main.source_table.`schema` as source_schema, -main.source_table.table_name as source_table_name, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -main.target_table.`catalog` as target_catalog, -main.target_table.`schema` as target_schema, -main.target_table.table_name as target_table_name, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -metrics.run_metrics.status as status, -metrics.run_metrics.exception_message as exception, -metrics.recon_metrics.row_comparison.missing_in_source as missing_in_source, -metrics.recon_metrics.row_comparison.missing_in_target as missing_in_target, -metrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch, -metrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch, -metrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns, -metrics.recon_metrics.schema_comparison as schema_comparison, -metrics.run_metrics.run_by_user as executed_by, -main.start_ts as start_ts, -main.end_ts as end_ts -from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -inner join -IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics -on main.recon_table_id = metrics.recon_table_id -order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file +/* --filter recon_id target_table_name source_table_name source_type report_type executed_by */ +SELECT + main.recon_id, + main.source_type, + main.report_type, + main.source_table.`CATALOG` AS source_catalog, + main.source_table.`SCHEMA` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( + main.source_table.catalog, + '.', + main.source_table.schema, + '.', + main.source_table.table_name + ) + ELSE CONCAT( + main.source_table.schema, + '.', + main.source_table.table_name + ) + END AS source_table, + main.target_table.`CATALOG` AS target_catalog, + main.target_table.`SCHEMA` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT( + main.target_table.catalog, + '.', + main.target_table.schema, + '.', + main.target_table.table_name + ) AS target_table, + metrics.run_metrics.status AS status, + metrics.run_metrics.exception_message AS exception, + metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source, + metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target, + metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch, + metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch, + metrics.recon_metrics.column_comparison.mismatch_columns AS mismatch_columns, + metrics.recon_metrics.schema_comparison AS schema_comparison, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts +FROM + remorph_catalog.remorph_schema.main AS main + INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql deleted file mode 100644 index 804be609f..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_recon_details_schema.sql +++ /dev/null @@ -1,31 +0,0 @@ -with tmp as (select -recon_table_id, -inserted_ts, -explode(data) as schema_data -from IDENTIFIER(:catalog || '.' || :schema || '.details' ) -where recon_type='schema' -) -select -main.recon_id, -main.source_table.`catalog` as source_catalog, -main.source_table.`schema` as source_schema, -main.source_table.table_name as source_table_name, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -main.target_table.`catalog` as target_catalog, -main.target_table.`schema` as target_schema, -main.target_table.table_name as target_table_name, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -schema_data['source_column'] as source_column, -schema_data['source_datatype'] as source_datatype, -schema_data['databricks_column'] as databricks_column, -schema_data['databricks_datatype'] as databricks_datatype, -schema_data['is_valid'] as is_valid -from -IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -inner join -tmp -on main.recon_table_id = tmp.recon_table_id -order by tmp.inserted_ts desc, main.recon_id, main.target_table diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md new file mode 100644 index 000000000..b5496efdf --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md @@ -0,0 +1,2 @@ +# Schema Comparison Details +## This table provides a detailed view of schema mismatches. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql deleted file mode 100644 index 430f32489..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_pivot.sql +++ /dev/null @@ -1,22 +0,0 @@ -with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, -row_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn -from IDENTIFIER(:catalog || '.' || :schema || '.details' ) -where recon_type != 'schema') -select main.recon_id, -main.source_table.`catalog` as source_catalog, -main.source_table.`schema` as source_schema, -main.source_table.table_name as source_table_name, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -main.target_table.`catalog` as target_catalog, -main.target_table.`schema` as target_schema, -main.target_table.table_name as target_table_name, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -recon_type, key, value, rn -from tmp -inner join -IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -on main.recon_table_id = tmp.recon_table_id -lateral view explode(data) exploded_data AS key, value diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql new file mode 100644 index 000000000..64eef74fb --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql @@ -0,0 +1,52 @@ +/* --filter recon_id target_table_name source_table_name */ +WITH tmp AS ( + SELECT + recon_table_id, + inserted_ts, + EXPLODE(data) AS schema_data + FROM + remorph_catalog.remorph_schema.details + WHERE + recon_type = 'SCHEMA' +) +SELECT + main.recon_id, + main.source_table.`CATALOG` AS source_catalog, + main.source_table.`SCHEMA` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( + main.source_table.catalog, + '.', + main.source_table.schema, + '.', + main.source_table.table_name + ) + ELSE CONCAT( + main.source_table.schema, + '.', + main.source_table.table_name + ) + END AS source_table, + main.target_table.`CATALOG` AS target_catalog, + main.target_table.`SCHEMA` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT( + main.target_table.catalog, + '.', + main.target_table.schema, + '.', + main.target_table.table_name + ) AS target_table, + schema_data ['SOURCE_COLUMN'] AS source_column, + schema_data ['SOURCE_DATATYPE'] AS source_datatype, + schema_data ['DATABRICKS_COLUMN'] AS databricks_column, + schema_data ['DATABRICKS_DATATYPE'] AS databricks_datatype, + schema_data ['IS_VALID'] AS is_valid +FROM + remorph_catalog.remorph_schema.main AS main + INNER JOIN tmp ON main.recon_table_id = tmp.recon_table_id +ORDER BY + tmp.inserted_ts DESC, + main.recon_id, + main.target_table \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md new file mode 100644 index 000000000..67c5e3b2d --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md @@ -0,0 +1,2 @@ +# Drill Down +## The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql deleted file mode 100644 index a8dfea4f7..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_failed_table_list.sql +++ /dev/null @@ -1,26 +0,0 @@ -select -main.recon_id, -main.source_type, -main.report_type, -main.source_table.`catalog` as source_catalog, -main.source_table.`schema` as source_schema, -main.source_table.table_name as source_table_name, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -main.target_table.`catalog` as target_catalog, -main.target_table.`schema` as target_schema, -main.target_table.table_name as target_table_name, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -metrics.run_metrics.status as status, -metrics.run_metrics.run_by_user as executed_by, -main.start_ts as start_ts, -main.end_ts as end_ts, -date(main.start_ts) as start_date -from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -inner join -IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics -on main.recon_table_id = metrics.recon_table_id -where metrics.run_metrics.status = false -order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql new file mode 100644 index 000000000..51e6591db --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql @@ -0,0 +1,55 @@ +/* --filter start_date */ +WITH tmp AS ( + SELECT + recon_table_id, + inserted_ts, + recon_type, + EXPLODE(data) AS data, + ROW_NUMBER() OVER ( + PARTITION BY recon_table_id, + recon_type + ORDER BY + recon_table_id + ) AS rn + FROM + remorph_catalog.remorph_schema.details + WHERE + recon_type <> 'SCHEMA' +) +SELECT + main.recon_id, + main.source_table.`CATALOG` AS source_catalog, + main.source_table.`SCHEMA` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( + main.source_table.catalog, + '.', + main.source_table.schema, + '.', + main.source_table.table_name + ) + ELSE CONCAT( + main.source_table.schema, + '.', + main.source_table.table_name + ) + END AS source_table, + main.target_table.`CATALOG` AS target_catalog, + main.target_table.`SCHEMA` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT( + main.target_table.catalog, + '.', + main.target_table.schema, + '.', + main.target_table.table_name + ) AS target_table, + recon_type, + key, + value, + rn +FROM + tmp + INNER JOIN remorph_catalog.remorph_schema.main AS main ON main.recon_table_id = tmp.recon_table_id LATERAL VIEW EXPLODE(data) EXPLODED_DATA AS key, + value \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql deleted file mode 100644 index 898076bdf..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_2_success_table_list.sql +++ /dev/null @@ -1,20 +0,0 @@ -select -main.recon_id, -main.source_type, -main.report_type, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -metrics.run_metrics.status as status, -metrics.run_metrics.run_by_user as executed_by, -main.start_ts as start_ts, -main.end_ts as end_ts, -date(main.start_ts) as start_date -from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -inner join -IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics -on main.recon_table_id = metrics.recon_table_id -where metrics.run_metrics.status = true -order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md new file mode 100644 index 000000000..1795ecd89 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md @@ -0,0 +1,2 @@ +# Daily Data Validation Issues Report +### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql new file mode 100644 index 000000000..b95cb06b9 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql @@ -0,0 +1,45 @@ +SELECT + main.recon_id, + main.source_type, + main.report_type, + main.source_table.`CATALOG` AS source_catalog, + main.source_table.`SCHEMA` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( + main.source_table.catalog, + '.', + main.source_table.schema, + '.', + main.source_table.table_name + ) + ELSE CONCAT( + main.source_table.schema, + '.', + main.source_table.table_name + ) + END AS source_table, + main.target_table.`CATALOG` AS target_catalog, + main.target_table.`SCHEMA` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT( + main.target_table.catalog, + '.', + main.target_table.schema, + '.', + main.target_table.table_name + ) AS target_table, + metrics.run_metrics.status AS status, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts, + CAST(main.start_ts AS DATE) AS start_date +FROM + remorph_catalog.remorph_schema.main AS main + INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = FALSE +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql new file mode 100644 index 000000000..194fef2e5 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql @@ -0,0 +1,40 @@ +/* --filter start_date */ +SELECT + main.recon_id, + main.source_type, + main.report_type, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( + main.source_table.catalog, + '.', + main.source_table.schema, + '.', + main.source_table.table_name + ) + ELSE CONCAT( + main.source_table.schema, + '.', + main.source_table.table_name + ) + END AS source_table, + CONCAT( + main.target_table.catalog, + '.', + main.target_table.schema, + '.', + main.target_table.table_name + ) AS target_table, + metrics.run_metrics.status AS status, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts, + CAST(main.start_ts AS DATE) AS start_date +FROM + remorph_catalog.remorph_schema.main AS main + INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = TRUE +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name \ No newline at end of file diff --git a/tests/unit/deployment/test_dashboard.py b/tests/unit/deployment/test_dashboard.py index dce4d25e4..b690dc339 100644 --- a/tests/unit/deployment/test_dashboard.py +++ b/tests/unit/deployment/test_dashboard.py @@ -13,7 +13,8 @@ def test_deploy_new_dashboard(): dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" + ) ws = create_autospec(WorkspaceClient) dashboard = Dashboard( dashboard_id="9c1fbf4ad3449be67d6cb64c8acc730b", @@ -32,7 +33,8 @@ def test_deploy_new_dashboard(): def test_deploy_new_dashboard_with_params(): dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" + ) substituted_dashboard_file = Path(__file__).parent / Path( '../../resources/dashboards/queries/Remorph-Reconciliation-Substituted.lvdash.json' ) @@ -56,7 +58,8 @@ def test_deploy_new_dashboard_with_params(): def test_deploy_new_parameterless_dashboard_with_user_params(): dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Test_Dashboard_No_Param.lvdash.json") + "../../resources/dashboards/queries/Test_Dashboard_No_Param.lvdash.json" + ) ws = create_autospec(WorkspaceClient) dashboard = Dashboard( dashboard_id="8c1fbf4ad3449be67d6cb64c8acc730b", @@ -74,7 +77,8 @@ def test_deploy_new_parameterless_dashboard_with_user_params(): def test_deploy_existing_dashboard(): dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" + ) ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( @@ -94,7 +98,8 @@ def test_deploy_existing_dashboard(): def test_deploy_missing_dashboard(): dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json") + "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" + ) ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( From 387290cb7307c2e8cd825bdf38442c4a004d1f3e Mon Sep 17 00:00:00 2001 From: "sundar.shankar" Date: Wed, 7 Aug 2024 16:23:20 +0530 Subject: [PATCH 07/10] test case fixes --- .../labs/remorph/deployment/dashboard.py | 7 +- .../dashboards/3_1_failed_table_list.sql | 1 + .../Remorph-Reconciliation.lvdash.json | 2904 ----------------- .../dashboards/queries/01_queries.sql | 29 +- tests/unit/deployment/test_dashboard.py | 118 +- 5 files changed, 56 insertions(+), 3003 deletions(-) delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json diff --git a/src/databricks/labs/remorph/deployment/dashboard.py b/src/databricks/labs/remorph/deployment/dashboard.py index b3be62af1..884a738e2 100644 --- a/src/databricks/labs/remorph/deployment/dashboard.py +++ b/src/databricks/labs/remorph/deployment/dashboard.py @@ -32,6 +32,7 @@ def __init__( def _handle_existing_dashboard(self, dashboard_id: str, display_name: str, parent_path: str) -> str | None: try: dashboard = self._ws.lakeview.get(dashboard_id) + if dashboard.lifecycle_state is None: raise NotFound(f"Dashboard life cycle state: {display_name} ({dashboard_id})") if dashboard.lifecycle_state == LifecycleState.TRASHED: @@ -69,14 +70,10 @@ def deploy( database=config.schema, database_to_replace="remorph_schema", ) - print("**********************") - print(metadata) - print("***********************") metadata.display_name = self._name_with_prefix(metadata.display_name) - reference = f"{folder.parent.stem}_{folder.stem}".lower() - dashboard_id = self._install_state.dashboards.get(reference) + dashboard_id = self._install_state.dashboards.get(name) if dashboard_id is not None: dashboard_id = self._handle_existing_dashboard(dashboard_id, metadata.display_name, parent_path) diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql index b95cb06b9..0df494e8a 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql @@ -1,3 +1,4 @@ +/* --filter start_date */ SELECT main.recon_id, main.source_type, diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json b/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json deleted file mode 100644 index 80419b4f2..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/Remorph-Reconciliation.lvdash.json +++ /dev/null @@ -1,2904 +0,0 @@ -{ - "datasets": [ - { - "name": "655d5caf", - "displayName": "recon_main", - "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.exception_message as exception,\nmetrics.recon_metrics.row_comparison.missing_in_source as missing_in_source,\nmetrics.recon_metrics.row_comparison.missing_in_target as missing_in_target,\nmetrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch,\nmetrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch,\nmetrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns,\nmetrics.recon_metrics.schema_comparison as schema_comparison,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", - "parameters": [ - { - "displayName": "catalog", - "keyword": "catalog", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "remorph" - } - ] - } - } - }, - { - "displayName": "schema", - "keyword": "schema", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "reconcile" - } - ] - } - } - } - ] - }, - { - "name": "78e296d2", - "displayName": "recon_details_schema", - "query": "with tmp as (select \nrecon_table_id,\ninserted_ts,\nexplode(data) as schema_data\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type='schema'\n) \nselect \nmain.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nschema_data['source_column'] as source_column,\nschema_data['source_datatype'] as source_datatype,\nschema_data['databricks_column'] as databricks_column,\nschema_data['databricks_datatype'] as databricks_datatype,\nschema_data['is_valid'] as is_valid\nfrom \nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \ntmp\non main.recon_table_id = tmp.recon_table_id\norder by tmp.inserted_ts desc, main.recon_id, main.target_table\n", - "parameters": [ - { - "displayName": "catalog", - "keyword": "catalog", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "remorph" - } - ] - } - } - }, - { - "displayName": "schema", - "keyword": "schema", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "reconcile" - } - ] - } - } - } - ] - }, - { - "name": "d4050b68", - "displayName": "recon_details_pivot", - "query": "with tmp as (select recon_table_id, inserted_ts ,recon_type, explode(data) as data, \nrow_number() over(partition by recon_table_id,recon_type order by recon_table_id) as rn\nfrom IDENTIFIER(:catalog || '.' || :schema || '.details' )\nwhere recon_type != 'schema')\nselect main.recon_id,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table,\nrecon_type, key, value, rn\nfrom tmp\ninner join\nIDENTIFIER(:catalog || '.' || :schema || '.main' ) main\non main.recon_table_id = tmp.recon_table_id\nlateral view explode(data) exploded_data AS key, value\n", - "parameters": [ - { - "displayName": "catalog", - "keyword": "catalog", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "remorph" - } - ] - } - } - }, - { - "displayName": "schema", - "keyword": "schema", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "reconcile" - } - ] - } - } - } - ] - }, - { - "name": "c132f0ca", - "displayName": "failed_table", - "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nmain.source_table.`catalog` as source_catalog,\nmain.source_table.`schema` as source_schema,\nmain.source_table.table_name as source_table_name,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nmain.target_table.`catalog` as target_catalog,\nmain.target_table.`schema` as target_schema,\nmain.target_table.table_name as target_table_name,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = false\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", - "parameters": [ - { - "displayName": "catalog", - "keyword": "catalog", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "remorph" - } - ] - } - } - }, - { - "displayName": "schema", - "keyword": "schema", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "reconcile" - } - ] - } - } - } - ] - }, - { - "name": "7637e124", - "displayName": "success_table", - "query": "select \nmain.recon_id,\nmain.source_type,\nmain.report_type,\nCASE \n WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) \n END AS source_table,\nCONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, \nmetrics.run_metrics.status as status,\nmetrics.run_metrics.run_by_user as executed_by,\nmain.start_ts as start_ts,\nmain.end_ts as end_ts,\ndate(main.start_ts) as start_date\nfrom IDENTIFIER(:catalog || '.' || :schema || '.main' ) main\ninner join \nIDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics\non main.recon_table_id = metrics.recon_table_id\nwhere metrics.run_metrics.status = true\norder by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name", - "parameters": [ - { - "displayName": "catalog", - "keyword": "catalog", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "remorph" - } - ] - } - } - }, - { - "displayName": "schema", - "keyword": "schema", - "dataType": "STRING", - "defaultSelection": { - "values": { - "dataType": "STRING", - "values": [ - { - "value": "reconcile" - } - ] - } - } - } - ] - } - ], - "pages": [ - { - "name": "569063d3", - "displayName": "New Page", - "layout": [ - { - "widget": { - "name": "b4a9c7fd", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "source_type", - "expression": "`source_type`" - }, - { - "name": "report_type", - "expression": "`report_type`" - }, - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "status", - "expression": "`status`" - }, - { - "name": "exception", - "expression": "`exception`" - }, - { - "name": "missing_in_source", - "expression": "`missing_in_source`" - }, - { - "name": "missing_in_target", - "expression": "`missing_in_target`" - }, - { - "name": "absolute_mismatch", - "expression": "`absolute_mismatch`" - }, - { - "name": "threshold_mismatch", - "expression": "`threshold_mismatch`" - }, - { - "name": "mismatch_columns", - "expression": "`mismatch_columns`" - }, - { - "name": "schema_comparison", - "expression": "`schema_comparison`" - }, - { - "name": "executed_by", - "expression": "`executed_by`" - }, - { - "name": "start_ts", - "expression": "`start_ts`" - }, - { - "name": "end_ts", - "expression": "`end_ts`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 1, - "widgetType": "table", - "encodings": { - "columns": [ - { - "fieldName": "recon_id", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 0, - "title": "recon_id", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "cellFormat": { - "default": { - "foregroundColor": null - }, - "rules": [ - { - "if": { - "column": "status", - "fn": "=", - "literal": "true" - }, - "value": { - "foregroundColor": "#3BD973" - } - }, - { - "if": { - "column": "status", - "fn": "=", - "literal": "false" - }, - "value": { - "foregroundColor": "#E92828" - } - } - ] - }, - "displayName": "recon_id" - }, - { - "fieldName": "source_type", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 1, - "title": "source_type", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "source_type" - }, - { - "fieldName": "report_type", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 2, - "title": "report_type", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "report_type" - }, - { - "fieldName": "source_table", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 5, - "title": "source_table", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "source_table" - }, - { - "fieldName": "target_table", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 8, - "title": "target_table", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "target_table" - }, - { - "fieldName": "status", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "boolean", - "displayAs": "boolean", - "visible": true, - "order": 9, - "title": "status", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "status" - }, - { - "fieldName": "exception", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 10, - "title": "exception", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "exception" - }, - { - "fieldName": "missing_in_source", - "numberFormat": "0", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "integer", - "displayAs": "number", - "visible": true, - "order": 11, - "title": "missing_in_source", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "missing_in_source" - }, - { - "fieldName": "missing_in_target", - "numberFormat": "0", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "integer", - "displayAs": "number", - "visible": true, - "order": 12, - "title": "missing_in_target", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "missing_in_target" - }, - { - "fieldName": "absolute_mismatch", - "numberFormat": "0", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "integer", - "displayAs": "number", - "visible": true, - "order": 13, - "title": "absolute_mismatch", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "absolute_mismatch" - }, - { - "fieldName": "threshold_mismatch", - "numberFormat": "0", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "integer", - "displayAs": "number", - "visible": true, - "order": 14, - "title": "threshold_mismatch", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "threshold_mismatch" - }, - { - "fieldName": "mismatch_columns", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 15, - "title": "mismatch_columns", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "mismatch_columns" - }, - { - "fieldName": "schema_comparison", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "boolean", - "displayAs": "boolean", - "visible": true, - "order": 16, - "title": "schema_comparison", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "schema_comparison" - }, - { - "fieldName": "executed_by", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 19, - "title": "executed_by", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "executed_by" - }, - { - "fieldName": "start_ts", - "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "datetime", - "displayAs": "datetime", - "visible": true, - "order": 20, - "title": "start_ts", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "start_ts" - }, - { - "fieldName": "end_ts", - "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "datetime", - "displayAs": "datetime", - "visible": true, - "order": 21, - "title": "end_ts", - "allowSearch": false, - "alignContent": "right", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "end_ts" - } - ] - }, - "invisibleColumns": [ - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_catalog", - "type": "string", - "displayAs": "string", - "order": 3, - "title": "source_catalog", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_schema", - "type": "string", - "displayAs": "string", - "order": 4, - "title": "source_schema", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_catalog", - "type": "string", - "displayAs": "string", - "order": 6, - "title": "target_catalog", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_schema", - "type": "string", - "displayAs": "string", - "order": 7, - "title": "target_schema", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_table_name", - "type": "string", - "displayAs": "string", - "order": 17, - "title": "source_table_name", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_table_name", - "type": "string", - "displayAs": "string", - "order": 18, - "title": "target_table_name", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - } - ], - "allowHTMLByDefault": false, - "itemsPerPage": 100, - "paginationSize": "default", - "condensed": true, - "withRowNumber": true, - "frame": { - "showDescription": true, - "description": "Summary Table", - "showTitle": false - } - } - }, - "position": { - "x": 0, - "y": 10, - "width": 6, - "height": 7 - } - }, - { - "widget": { - "name": "32ea02c2", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "recon_id_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id", - "query": { - "datasetName": "78e296d2", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "recon_id_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "recon_id_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "recon_id", - "displayName": "recon_id", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_recon_id" - }, - { - "fieldName": "recon_id", - "displayName": "recon_id", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_recon_id" - }, - { - "fieldName": "recon_id", - "displayName": "recon_id", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": false, - "title": "Recon Id", - "description": "" - } - } - }, - "position": { - "x": 0, - "y": 4, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "1be6ab4a", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "target_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table", - "query": { - "datasetName": "78e296d2", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "target_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "target_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "target_table", - "displayName": "target_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_target_table" - }, - { - "fieldName": "target_table", - "displayName": "target_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_target_table" - }, - { - "fieldName": "target_table", - "displayName": "target_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": false, - "title": "Traget Table Name", - "description": "" - } - } - }, - "position": { - "x": 2, - "y": 4, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "3ec3ec2d", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "source_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table", - "query": { - "datasetName": "78e296d2", - "fields": [ - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "source_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "source_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "source_table", - "displayName": "source_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_table" - }, - { - "fieldName": "source_table", - "displayName": "source_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d615a59bb80b99ca72d1db_source_table" - }, - { - "fieldName": "source_table", - "displayName": "source_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": false, - "title": "Source Table Name", - "description": "" - } - } - }, - "position": { - "x": 4, - "y": 4, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "8cf05b2f", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "source_type", - "expression": "`source_type`" - }, - { - "name": "source_type_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "source_type", - "displayName": "source_type", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_source_type" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Source Type", - "description": "Applied only on summary table" - } - } - }, - "position": { - "x": 0, - "y": 6, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "21add8d7", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "recon_type", - "expression": "`recon_type`" - }, - { - "name": "recon_type_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-single-select", - "encodings": { - "fields": [ - { - "fieldName": "recon_type", - "displayName": "recon_type", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_type" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Category", - "description": "Applied only on details table" - } - } - }, - "position": { - "x": 3, - "y": 27, - "width": 3, - "height": 1 - } - }, - { - "widget": { - "name": "49e40526", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "hourly(start_ts)", - "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" - }, - { - "name": "absolute_mismatch", - "expression": "`absolute_mismatch`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 3, - "widgetType": "area", - "encodings": { - "x": { - "fieldName": "hourly(start_ts)", - "scale": { - "type": "temporal" - }, - "displayName": "start_ts" - }, - "y": { - "fieldName": "absolute_mismatch", - "scale": { - "type": "quantitative" - }, - "displayName": "absolute_mismatch" - }, - "color": { - "fieldName": "target_table", - "scale": { - "type": "categorical" - }, - "displayName": "target_table" - }, - "label": { - "show": false - } - }, - "frame": { - "showTitle": true, - "title": "Mismatched Records", - "showDescription": false - } - } - }, - "position": { - "x": 0, - "y": 47, - "width": 3, - "height": 6 - } - }, - { - "widget": { - "name": "2cbc9f42", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "hourly(start_ts)", - "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" - }, - { - "name": "missing_in_target", - "expression": "`missing_in_target`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 3, - "widgetType": "area", - "encodings": { - "x": { - "fieldName": "hourly(start_ts)", - "scale": { - "type": "temporal" - }, - "displayName": "start_ts" - }, - "y": { - "fieldName": "missing_in_target", - "scale": { - "type": "quantitative" - }, - "displayName": "missing_in_target" - }, - "color": { - "fieldName": "target_table", - "scale": { - "type": "categorical" - }, - "displayName": "target_table" - }, - "label": { - "show": false - } - }, - "frame": { - "showTitle": true, - "title": "Missing in Source" - } - } - }, - "position": { - "x": 3, - "y": 53, - "width": 3, - "height": 6 - } - }, - { - "widget": { - "name": "5ae38078", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "hourly(start_ts)", - "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" - }, - { - "name": "missing_in_source", - "expression": "`missing_in_source`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 3, - "widgetType": "area", - "encodings": { - "x": { - "fieldName": "hourly(start_ts)", - "scale": { - "type": "temporal" - }, - "displayName": "start_ts" - }, - "y": { - "fieldName": "missing_in_source", - "scale": { - "type": "quantitative" - }, - "displayName": "missing_in_source" - }, - "color": { - "fieldName": "target_table", - "scale": { - "type": "categorical" - }, - "displayName": "target_table" - }, - "label": { - "show": false - } - }, - "frame": { - "showTitle": true, - "title": "Missing in Databricks" - } - } - }, - "position": { - "x": 0, - "y": 53, - "width": 3, - "height": 6 - } - }, - { - "widget": { - "name": "2ea234ee", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "hourly(start_ts)", - "expression": "DATE_TRUNC(\"HOUR\", `start_ts`)" - }, - { - "name": "threshold_mismatch", - "expression": "`threshold_mismatch`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 3, - "widgetType": "area", - "encodings": { - "x": { - "fieldName": "hourly(start_ts)", - "scale": { - "type": "temporal" - }, - "displayName": "start_ts" - }, - "y": { - "fieldName": "threshold_mismatch", - "scale": { - "type": "quantitative" - }, - "displayName": "threshold_mismatch" - }, - "color": { - "fieldName": "target_table", - "scale": { - "type": "categorical" - }, - "displayName": "target_table" - } - }, - "frame": { - "showTitle": true, - "title": "Threshold Mismatches" - } - } - }, - "position": { - "x": 3, - "y": 47, - "width": 3, - "height": 6 - } - }, - { - "widget": { - "name": "403919aa", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "78e296d2", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "source_column", - "expression": "`source_column`" - }, - { - "name": "source_datatype", - "expression": "`source_datatype`" - }, - { - "name": "databricks_column", - "expression": "`databricks_column`" - }, - { - "name": "databricks_datatype", - "expression": "`databricks_datatype`" - }, - { - "name": "is_valid", - "expression": "`is_valid`" - } - ], - "disaggregated": true - } - } - ], - "spec": { - "version": 1, - "widgetType": "table", - "encodings": { - "columns": [ - { - "fieldName": "recon_id", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 0, - "title": "recon_id", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "cellFormat": { - "default": { - "foregroundColor": null - }, - "rules": [ - { - "if": { - "column": "is_valid", - "fn": "=", - "literal": "false" - }, - "value": { - "foregroundColor": "#E92828" - } - }, - { - "if": { - "column": "is_valid", - "fn": "=", - "literal": "true" - }, - "value": { - "foregroundColor": "#3BD973" - } - } - ] - }, - "displayName": "recon_id" - }, - { - "fieldName": "source_table", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 4, - "title": "source_table", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "source_table" - }, - { - "fieldName": "target_table", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 8, - "title": "target_table", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "target_table" - }, - { - "fieldName": "source_column", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 9, - "title": "source_column", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "source_column" - }, - { - "fieldName": "source_datatype", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 10, - "title": "source_datatype", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "source_datatype" - }, - { - "fieldName": "databricks_column", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 11, - "title": "databricks_column", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "databricks_column" - }, - { - "fieldName": "databricks_datatype", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 12, - "title": "databricks_datatype", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "databricks_datatype" - }, - { - "fieldName": "is_valid", - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "type": "string", - "displayAs": "string", - "visible": true, - "order": 13, - "title": "is_valid", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false, - "displayName": "is_valid" - } - ] - }, - "invisibleColumns": [ - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_table_name", - "type": "string", - "displayAs": "string", - "order": 1, - "title": "source_table_name", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_catalog", - "type": "string", - "displayAs": "string", - "order": 2, - "title": "source_catalog", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "source_schema", - "type": "string", - "displayAs": "string", - "order": 3, - "title": "source_schema", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_catalog", - "type": "string", - "displayAs": "string", - "order": 5, - "title": "target_catalog", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_schema", - "type": "string", - "displayAs": "string", - "order": 6, - "title": "target_schema", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - }, - { - "booleanValues": [ - "false", - "true" - ], - "imageUrlTemplate": "{{ @ }}", - "imageTitleTemplate": "{{ @ }}", - "imageWidth": "", - "imageHeight": "", - "linkUrlTemplate": "{{ @ }}", - "linkTextTemplate": "{{ @ }}", - "linkTitleTemplate": "{{ @ }}", - "linkOpenInNewTab": true, - "name": "target_table_name", - "type": "string", - "displayAs": "string", - "order": 7, - "title": "target_table_name", - "allowSearch": false, - "alignContent": "left", - "allowHTML": false, - "highlightLinks": false, - "useMonospaceFont": false, - "preserveWhitespace": false - } - ], - "allowHTMLByDefault": false, - "itemsPerPage": 25, - "paginationSize": "default", - "condensed": true, - "withRowNumber": false, - "frame": { - "showTitle": true, - "title": "Schema Details" - } - } - }, - "position": { - "x": 0, - "y": 19, - "width": 6, - "height": 6 - } - }, - { - "widget": { - "name": "b154de70", - "queries": [ - { - "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog", - "query": { - "datasetName": "655d5caf", - "parameters": [ - { - "name": "catalog", - "keyword": "catalog" - } - ], - "disaggregated": false - } - }, - { - "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog", - "query": { - "datasetName": "78e296d2", - "parameters": [ - { - "name": "catalog", - "keyword": "catalog" - } - ], - "disaggregated": false - } - }, - { - "name": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog", - "query": { - "datasetName": "d4050b68", - "parameters": [ - { - "name": "catalog", - "keyword": "catalog" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-single-select", - "encodings": { - "fields": [ - { - "parameterName": "catalog", - "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef4551b249988ed6a096f974f_catalog" - }, - { - "parameterName": "catalog", - "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef45a1feca2f9f5ad2fae413d_catalog" - }, - { - "parameterName": "catalog", - "queryName": "parameter_dashboards/01ef3daef44f1b809240fd537995dc43/datasets/01ef3daef460116ebd0344d49c89637c_catalog" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Catalog", - "description": "’remorph' is the default catalog for the metrics table; specify a different catalog name if the metrics are stored elsewhere." - } - } - }, - "position": { - "x": 0, - "y": 2, - "width": 3, - "height": 2 - } - }, - { - "widget": { - "name": "05101c82", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "report_type", - "expression": "`report_type`" - }, - { - "name": "report_type_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "report_type", - "displayName": "report_type", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_report_type" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Report", - "description": "Applied only on summary table" - } - } - }, - "position": { - "x": 2, - "y": 6, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "081790c2", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "start_ts", - "expression": "`start_ts`" - }, - { - "name": "start_ts_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-date-range-picker", - "encodings": { - "fields": [ - { - "fieldName": "start_ts", - "displayName": "start_ts", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_start_ts" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "started at", - "description": "Applied on all tables" - } - } - }, - "position": { - "x": 0, - "y": 8, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "ffcb9c29", - "textbox_spec": "# Main Reconciliation Table\n### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records." - }, - "position": { - "x": 0, - "y": 0, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "acd1adcc", - "textbox_spec": "# Drill Down\n### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues." - }, - "position": { - "x": 0, - "y": 25, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "680c9fe9", - "textbox_spec": "# Schema Comparison Details\n### This table provides a detailed view of schema mismatches." - }, - "position": { - "x": 0, - "y": 17, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "b1a2db69", - "textbox_spec": "# Visualization of Missing and Mismatched Records" - }, - "position": { - "x": 0, - "y": 45, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "2199286c", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "value", - "expression": "`value`" - }, - { - "name": "key", - "expression": "`key`" - }, - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "recon_type", - "expression": "`recon_type`" - }, - { - "name": "rn", - "expression": "`rn`" - } - ], - "disaggregated": false, - "orders": [ - { - "direction": "ASC", - "expression": "`recon_id`" - }, - { - "direction": "ASC", - "expression": "`source_table`" - }, - { - "direction": "ASC", - "expression": "`target_table`" - }, - { - "direction": "ASC", - "expression": "`recon_type`" - }, - { - "direction": "ASC", - "expression": "`rn`" - }, - { - "direction": "ASC", - "expression": "`key`" - } - ] - } - } - ], - "spec": { - "version": 3, - "widgetType": "pivot", - "encodings": { - "rows": [ - { - "fieldName": "recon_id", - "displayName": "recon_id" - }, - { - "fieldName": "source_table", - "displayName": "source_table" - }, - { - "fieldName": "target_table", - "displayName": "target_table" - }, - { - "fieldName": "recon_type", - "displayName": "recon_type" - }, - { - "fieldName": "rn", - "displayName": "rn" - } - ], - "columns": [ - { - "fieldName": "key", - "displayName": "key" - } - ], - "cell": { - "fieldName": "value", - "cellType": "text", - "displayName": "value" - } - } - } - }, - "position": { - "x": 0, - "y": 29, - "width": 6, - "height": 6 - } - }, - { - "widget": { - "name": "9c0a2a5e", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "target_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "target_table", - "displayName": "target_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_target_table" - } - ] - }, - "frame": { - "showDescription": true, - "showTitle": true, - "title": "Target Table Name", - "description": "Applied only on details table" - } - } - }, - "position": { - "x": 0, - "y": 28, - "width": 3, - "height": 1 - } - }, - { - "widget": { - "name": "1e536216", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "source_table", - "expression": "`source_table`" - }, - { - "name": "source_table_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "source_table", - "displayName": "source_table", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_source_table" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Source Table Name", - "description": "Applied only on details table" - } - } - }, - "position": { - "x": 3, - "y": 28, - "width": 3, - "height": 1 - } - }, - { - "widget": { - "name": "8e96d5ba", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id", - "query": { - "datasetName": "d4050b68", - "fields": [ - { - "name": "recon_id", - "expression": "`recon_id`" - }, - { - "name": "recon_id_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "recon_id", - "displayName": "recon_id", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257d91a65971719236fe20951_recon_id" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Recon Id", - "description": "Applied only on details table" - } - } - }, - "position": { - "x": 0, - "y": 27, - "width": 3, - "height": 1 - } - }, - { - "widget": { - "name": "6b586312", - "queries": [ - { - "name": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by", - "query": { - "datasetName": "655d5caf", - "fields": [ - { - "name": "executed_by", - "expression": "`executed_by`" - }, - { - "name": "executed_by_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [ - { - "fieldName": "executed_by", - "displayName": "executed_by", - "queryName": "dashboards/01ef183257cb15078fcc146dc3d5dffc/datasets/01ef183257cf103a894826ffd7e2f57d_executed_by" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Executed by", - "description": "Applied only on summary table" - } - } - }, - "position": { - "x": 4, - "y": 6, - "width": 2, - "height": 2 - } - }, - { - "widget": { - "name": "9f0e38d5", - "queries": [ - { - "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema", - "query": { - "datasetName": "655d5caf", - "parameters": [ - { - "name": "schema", - "keyword": "schema" - } - ], - "disaggregated": false - } - }, - { - "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema", - "query": { - "datasetName": "78e296d2", - "parameters": [ - { - "name": "schema", - "keyword": "schema" - } - ], - "disaggregated": false - } - }, - { - "name": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema", - "query": { - "datasetName": "d4050b68", - "parameters": [ - { - "name": "schema", - "keyword": "schema" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-single-select", - "encodings": { - "fields": [ - { - "parameterName": "schema", - "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35b1545a4c98317212e934d_schema" - }, - { - "parameterName": "schema", - "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f35e15fbac6368c663d1b2c1_schema" - }, - { - "parameterName": "schema", - "queryName": "parameter_dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef2274f3611637b915d9ec4b58de07_schema" - } - ] - }, - "frame": { - "showTitle": true, - "showDescription": true, - "title": "Schema", - "description": "’reconcile’ is the default schema for the metrics table; specify a different schema name if the metrics are stored elsewhere." - } - } - }, - "position": { - "x": 3, - "y": 2, - "width": 3, - "height": 2 - } - }, - { - "widget": { - "name": "482b5576", - "textbox_spec": "# Daily Data Validation Issues Report\n### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day." - }, - "position": { - "x": 0, - "y": 35, - "width": 6, - "height": 2 - } - }, - { - "widget": { - "name": "fc96dc6d", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "c132f0ca", - "fields": [ - { - "name": "target_table", - "expression": "`target_table`" - }, - { - "name": "countdistinct(recon_id)", - "expression": "COUNT(DISTINCT `recon_id`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 3, - "widgetType": "bar", - "encodings": { - "x": { - "fieldName": "target_table", - "scale": { - "type": "categorical", - "sort": { - "by": "y-reversed" - } - }, - "displayName": "target_table" - }, - "y": { - "fieldName": "countdistinct(recon_id)", - "scale": { - "type": "quantitative" - }, - "displayName": "Count of Unique recon_id" - }, - "label": { - "show": true - } - }, - "frame": { - "showDescription": false, - "title": "Number of Distinct Recon IDs per Target Table Failed", - "showTitle": true - } - } - }, - "position": { - "x": 0, - "y": 39, - "width": 5, - "height": 6 - } - }, - { - "widget": { - "name": "4c17bbd2", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "c132f0ca", - "fields": [ - { - "name": "countdistinct(target_table)", - "expression": "COUNT(DISTINCT `target_table`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": { - "fieldName": "countdistinct(target_table)", - "displayName": "Count of Unique target_table" - } - }, - "frame": { - "title": "Unique target tables failed", - "showTitle": true, - "showDescription": false, - "description": "Unique target tables failed" - } - } - }, - "position": { - "x": 5, - "y": 41, - "width": 1, - "height": 2 - } - }, - { - "widget": { - "name": "d73ad40b", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "c132f0ca", - "fields": [ - { - "name": "countdistinct(recon_id)", - "expression": "COUNT(DISTINCT `recon_id`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": { - "fieldName": "countdistinct(recon_id)", - "displayName": "Count of Unique recon_id" - } - }, - "frame": { - "title": "Total number of runs failed", - "showTitle": true - } - } - }, - "position": { - "x": 5, - "y": 39, - "width": 1, - "height": 2 - } - }, - { - "widget": { - "name": "482ad4f3", - "queries": [ - { - "name": "main_query", - "query": { - "datasetName": "7637e124", - "fields": [ - { - "name": "countdistinct(target_table)", - "expression": "COUNT(DISTINCT `target_table`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": { - "fieldName": "countdistinct(target_table)", - "displayName": "Count of Unique target_table" - } - }, - "frame": { - "showTitle": true, - "title": "Unique target tables success" - } - } - }, - "position": { - "x": 5, - "y": 43, - "width": 1, - "height": 2 - } - }, - { - "widget": { - "name": "0aa08479", - "queries": [ - { - "name": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date", - "query": { - "datasetName": "c132f0ca", - "fields": [ - { - "name": "start_date", - "expression": "`start_date`" - }, - { - "name": "start_date_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - }, - { - "name": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date", - "query": { - "datasetName": "7637e124", - "fields": [ - { - "name": "start_date", - "expression": "`start_date`" - }, - { - "name": "start_date_associativity", - "expression": "COUNT_IF(`associative_filter_predicate_group`)" - } - ], - "disaggregated": false - } - } - ], - "spec": { - "version": 2, - "widgetType": "filter-date-range-picker", - "encodings": { - "fields": [ - { - "fieldName": "start_date", - "displayName": "start_date", - "queryName": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3c63c5c6160687607291378cdcfc_start_date" - }, - { - "fieldName": "start_date", - "displayName": "start_date", - "queryName": "dashboards/01ef2274f35812e19ad423f93c8e062b/datasets/01ef3cdefe09194f96a029cc72193b8f_start_date" - } - ] - } - } - }, - "position": { - "x": 0, - "y": 37, - "width": 6, - "height": 2 - } - } - ] - } - ] -} \ No newline at end of file diff --git a/tests/resources/dashboards/queries/01_queries.sql b/tests/resources/dashboards/queries/01_queries.sql index 01842b489..376b0e3aa 100644 --- a/tests/resources/dashboards/queries/01_queries.sql +++ b/tests/resources/dashboards/queries/01_queries.sql @@ -1,31 +1,8 @@ -select +SELECT main.recon_id, main.source_type, main.report_type, main.source_table.`catalog` as source_catalog, main.source_table.`schema` as source_schema, -main.source_table.table_name as source_table_name, -CASE - WHEN COALESCE(MAIN.SOURCE_TABLE.CATALOG, '') <> '' THEN CONCAT(MAIN.SOURCE_TABLE.CATALOG, '.', MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - ELSE CONCAT(MAIN.SOURCE_TABLE.SCHEMA, '.', MAIN.SOURCE_TABLE.TABLE_NAME) - END AS source_table, -main.target_table.`catalog` as target_catalog, -main.target_table.`schema` as target_schema, -main.target_table.table_name as target_table_name, -CONCAT(MAIN.TARGET_TABLE.CATALOG, '.', MAIN.TARGET_TABLE.SCHEMA, '.', MAIN.TARGET_TABLE.TABLE_NAME) AS target_table, -metrics.run_metrics.status as status, -metrics.run_metrics.exception_message as exception, -metrics.recon_metrics.row_comparison.missing_in_source as missing_in_source, -metrics.recon_metrics.row_comparison.missing_in_target as missing_in_target, -metrics.recon_metrics.column_comparison.absolute_mismatch as absolute_mismatch, -metrics.recon_metrics.column_comparison.threshold_mismatch as threshold_mismatch, -metrics.recon_metrics.column_comparison.mismatch_columns as mismatch_columns, -metrics.recon_metrics.schema_comparison as schema_comparison, -metrics.run_metrics.run_by_user as executed_by, -main.start_ts as start_ts, -main.end_ts as end_ts -from IDENTIFIER(:catalog || '.' || :schema || '.main' ) main -inner join -IDENTIFIER(:catalog || '.' || :schema || '.metrics' ) metrics -on main.recon_table_id = metrics.recon_table_id -order by metrics.inserted_ts desc, main.recon_id, main.target_table.table_name \ No newline at end of file +main.source_table.table_name as source_table_name +FROM remorph_catalog.remorph_schema.main main diff --git a/tests/unit/deployment/test_dashboard.py b/tests/unit/deployment/test_dashboard.py index b690dc339..2129be0be 100644 --- a/tests/unit/deployment/test_dashboard.py +++ b/tests/unit/deployment/test_dashboard.py @@ -1,105 +1,86 @@ +import json from pathlib import Path from unittest.mock import create_autospec - +import logging +import pytest from databricks.labs.blueprint.installation import MockInstallation from databricks.labs.blueprint.installer import InstallState from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import InvalidParameterValue +from databricks.sdk.errors import InvalidParameterValue, NotFound from databricks.sdk.service.dashboards import Dashboard +from databricks.sdk.service.dashboards import LifecycleState from databricks.labs.remorph.config import ReconcileMetadataConfig from databricks.labs.remorph.deployment.dashboard import DashboardDeployment -def test_deploy_new_dashboard(): - dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" - ) - ws = create_autospec(WorkspaceClient) - dashboard = Dashboard( - dashboard_id="9c1fbf4ad3449be67d6cb64c8acc730b", - display_name="Remorph-Reconciliation", - ) - ws.lakeview.create.return_value = dashboard - installation = MockInstallation(is_global=False) - install_state = InstallState.from_installation(installation) - name = "Remorph-Reconciliation" - dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) - _, kwargs = ws.lakeview.create.call_args - assert kwargs["serialized_dashboard"] == dashboard_file.read_text() - assert install_state.dashboards[name] == dashboard.dashboard_id - +def _get_dashboard_query(kwargs): + serialized_dashboard = json.loads(kwargs['serialized_dashboard']) + return serialized_dashboard['datasets'][0]['query'] -def test_deploy_new_dashboard_with_params(): - dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" - ) - substituted_dashboard_file = Path(__file__).parent / Path( - '../../resources/dashboards/queries/Remorph-Reconciliation-Substituted.lvdash.json' - ) +def test_deploy_dashboard(): ws = create_autospec(WorkspaceClient) + expected_query = """SELECT + main.recon_id, + main.source_type, + main.report_type, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name\nFROM remorph.reconcile.main AS main""".strip() + + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") dashboard = Dashboard( dashboard_id="9c1fbf4ad3449be67d6cb64c8acc730b", display_name="Remorph-Reconciliation", ) ws.lakeview.create.return_value = dashboard - installation = MockInstallation(is_global=False) install_state = InstallState.from_installation(installation) name = "Remorph-Reconciliation" dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) + dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) _, kwargs = ws.lakeview.create.call_args - assert kwargs["serialized_dashboard"] == substituted_dashboard_file.read_text() + query = _get_dashboard_query(kwargs) + assert query == expected_query assert install_state.dashboards[name] == dashboard.dashboard_id -def test_deploy_new_parameterless_dashboard_with_user_params(): - dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Test_Dashboard_No_Param.lvdash.json" - ) - ws = create_autospec(WorkspaceClient) - dashboard = Dashboard( - dashboard_id="8c1fbf4ad3449be67d6cb64c8acc730b", - display_name="Test_Dashboard_No_Param", - ) - ws.lakeview.create.return_value = dashboard - - installation = MockInstallation(is_global=False) - install_state = InstallState.from_installation(installation) - name = "Test_Dashboard_No_Param" - dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) - assert install_state.dashboards[name] == dashboard.dashboard_id +@pytest.mark.parametrize("exception", [InvalidParameterValue, NotFound]) +def test_recovery_invalid_dashboard(caplog, exception): + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") - -def test_deploy_existing_dashboard(): - dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" - ) ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( dashboard_id=dashboard_id, display_name="Remorph-Reconciliation", ) - ws.lakeview.update.return_value = dashboard + ws.lakeview.create.return_value = dashboard + ws.lakeview.get.side_effect = exception name = "Remorph-Reconciliation" - installation = MockInstallation({"state.json": {"resources": {"dashboards": {name: dashboard_id}}, "version": 1}}) + installation = MockInstallation( + { + "state.json": { + "resources": {"dashboards": {name: "8c1fbf4ad3449be67d6cb64c8acc730b"}}, + "version": 1, + }, + } + ) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) - _, kwargs = ws.lakeview.update.call_args - assert kwargs["serialized_dashboard"] == dashboard_file.read_text() - assert install_state.dashboards[name] == dashboard.dashboard_id + with caplog.at_level(logging.DEBUG, logger="databricks.labs.remorph.deployment.dashboard"): + dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) + assert "Recovering invalid dashboard" in caplog.text + assert "Deleted dangling dashboard" in caplog.text + ws.workspace.delete.assert_called() + ws.lakeview.create.assert_called() + ws.lakeview.update.assert_not_called() -def test_deploy_missing_dashboard(): - dashboard_file = Path(__file__).parent / Path( - "../../resources/dashboards/queries/Remorph-Reconciliation.lvdash.json" - ) +def test_recovery_trashed_dashboard(caplog): + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") + ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" dashboard = Dashboard( @@ -107,7 +88,7 @@ def test_deploy_missing_dashboard(): display_name="Remorph-Reconciliation", ) ws.lakeview.create.return_value = dashboard - ws.lakeview.update.side_effect = InvalidParameterValue("Dashboard not found") + ws.lakeview.get.return_value = Dashboard(lifecycle_state=LifecycleState.TRASHED) name = "Remorph-Reconciliation" installation = MockInstallation( { @@ -119,7 +100,8 @@ def test_deploy_missing_dashboard(): ) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_file, ReconcileMetadataConfig()) - _, kwargs = ws.lakeview.create.call_args - assert kwargs["serialized_dashboard"] == dashboard_file.read_text() - assert install_state.dashboards[name] == dashboard.dashboard_id + with caplog.at_level(logging.DEBUG, logger="databricks.labs.remorph.deployment.dashboard"): + dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) + assert "Recreating trashed dashboard" in caplog.text + ws.lakeview.create.assert_called() + ws.lakeview.update.assert_not_called() From 9e6d5902f6d6a0cb9a95232a19f004f2f55f533f Mon Sep 17 00:00:00 2001 From: Bishwajit Date: Fri, 20 Sep 2024 17:33:11 +0530 Subject: [PATCH 08/10] Add Recon dashboard tiles --- .../reconcile/dashboards/0_0_recon_main.md | 2 - .../reconcile/dashboards/0_1_recon_main.sql | 50 --- .../1_0_schema_comparison_details.md | 2 - .../dashboards/1_1_recon_details_schema.sql | 52 --- .../dashboards/2_0_details_drill_down.md | 2 - .../dashboards/2_1_recon_details_pivot.sql | 55 --- .../dashboards/3_1_failed_table_list.sql | 46 --- .../dashboards/3_2_success_table_list.sql | 40 -- .../reconcile/dashboards/dashboard.yml | 1 - .../reconciliation_metrics/00_0_recon_main.md | 5 + .../01_0_recon_id.filter.yml | 5 + .../01_1_source_table.filter.yml | 5 + .../01_2_target_table.filter.yml | 5 + .../02_0_source_type.filter.yml | 5 + .../02_1_report_type.filter.yml | 5 + .../02_2_executed_by.filter.yml | 5 + .../03_0_started_at.filter.yml | 5 + .../03_1_category.filter.yml | 4 + .../04_0_summary_table.sql | 32 ++ .../05_0_schema_comparison_header.md | 3 + .../06_0_schema_details_table.sql | 36 ++ .../07_0_drill_down_header.md | 3 + .../08_0_recon_details_pivot.sql | 34 ++ ...9_0_daily_data_validation_issue_header.md} | 1 + .../10_0_success_fail_.filter.yml | 4 + .../11_0_failed_recon_ids.sql | 31 ++ .../12_0_total_failed_runs.sql | 10 + .../12_1_failed_targets.sql | 10 + .../12_3_successful_targets.sql | 10 + .../13_0_missing_mismatch_header.md | 1 + .../14_0_mismatched_records.sql | 14 + .../14_1_threshold_mismatches.sql | 14 + .../15_0_missing_in_databricks.sql | 14 + .../15_1_missing_in_source.sql | 14 + .../16_0_aggregate_recon_header.md | 6 + .../17_0_aggregate_summary_table.sql | 46 +++ .../18_0_aggregate_recon_drilldown_header.md | 2 + .../19_0_aggregate_details_table.sql | 92 +++++ .../20_0_aggregate_missing_mismatch_header.md | 1 + .../21_0_aggr_mismatched_records.sql | 19 + .../22_0_aggr_missing_in_databricks.sql | 19 + .../22_1_aggr_missing_in_source.sql | 19 + .../reconciliation_metrics/dashboard.yml | 352 ++++++++++++++++++ 43 files changed, 831 insertions(+), 250 deletions(-) delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql rename src/databricks/labs/remorph/resources/reconcile/dashboards/{3_0_data_validation_report.md => reconciliation_metrics/09_0_daily_data_validation_issue_header.md} (99%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md deleted file mode 100644 index e60f5369c..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_0_recon_main.md +++ /dev/null @@ -1,2 +0,0 @@ -# Main Reconciliation Table -## This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql deleted file mode 100644 index 9bce7a142..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/0_1_recon_main.sql +++ /dev/null @@ -1,50 +0,0 @@ -/* --filter recon_id target_table_name source_table_name source_type report_type executed_by */ -SELECT - main.recon_id, - main.source_type, - main.report_type, - main.source_table.`CATALOG` AS source_catalog, - main.source_table.`SCHEMA` AS source_schema, - main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( - main.source_table.catalog, - '.', - main.source_table.schema, - '.', - main.source_table.table_name - ) - ELSE CONCAT( - main.source_table.schema, - '.', - main.source_table.table_name - ) - END AS source_table, - main.target_table.`CATALOG` AS target_catalog, - main.target_table.`SCHEMA` AS target_schema, - main.target_table.table_name AS target_table_name, - CONCAT( - main.target_table.catalog, - '.', - main.target_table.schema, - '.', - main.target_table.table_name - ) AS target_table, - metrics.run_metrics.status AS status, - metrics.run_metrics.exception_message AS exception, - metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source, - metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target, - metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch, - metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch, - metrics.recon_metrics.column_comparison.mismatch_columns AS mismatch_columns, - metrics.recon_metrics.schema_comparison AS schema_comparison, - metrics.run_metrics.run_by_user AS executed_by, - main.start_ts AS start_ts, - main.end_ts AS end_ts -FROM - remorph_catalog.remorph_schema.main AS main - INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id -ORDER BY - metrics.inserted_ts DESC, - main.recon_id, - main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md deleted file mode 100644 index b5496efdf..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_0_schema_comparison_details.md +++ /dev/null @@ -1,2 +0,0 @@ -# Schema Comparison Details -## This table provides a detailed view of schema mismatches. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql deleted file mode 100644 index 64eef74fb..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/1_1_recon_details_schema.sql +++ /dev/null @@ -1,52 +0,0 @@ -/* --filter recon_id target_table_name source_table_name */ -WITH tmp AS ( - SELECT - recon_table_id, - inserted_ts, - EXPLODE(data) AS schema_data - FROM - remorph_catalog.remorph_schema.details - WHERE - recon_type = 'SCHEMA' -) -SELECT - main.recon_id, - main.source_table.`CATALOG` AS source_catalog, - main.source_table.`SCHEMA` AS source_schema, - main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( - main.source_table.catalog, - '.', - main.source_table.schema, - '.', - main.source_table.table_name - ) - ELSE CONCAT( - main.source_table.schema, - '.', - main.source_table.table_name - ) - END AS source_table, - main.target_table.`CATALOG` AS target_catalog, - main.target_table.`SCHEMA` AS target_schema, - main.target_table.table_name AS target_table_name, - CONCAT( - main.target_table.catalog, - '.', - main.target_table.schema, - '.', - main.target_table.table_name - ) AS target_table, - schema_data ['SOURCE_COLUMN'] AS source_column, - schema_data ['SOURCE_DATATYPE'] AS source_datatype, - schema_data ['DATABRICKS_COLUMN'] AS databricks_column, - schema_data ['DATABRICKS_DATATYPE'] AS databricks_datatype, - schema_data ['IS_VALID'] AS is_valid -FROM - remorph_catalog.remorph_schema.main AS main - INNER JOIN tmp ON main.recon_table_id = tmp.recon_table_id -ORDER BY - tmp.inserted_ts DESC, - main.recon_id, - main.target_table \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md deleted file mode 100644 index 67c5e3b2d..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_0_details_drill_down.md +++ /dev/null @@ -1,2 +0,0 @@ -# Drill Down -## The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql deleted file mode 100644 index 51e6591db..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/2_1_recon_details_pivot.sql +++ /dev/null @@ -1,55 +0,0 @@ -/* --filter start_date */ -WITH tmp AS ( - SELECT - recon_table_id, - inserted_ts, - recon_type, - EXPLODE(data) AS data, - ROW_NUMBER() OVER ( - PARTITION BY recon_table_id, - recon_type - ORDER BY - recon_table_id - ) AS rn - FROM - remorph_catalog.remorph_schema.details - WHERE - recon_type <> 'SCHEMA' -) -SELECT - main.recon_id, - main.source_table.`CATALOG` AS source_catalog, - main.source_table.`SCHEMA` AS source_schema, - main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( - main.source_table.catalog, - '.', - main.source_table.schema, - '.', - main.source_table.table_name - ) - ELSE CONCAT( - main.source_table.schema, - '.', - main.source_table.table_name - ) - END AS source_table, - main.target_table.`CATALOG` AS target_catalog, - main.target_table.`SCHEMA` AS target_schema, - main.target_table.table_name AS target_table_name, - CONCAT( - main.target_table.catalog, - '.', - main.target_table.schema, - '.', - main.target_table.table_name - ) AS target_table, - recon_type, - key, - value, - rn -FROM - tmp - INNER JOIN remorph_catalog.remorph_schema.main AS main ON main.recon_table_id = tmp.recon_table_id LATERAL VIEW EXPLODE(data) EXPLODED_DATA AS key, - value \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql deleted file mode 100644 index 0df494e8a..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_1_failed_table_list.sql +++ /dev/null @@ -1,46 +0,0 @@ -/* --filter start_date */ -SELECT - main.recon_id, - main.source_type, - main.report_type, - main.source_table.`CATALOG` AS source_catalog, - main.source_table.`SCHEMA` AS source_schema, - main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( - main.source_table.catalog, - '.', - main.source_table.schema, - '.', - main.source_table.table_name - ) - ELSE CONCAT( - main.source_table.schema, - '.', - main.source_table.table_name - ) - END AS source_table, - main.target_table.`CATALOG` AS target_catalog, - main.target_table.`SCHEMA` AS target_schema, - main.target_table.table_name AS target_table_name, - CONCAT( - main.target_table.catalog, - '.', - main.target_table.schema, - '.', - main.target_table.table_name - ) AS target_table, - metrics.run_metrics.status AS status, - metrics.run_metrics.run_by_user AS executed_by, - main.start_ts AS start_ts, - main.end_ts AS end_ts, - CAST(main.start_ts AS DATE) AS start_date -FROM - remorph_catalog.remorph_schema.main AS main - INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id -WHERE - metrics.run_metrics.status = FALSE -ORDER BY - metrics.inserted_ts DESC, - main.recon_id, - main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql deleted file mode 100644 index 194fef2e5..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_2_success_table_list.sql +++ /dev/null @@ -1,40 +0,0 @@ -/* --filter start_date */ -SELECT - main.recon_id, - main.source_type, - main.report_type, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT( - main.source_table.catalog, - '.', - main.source_table.schema, - '.', - main.source_table.table_name - ) - ELSE CONCAT( - main.source_table.schema, - '.', - main.source_table.table_name - ) - END AS source_table, - CONCAT( - main.target_table.catalog, - '.', - main.target_table.schema, - '.', - main.target_table.table_name - ) AS target_table, - metrics.run_metrics.status AS status, - metrics.run_metrics.run_by_user AS executed_by, - main.start_ts AS start_ts, - main.end_ts AS end_ts, - CAST(main.start_ts AS DATE) AS start_date -FROM - remorph_catalog.remorph_schema.main AS main - INNER JOIN remorph_catalog.remorph_schema.metrics AS metrics ON main.recon_table_id = metrics.recon_table_id -WHERE - metrics.run_metrics.status = TRUE -ORDER BY - metrics.inserted_ts DESC, - main.recon_id, - main.target_table.table_name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml deleted file mode 100644 index c331d2fd8..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/dashboard.yml +++ /dev/null @@ -1 +0,0 @@ -display_name: "Reconciliation Metrics" \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md new file mode 100644 index 000000000..2854d9769 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md @@ -0,0 +1,5 @@ +# Main Reconciliation Table + +### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records. + +[Aggregates Reconciliation Table](#aggregates-reconcile-table-metrics) \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml new file mode 100644 index 000000000..f9e5e7fb7 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml @@ -0,0 +1,5 @@ +columns: +- recon_id +type: MULTI_SELECT +title: Recon Id +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml new file mode 100644 index 000000000..bdec6c356 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml @@ -0,0 +1,5 @@ +columns: +- source_table +type: MULTI_SELECT +title: Source Table Name +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml new file mode 100644 index 000000000..94d09e7ed --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml @@ -0,0 +1,5 @@ +columns: +- target_table +type: MULTI_SELECT +title: Target Table Name +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml new file mode 100644 index 000000000..7166bd540 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml @@ -0,0 +1,5 @@ +columns: +- source_type +type: MULTI_SELECT +title: Source Type +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml new file mode 100644 index 000000000..b8e09621e --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml @@ -0,0 +1,5 @@ +columns: +- report_type +type: MULTI_SELECT +title: Report Type +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml new file mode 100644 index 000000000..96acea41a --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml @@ -0,0 +1,5 @@ +columns: +- executed_by +type: MULTI_SELECT +title: Executed by +description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml new file mode 100644 index 000000000..4e2daa6ba --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml @@ -0,0 +1,5 @@ +columns: +- start_ts +title: Started At +description: Applied to all datasets +type: DATE_RANGE_PICKER \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml new file mode 100644 index 000000000..e0272f5c3 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml @@ -0,0 +1,4 @@ +columns: +- recon_type +type: MULTI_SELECT +title: Category \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql new file mode 100644 index 000000000..dcca05eba --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql @@ -0,0 +1,32 @@ +/* --title 'Summary Table' --width 6 --height 6 */ +SELECT main.recon_id, + main.source_type, + main.report_type, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) + ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) + END AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.run_metrics.status AS status, + metrics.run_metrics.exception_message AS exception, + metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source, + metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target, + metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch, + metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch, + metrics.recon_metrics.column_comparison.mismatch_columns AS mismatch_columns, + metrics.recon_metrics.schema_comparison AS schema_comparison, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts +FROM remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md new file mode 100644 index 000000000..ee4cb4018 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md @@ -0,0 +1,3 @@ +# Schema Comparison Details + +### This table provides a detailed view of schema mismatches. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql new file mode 100644 index 000000000..731ef4d57 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql @@ -0,0 +1,36 @@ +/* --title 'Schema Details' --width 6 */ +WITH tmp AS ( + SELECT + recon_table_id, + inserted_ts, + explode(data) AS schema_data + FROM + remorph.reconcile.details + WHERE + recon_type = 'schema' +) +SELECT + main.recon_id, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) + ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) + END AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + schema_data['source_column'] AS source_column, + schema_data['source_datatype'] AS source_datatype, + schema_data['databricks_column'] AS databricks_column, + schema_data['databricks_datatype'] AS databricks_datatype, + schema_data['is_valid'] AS is_valid +FROM + remorph.reconcile.main main + INNER JOIN tmp ON main.recon_table_id = tmp.recon_table_id +ORDER BY + tmp.inserted_ts DESC, + main.recon_id, + main.target_table diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md new file mode 100644 index 000000000..699e879d1 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md @@ -0,0 +1,3 @@ +# Drill Down + +### The details table contains all the sample records for mismatches and missing entries, providing users with exact details to pinpoint the issues. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql new file mode 100644 index 000000000..e5fad5f9a --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql @@ -0,0 +1,34 @@ +/* --height 6 --width 6 */ +WITH tmp AS ( + SELECT + recon_table_id, + inserted_ts, + recon_type, + explode(data) AS data, + row_number() OVER (PARTITION BY recon_table_id, recon_type ORDER BY recon_table_id) AS rn + FROM + remorph.reconcile.details + WHERE + recon_type != 'schema' +) +SELECT + main.recon_id, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) + ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) + END AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + recon_type, + key, + value, + rn +FROM tmp + INNER JOIN remorph.reconcile.main main + ON main.recon_table_id = tmp.recon_table_id + LATERAL VIEW explode(data) exploded_data AS key, value diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_daily_data_validation_issue_header.md similarity index 99% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_daily_data_validation_issue_header.md index 1795ecd89..a83e27ebb 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/3_0_data_validation_report.md +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_daily_data_validation_issue_header.md @@ -1,2 +1,3 @@ # Daily Data Validation Issues Report + ### This summary report provides an overview of all data validation runs conducted on a specific day. It highlights whether each table has encountered any validation issues, without delving into the low-level details. This report aims to give a quick and clear status of data integrity across all tables for the day. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml new file mode 100644 index 000000000..899e08681 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml @@ -0,0 +1,4 @@ +columns: +- start_date +type: DATE_RANGE_PICKER +width: 6 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql new file mode 100644 index 000000000..617fba669 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql @@ -0,0 +1,31 @@ +/* --title 'Number of Distinct Recon IDs per Target Table Failed' --width 6 */ +SELECT + main.recon_id, + main.source_type, + main.report_type, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + CASE + WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) + ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) + END AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.run_metrics.status AS status, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts, + DATE(main.start_ts) AS start_date +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics +ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = FALSE +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql new file mode 100644 index 000000000..fc50593b0 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql @@ -0,0 +1,10 @@ +/* --title 'Total number of runs failed' --width 2 */ +SELECT + main.recon_id AS recon_id, + DATE(main.start_ts) AS start_date +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics +ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = FALSE diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql new file mode 100644 index 000000000..f48947de5 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql @@ -0,0 +1,10 @@ +/* --title 'Unique target tables failed' --width 2 */ +SELECT + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + DATE(main.start_ts) AS start_date +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics +ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = FALSE diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql new file mode 100644 index 000000000..552f77bad --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql @@ -0,0 +1,10 @@ +/* --title 'Unique target tables successful' --width 2 */ +SELECT + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + DATE(main.start_ts) AS start_date +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics +ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = TRUE diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md new file mode 100644 index 000000000..b6b3dc271 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md @@ -0,0 +1 @@ +# Visualization of Missing and Mismatched Records \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql new file mode 100644 index 000000000..9f4f45ae8 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql @@ -0,0 +1,14 @@ +/* --title 'Mismatched Records' --width 3 */ +SELECT + main.recon_id, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch, + main.start_ts AS start_ts +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql new file mode 100644 index 000000000..f621e754c --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql @@ -0,0 +1,14 @@ +/* --title 'Threshold Mismatches' --width 3 */ +SELECT + main.recon_id, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch, + main.start_ts AS start_ts +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql new file mode 100644 index 000000000..1dbcf8073 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql @@ -0,0 +1,14 @@ +/* --title 'Missing in Databricks' --width 3 */ +SELECT + main.recon_id, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.recon_metrics.row_comparison.missing_in_target AS missing_in_target, + main.start_ts AS start_ts +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql new file mode 100644 index 000000000..f6b392d99 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql @@ -0,0 +1,14 @@ +/* --title 'Missing in Source' --width 3 */ +SELECT + main.recon_id, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + metrics.recon_metrics.row_comparison.missing_in_source AS missing_in_source, + main.start_ts AS start_ts +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md new file mode 100644 index 000000000..043ef2354 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md @@ -0,0 +1,6 @@ +# Aggregates Reconcile Table Metrics +### It provides the following information: + +* Mismatch +* Missing in Source +* Missing in Target diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql new file mode 100644 index 000000000..a35c69956 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql @@ -0,0 +1,46 @@ +/* --title 'Aggregates Summary Table' --width 6 --height 6 */ +SELECT + main.recon_id, + main.source_type, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + IF( + ISNULL(source_catalog), + CONCAT_WS('.', source_schema, source_table_name), + CONCAT_WS( + '.', + source_catalog, + source_schema, + source_table_name + ) + ) AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT_WS( + '.', + target_catalog, + target_schema, + target_table_name + ) AS target_table, + UPPER(rules.rule_info.agg_type) || CONCAT('(', rules.rule_info.agg_column, ')') AS aggregate_column, + rules.rule_info.group_by_columns, + metrics.run_metrics.status AS status, + metrics.run_metrics.exception_message AS exception, + metrics.recon_metrics.missing_in_source AS missing_in_source, + metrics.recon_metrics.missing_in_target AS missing_in_target, + metrics.recon_metrics.mismatch AS mismatch, + metrics.run_metrics.run_by_user AS executed_by, + main.start_ts AS start_ts, + main.end_ts AS end_ts +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.aggregate_metrics metrics + INNER JOIN remorph.reconcile.aggregate_rules rules + ON main.recon_table_id = metrics.recon_table_id + AND rules.rule_id = metrics.rule_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md new file mode 100644 index 000000000..80ab14eed --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md @@ -0,0 +1,2 @@ +# Drill Down +### The Aggregates Reconcile details table contains all the sample records information of mismatches and missing entries. \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql new file mode 100644 index 000000000..b08fba273 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql @@ -0,0 +1,92 @@ +/* --title 'Aggregates Reconciliation Details' --width 6 --height 6 */ +WITH details_view AS ( + SELECT + recon_table_id, + rule_id, + recon_type, + explode(data) AS agg_details + FROM + remorph.reconcile.aggregate_details +), + metrics_view AS ( + SELECT + recon_table_id, + rule_id, + recon_metrics, + run_metrics + FROM + remorph.reconcile.aggregate_metrics + ) +SELECT + recon_id, + source_table, + target_table, + recon_type, + aggregate_type, + rule AS aggregate_column, + source_value, + target_value, + zip_with(rule_group_by_columns, group_by_column_values, (groupby, value) -> CONCAT_WS(':', TRIM(groupby), value)) AS group_by_columns, + COALESCE(status, 'false') AS status +FROM ( + SELECT + main.recon_id, + main.source_table.`catalog` AS source_catalog, + main.source_table.`schema` AS source_schema, + main.source_table.table_name AS source_table_name, + IF( + ISNULL(source_catalog), + CONCAT_WS('.', source_schema, source_table_name), + CONCAT_WS( + '.', + source_catalog, + source_schema, + source_table_name + ) + ) AS source_table, + main.target_table.`catalog` AS target_catalog, + main.target_table.`schema` AS target_schema, + main.target_table.table_name AS target_table_name, + CONCAT_WS( + '.', + target_catalog, + target_schema, + target_table_name + ) AS target_table, + dtl.recon_type, + rul.rule_info.agg_type AS aggregate_type, + UPPER(rul.rule_info.agg_type) || CONCAT('(', rul.rule_info.agg_column, ')') AS rule, + CONCAT_WS( + '_', + 'source', + rul.rule_info.agg_type, + rul.rule_info.agg_column + ) AS source_agg_column, + dtl.agg_details[source_agg_column] AS source_value, + CONCAT_WS( + '_', + 'target', + rul.rule_info.agg_type, + rul.rule_info.agg_column + ) AS target_agg_column, + dtl.agg_details[target_agg_column] AS target_value, + SPLIT(rul.rule_info.group_by_columns, ',') AS rule_group_by_columns, + TRANSFORM(rule_group_by_columns, colm -> + COALESCE(dtl.agg_details[CONCAT('source_group_by_', TRIM(colm))], + dtl.agg_details[CONCAT('target_group_by_', TRIM(colm))])) AS group_by_column_values, + CONCAT_WS( + '_', + 'match', + rul.rule_info.agg_type, + rul.rule_info.agg_column + ) AS status_column, + dtl.agg_details[status_column] AS status + FROM + metrics_view mtc + INNER JOIN remorph.reconcile.main main ON main.recon_table_id = mtc.recon_table_id + INNER JOIN details_view dtl ON mtc.recon_table_id = dtl.recon_table_id + INNER JOIN remorph.reconcile.aggregate_rules rul ON mtc.rule_id = dtl.rule_id + AND dtl.rule_id = rul.rule_id + ) +ORDER BY + recon_id diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md new file mode 100644 index 000000000..b6b3dc271 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md @@ -0,0 +1 @@ +# Visualization of Missing and Mismatched Records \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql new file mode 100644 index 000000000..26b5fe9af --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql @@ -0,0 +1,19 @@ +/* --title 'Mismatched Records' --width 6 */ +SELECT + main.recon_id, + CONCAT_WS( + '.', + main.target_table.`catalog`, + main.target_table.`schema`, + main.target_table.table_name + ) AS target_table, + main.start_ts, + metrics.recon_metrics.mismatch AS mismatch +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.aggregate_metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql new file mode 100644 index 000000000..bcd0113d7 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql @@ -0,0 +1,19 @@ +/* --title 'Missing in Databricks' --width 3 */ +SELECT + main.recon_id, + CONCAT_WS( + '.', + main.target_table.`catalog`, + main.target_table.`schema`, + main.target_table.table_name + ) AS target_table, + main.start_ts, + metrics.recon_metrics.missing_in_target AS missing_in_target +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.aggregate_metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql new file mode 100644 index 000000000..4bde21239 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql @@ -0,0 +1,19 @@ +/* --title 'Missing in Source' --width 3 */ +SELECT + main.recon_id, + CONCAT_WS( + '.', + main.target_table.`catalog`, + main.target_table.`schema`, + main.target_table.table_name + ) AS target_table, + main.start_ts, + metrics.recon_metrics.missing_in_source AS missing_in_source +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.aggregate_metrics metrics + ON main.recon_table_id = metrics.recon_table_id +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml new file mode 100644 index 000000000..3893f2521 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml @@ -0,0 +1,352 @@ +display_name: "Reconciliation Metrics" +tiles: + 08_0_recon_details_pivot: + overrides: + spec: + version: 3 + widgetType: pivot + encodings: + rows: + - fieldName: recon_id + displayName: recon_id + - fieldName: source_table + displayName: source_table + - fieldName: target_table + displayName: target_table + - fieldName: recon_type + displayName: recon_type + - fieldName: rn + displayName: rn + columns: + - fieldName: key + displayName: key + cell: + fieldName: value + cellType: text + displayName: value + 11_0_failed_recon_ids: + overrides: + spec: + version: 3 + widgetType: bar + encodings: + x: + fieldName: target_table + scale: + type: categorical + sort: + by: y-reversed + displayName: target_table + 'y': + fieldName: countdistinct(recon_id) + scale: + type: quantitative + displayName: Count of Unique recon_id + label: + show: true + queries: + - name: main_query + query: + datasetName: 11_0_failed_recon_ids + fields: + - name: target_table + expression: '`target_table`' + - name: countdistinct(recon_id) + expression: COUNT(DISTINCT `recon_id`) + disaggregated: false + 12_0_total_failed_runs: + overrides: + spec: + version: 2 + widgetType: counter + encodings: + value: + fieldName: countdistinct(recon_id) + displayName: countdistinct(recon_id) + queries: + - name: main_query + query: + datasetName: 12_0_total_failed_runs + fields: + - name: countdistinct(recon_id) + expression: 'COUNT(DISTINCT `recon_id`)' + disaggregated: false + 12_1_failed_targets: + overrides: + spec: + version: 2 + widgetType: counter + encodings: + value: + fieldName: countdistinct(target_table) + displayName: countdistinct(target_table) + queries: + - name: main_query + query: + datasetName: 12_1_failed_targets + fields: + - name: countdistinct(target_table) + expression: 'COUNT(DISTINCT `target_table`)' + disaggregated: false + 12_3_successful_targets: + overrides: + spec: + version: 2 + widgetType: counter + encodings: + value: + fieldName: countdistinct(target_table) + displayName: countdistinct(target_table) + queries: + - name: main_query + query: + datasetName: 12_3_successful_targets + fields: + - name: countdistinct(target_table) + expression: 'COUNT(DISTINCT `target_table`)' + disaggregated: false + 14_0_mismatched_records: + overrides: + queries: + - name: main_query + query: + datasetName: 14_0_mismatched_records + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: absolute_mismatch + expression: '`absolute_mismatch`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: absolute_mismatch + scale: + type: quantitative + displayName: absolute_mismatch + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 14_1_threshold_mismatches: + overrides: + queries: + - name: main_query + query: + datasetName: 14_1_threshold_mismatches + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: threshold_mismatch + expression: '`threshold_mismatch`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: threshold_mismatch + scale: + type: quantitative + displayName: threshold_mismatch + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 15_0_missing_in_databricks: + overrides: + queries: + - name: main_query + query: + datasetName: 15_0_missing_in_databricks + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_target + expression: '`missing_in_target`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_target + scale: + type: quantitative + displayName: missing_in_target + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 15_1_missing_in_source: + overrides: + queries: + - name: main_query + query: + datasetName: 15_1_missing_in_source + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_source + expression: '`missing_in_source`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_source + scale: + type: quantitative + displayName: missing_in_source + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 21_0_aggr_mismatched_records: + overrides: + queries: + - name: main_query + query: + datasetName: 21_0_aggr_mismatched_records + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: mismatch + expression: '`mismatch`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: mismatch + scale: + type: quantitative + displayName: mismatch + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 22_0_aggr_missing_in_databricks: + overrides: + queries: + - name: main_query + query: + datasetName: 22_0_aggr_missing_in_databricks + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_target + expression: '`missing_in_target`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_target + scale: + type: quantitative + displayName: missing_in_target + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 22_1_aggr_missing_in_source: + overrides: + queries: + - name: main_query + query: + datasetName: 22_1_aggr_missing_in_source + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_source + expression: '`missing_in_source`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_source + scale: + type: quantitative + displayName: missing_in_source + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false \ No newline at end of file From 8ca71f9f3022f94dd6d1a3a722738d34375c26f0 Mon Sep 17 00:00:00 2001 From: Bishwajit Date: Fri, 27 Sep 2024 18:06:40 +0530 Subject: [PATCH 09/10] Add Aggregate Recon dashboard tiles --- .../00_0_aggregate_recon_header.md} | 0 .../01_0_recon_id.filter.yml | 5 + .../01_1_source_table.filter.yml | 4 + .../02_0_target_table.filter.yml | 4 + .../02_1_source_type.filter.yml | 4 + .../03_0_executed_by.filter.yml | 4 + .../03_1_started_at.filter.yml} | 1 - .../04_0_aggregate_summary_table.sql} | 0 .../05_0_aggregate_recon_drilldown_header.md} | 0 .../06_0_recon_id.filter.yml | 5 + .../06_1_category.filter.yml | 5 + .../06_2_aggregate_type.filter.yml | 5 + .../07_0_target_table.filter.yml | 4 + .../07_1_source_table.filter.yml | 4 + .../08_0_aggregate_details_table.sql} | 10 +- ...09_0_aggregate_missing_mismatch_header.md} | 0 .../10_0_aggr_mismatched_records.sql} | 0 .../11_0_aggr_missing_in_databricks.sql} | 0 .../11_1_aggr_missing_in_source.sql} | 0 .../dashboard.yml | 365 +++++++++++++ .../reconciliation_metrics/00_0_recon_main.md | 2 - .../01_0_recon_id.filter.yml | 4 +- .../01_1_source_table.filter.yml | 4 +- .../01_2_target_table.filter.yml | 5 - .../02_0_source_type.filter.yml | 5 - .../02_0_target_table.filter.yml | 5 + .../02_1_report_type.filter.yml | 5 - .../02_1_source_type.filter.yml | 4 + .../02_2_executed_by.filter.yml | 5 - .../03_0_report_type.filter.yml | 4 + .../03_1_executed_by.filter.yml | 4 + .../04_0_started_at.filter.yml | 5 + ...mmary_table.sql => 05_0_summary_table.sql} | 0 ...er.md => 06_0_schema_comparison_header.md} | 0 ...able.sql => 07_0_schema_details_table.sql} | 0 ...wn_header.md => 08_0_drill_down_header.md} | 0 .../09_0_recon_id.filter.yml | 4 + ...ry.filter.yml => 09_1_category.filter.yml} | 2 +- .../10_0_target_table.filter.yml | 4 + .../10_1_source_table.filter.yml | 4 + .../11_0_failed_recon_ids.sql | 31 -- ...pivot.sql => 11_0_recon_details_pivot.sql} | 8 +- ...2_0_daily_data_validation_issue_header.md} | 0 ...lter.yml => 13_0_success_fail_.filter.yml} | 0 .../14_0_failed_recon_ids.sql | 15 + ...ed_runs.sql => 15_0_total_failed_runs.sql} | 2 +- ...ed_targets.sql => 15_1_failed_targets.sql} | 2 +- ...argets.sql => 15_2_successful_targets.sql} | 2 +- ...der.md => 16_0_missing_mismatch_header.md} | 0 ...ecords.sql => 17_0_mismatched_records.sql} | 0 ...ches.sql => 17_1_threshold_mismatches.sql} | 0 ...cks.sql => 18_0_missing_in_databricks.sql} | 0 ..._source.sql => 18_1_missing_in_source.sql} | 0 .../reconciliation_metrics/dashboard.yml | 485 ++++++++++++------ 54 files changed, 814 insertions(+), 217 deletions(-) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/16_0_aggregate_recon_header.md => aggregate_reconciliation_metrics/00_0_aggregate_recon_header.md} (100%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/03_0_started_at.filter.yml => aggregate_reconciliation_metrics/03_1_started_at.filter.yml} (62%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/17_0_aggregate_summary_table.sql => aggregate_reconciliation_metrics/04_0_aggregate_summary_table.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md => aggregate_reconciliation_metrics/05_0_aggregate_recon_drilldown_header.md} (100%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_0_recon_id.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_1_category.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_2_aggregate_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_0_target_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_1_source_table.filter.yml rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/19_0_aggregate_details_table.sql => aggregate_reconciliation_metrics/08_0_aggregate_details_table.sql} (95%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/13_0_missing_mismatch_header.md => aggregate_reconciliation_metrics/09_0_aggregate_missing_mismatch_header.md} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/21_0_aggr_mismatched_records.sql => aggregate_reconciliation_metrics/10_0_aggr_mismatched_records.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/22_0_aggr_missing_in_databricks.sql => aggregate_reconciliation_metrics/11_0_aggr_missing_in_databricks.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/22_1_aggr_missing_in_source.sql => aggregate_reconciliation_metrics/11_1_aggr_missing_in_source.sql} (100%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{04_0_summary_table.sql => 05_0_summary_table.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{05_0_schema_comparison_header.md => 06_0_schema_comparison_header.md} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{06_0_schema_details_table.sql => 07_0_schema_details_table.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{07_0_drill_down_header.md => 08_0_drill_down_header.md} (100%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_recon_id.filter.yml rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{03_1_category.filter.yml => 09_1_category.filter.yml} (72%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_target_table.filter.yml create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_1_source_table.filter.yml delete mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{08_0_recon_details_pivot.sql => 11_0_recon_details_pivot.sql} (88%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{09_0_daily_data_validation_issue_header.md => 12_0_daily_data_validation_issue_header.md} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{10_0_success_fail_.filter.yml => 13_0_success_fail_.filter.yml} (100%) create mode 100644 src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_failed_recon_ids.sql rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{12_0_total_failed_runs.sql => 15_0_total_failed_runs.sql} (89%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{12_1_failed_targets.sql => 15_1_failed_targets.sql} (85%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{12_3_successful_targets.sql => 15_2_successful_targets.sql} (85%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{20_0_aggregate_missing_mismatch_header.md => 16_0_missing_mismatch_header.md} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{14_0_mismatched_records.sql => 17_0_mismatched_records.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{14_1_threshold_mismatches.sql => 17_1_threshold_mismatches.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{15_0_missing_in_databricks.sql => 18_0_missing_in_databricks.sql} (100%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{15_1_missing_in_source.sql => 18_1_missing_in_source.sql} (100%) diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/00_0_aggregate_recon_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_aggregate_recon_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/00_0_aggregate_recon_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml new file mode 100644 index 000000000..3673bee42 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml @@ -0,0 +1,5 @@ +columns: +- recon_id +- dd_recon_id +type: MULTI_SELECT +title: Recon Id \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml new file mode 100644 index 000000000..a91c3228b --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- source_table +type: MULTI_SELECT +title: Source Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml new file mode 100644 index 000000000..da5c628fe --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- target_table +type: MULTI_SELECT +title: Target Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml new file mode 100644 index 000000000..cf97f4070 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml @@ -0,0 +1,4 @@ +columns: +- source_type +type: MULTI_SELECT +title: Source Type \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml new file mode 100644 index 000000000..f0ccee6e7 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml @@ -0,0 +1,4 @@ +columns: +- executed_by +type: MULTI_SELECT +title: Executed by \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml similarity index 62% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml index 4e2daa6ba..85fe8d11d 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml @@ -1,5 +1,4 @@ columns: - start_ts title: Started At -description: Applied to all datasets type: DATE_RANGE_PICKER \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/04_0_aggregate_summary_table.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_aggregate_summary_table.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/04_0_aggregate_summary_table.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/05_0_aggregate_recon_drilldown_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_aggregate_recon_drilldown_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/05_0_aggregate_recon_drilldown_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_0_recon_id.filter.yml new file mode 100644 index 000000000..93cd7e61d --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_0_recon_id.filter.yml @@ -0,0 +1,5 @@ +columns: +- dd_recon_id +type: MULTI_SELECT +title: Recon Id +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_1_category.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_1_category.filter.yml new file mode 100644 index 000000000..3315ef7f9 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_1_category.filter.yml @@ -0,0 +1,5 @@ +columns: +- dd_recon_type +type: MULTI_SELECT +title: Category +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_2_aggregate_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_2_aggregate_type.filter.yml new file mode 100644 index 000000000..4be0a0e3f --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/06_2_aggregate_type.filter.yml @@ -0,0 +1,5 @@ +columns: +- dd_aggregate_type +type: MULTI_SELECT +title: Aggregate Type +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_0_target_table.filter.yml new file mode 100644 index 000000000..61888e08e --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_0_target_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- dd_target_table +type: MULTI_SELECT +title: Target Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_1_source_table.filter.yml new file mode 100644 index 000000000..31366f2b0 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/07_1_source_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- dd_source_table +type: MULTI_SELECT +title: Source Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/08_0_aggregate_details_table.sql similarity index 95% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/08_0_aggregate_details_table.sql index b08fba273..ee3f70b1b 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/19_0_aggregate_details_table.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/08_0_aggregate_details_table.sql @@ -18,11 +18,11 @@ WITH details_view AS ( remorph.reconcile.aggregate_metrics ) SELECT - recon_id, - source_table, - target_table, - recon_type, - aggregate_type, + recon_id AS dd_recon_id, + source_table AS dd_source_table, + target_table AS dd_target_table, + recon_type AS dd_recon_type, + aggregate_type AS dd_aggregate_type, rule AS aggregate_column, source_value, target_value, diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/09_0_aggregate_missing_mismatch_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_missing_mismatch_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/09_0_aggregate_missing_mismatch_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/10_0_aggr_mismatched_records.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/21_0_aggr_mismatched_records.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/10_0_aggr_mismatched_records.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_0_aggr_missing_in_databricks.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_0_aggr_missing_in_databricks.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_0_aggr_missing_in_databricks.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_1_aggr_missing_in_source.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/22_1_aggr_missing_in_source.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/11_1_aggr_missing_in_source.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml new file mode 100644 index 000000000..6802a3a71 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml @@ -0,0 +1,365 @@ +display_name: "Aggregate Reconciliation Metrics" +tiles: + 04_0_aggregate_summary_table: + overrides: + spec: + withRowNumber: true + encodings: + columns: + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: recon_id + title: recon_id + type: string + cellFormat: + default: + foregroundColor: + rules: + - if: + column: status + fn: '=' + literal: 'true' + value: + foregroundColor: '#3BD973' + - if: + column: status + fn: '=' + literal: 'false' + value: + foregroundColor: '#E92828' + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_type + title: source_type + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_catalog + title: source_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_schema + title: source_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table_name + title: source_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table + title: source_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_catalog + title: target_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_schema + title: target_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table_name + title: target_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table + title: target_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: aggregate_column + title: aggregate_column + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: group_by_columns + title: group_by_columns + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: boolean + fieldName: status + title: status + type: boolean + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: exception + title: exception + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: missing_in_source + title: missing_in_source + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: missing_in_target + title: missing_in_target + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: mismatch + title: mismatch + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: executed_by + title: executed_by + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: datetime + fieldName: start_ts + title: start_ts + type: datetime + dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS' + - booleanValues: + - 'false' + - 'true' + displayAs: datetime + fieldName: end_ts + title: end_ts + type: datetime + dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS' + 08_0_aggregate_details_table: + overrides: + spec: + withRowNumber: true + encodings: + columns: + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: dd_recon_id + title: recon_id + type: string + cellFormat: + default: + foregroundColor: + rules: + - if: + column: status + fn: '=' + literal: 'true' + value: + foregroundColor: '#3BD973' + - if: + column: status + fn: '=' + literal: 'false' + value: + foregroundColor: '#E92828' + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: dd_source_table + title: source_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: dd_target_table + title: target_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: dd_recon_type + title: recon_type + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: dd_aggregate_type + title: aggregate_type + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: aggregate_column + title: aggregate_column + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_value + title: source_value + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_value + title: target_value + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: group_by_columns + title: group_by_columns + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: boolean + fieldName: status + title: status + type: boolean + 10_0_aggr_mismatched_records: + overrides: + queries: + - name: main_query + query: + datasetName: 10_0_aggr_mismatched_records + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: mismatch + expression: '`mismatch`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: mismatch + scale: + type: quantitative + displayName: mismatch + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 11_0_aggr_missing_in_databricks: + overrides: + queries: + - name: main_query + query: + datasetName: 11_0_aggr_missing_in_databricks + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_target + expression: '`missing_in_target`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_target + scale: + type: quantitative + displayName: missing_in_target + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + 11_1_aggr_missing_in_source: + overrides: + queries: + - name: main_query + query: + datasetName: 11_1_aggr_missing_in_source + fields: + - name: target_table + expression: '`target_table`' + - name: hourly(start_ts) + expression: 'DATE_TRUNC("HOUR", `start_ts`)' + - name: missing_in_source + expression: '`missing_in_source`' + disaggregated: true + spec: + version: 3 + widgetType: area + encodings: + x: + fieldName: hourly(start_ts) + scale: + type: temporal + displayName: start_ts + 'y': + fieldName: missing_in_source + scale: + type: quantitative + displayName: missing_in_source + color: + fieldName: target_table + scale: + type: categorical + displayName: target_table + label: + show: false + + diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md index 2854d9769..aaa3cf8aa 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/00_0_recon_main.md @@ -1,5 +1,3 @@ # Main Reconciliation Table ### This table provides comprehensive information on the report's status, including failure indications, schema matching status, and details on missing and mismatched records. - -[Aggregates Reconciliation Table](#aggregates-reconcile-table-metrics) \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml index f9e5e7fb7..3673bee42 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml @@ -1,5 +1,5 @@ columns: - recon_id +- dd_recon_id type: MULTI_SELECT -title: Recon Id -description: Applied to all datasets \ No newline at end of file +title: Recon Id \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml index bdec6c356..58ed8848c 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml @@ -1,5 +1,5 @@ columns: - source_table +- dd_source_table type: MULTI_SELECT -title: Source Table Name -description: Applied to all datasets \ No newline at end of file +title: Source Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml deleted file mode 100644 index 94d09e7ed..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_target_table.filter.yml +++ /dev/null @@ -1,5 +0,0 @@ -columns: -- target_table -type: MULTI_SELECT -title: Target Table Name -description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml deleted file mode 100644 index 7166bd540..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml +++ /dev/null @@ -1,5 +0,0 @@ -columns: -- source_type -type: MULTI_SELECT -title: Source Type -description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml new file mode 100644 index 000000000..337adb9c8 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml @@ -0,0 +1,5 @@ +columns: +- target_table +- dd_target_table +type: MULTI_SELECT +title: Target Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml deleted file mode 100644 index b8e09621e..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_report_type.filter.yml +++ /dev/null @@ -1,5 +0,0 @@ -columns: -- report_type -type: MULTI_SELECT -title: Report Type -description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml new file mode 100644 index 000000000..cf97f4070 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml @@ -0,0 +1,4 @@ +columns: +- source_type +type: MULTI_SELECT +title: Source Type \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml deleted file mode 100644 index 96acea41a..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_executed_by.filter.yml +++ /dev/null @@ -1,5 +0,0 @@ -columns: -- executed_by -type: MULTI_SELECT -title: Executed by -description: Applied to all datasets \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml new file mode 100644 index 000000000..7e24cf414 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml @@ -0,0 +1,4 @@ +columns: +- report_type +type: MULTI_SELECT +title: Report Type \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml new file mode 100644 index 000000000..f0ccee6e7 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml @@ -0,0 +1,4 @@ +columns: +- executed_by +type: MULTI_SELECT +title: Executed by \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml new file mode 100644 index 000000000..ca069bef8 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml @@ -0,0 +1,5 @@ +columns: +- start_ts +title: Started At +type: DATE_RANGE_PICKER +width: 6 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_summary_table.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_comparison_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_schema_comparison_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_comparison_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/06_0_schema_details_table.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_drill_down_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_drill_down_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_drill_down_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_recon_id.filter.yml new file mode 100644 index 000000000..205b9a8ba --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_recon_id.filter.yml @@ -0,0 +1,4 @@ +columns: +- dd_recon_id +type: MULTI_SELECT +title: Recon Id \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_1_category.filter.yml similarity index 72% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_1_category.filter.yml index e0272f5c3..094ec93ab 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_category.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_1_category.filter.yml @@ -1,4 +1,4 @@ columns: -- recon_type +- dd_recon_type type: MULTI_SELECT title: Category \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_target_table.filter.yml new file mode 100644 index 000000000..61888e08e --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_target_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- dd_target_table +type: MULTI_SELECT +title: Target Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_1_source_table.filter.yml new file mode 100644 index 000000000..31366f2b0 --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_1_source_table.filter.yml @@ -0,0 +1,4 @@ +columns: +- dd_source_table +type: MULTI_SELECT +title: Source Table Name \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql deleted file mode 100644 index 617fba669..000000000 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_failed_recon_ids.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* --title 'Number of Distinct Recon IDs per Target Table Failed' --width 6 */ -SELECT - main.recon_id, - main.source_type, - main.report_type, - main.source_table.`catalog` AS source_catalog, - main.source_table.`schema` AS source_schema, - main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) - ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) - END AS source_table, - main.target_table.`catalog` AS target_catalog, - main.target_table.`schema` AS target_schema, - main.target_table.table_name AS target_table_name, - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, - metrics.run_metrics.status AS status, - metrics.run_metrics.run_by_user AS executed_by, - main.start_ts AS start_ts, - main.end_ts AS end_ts, - DATE(main.start_ts) AS start_date -FROM - remorph.reconcile.main main - INNER JOIN remorph.reconcile.metrics metrics -ON main.recon_table_id = metrics.recon_table_id -WHERE - metrics.run_metrics.status = FALSE -ORDER BY - metrics.inserted_ts DESC, - main.recon_id, - main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql similarity index 88% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql index e5fad5f9a..6c928aca1 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/08_0_recon_details_pivot.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql @@ -12,19 +12,19 @@ WITH tmp AS ( recon_type != 'schema' ) SELECT - main.recon_id, + main.recon_id AS dd_recon_id, main.source_table.`catalog` AS source_catalog, main.source_table.`schema` AS source_schema, main.source_table.table_name AS source_table_name, CASE WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) - END AS source_table, + END AS dd_source_table, main.target_table.`catalog` AS target_catalog, main.target_table.`schema` AS target_schema, main.target_table.table_name AS target_table_name, - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, - recon_type, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS dd_target_table, + recon_type AS dd_recon_type, key, value, rn diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_daily_data_validation_issue_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_daily_data_validation_issue_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/09_0_daily_data_validation_issue_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_daily_data_validation_issue_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_success_fail_.filter.yml similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/10_0_success_fail_.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/13_0_success_fail_.filter.yml diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_failed_recon_ids.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_failed_recon_ids.sql new file mode 100644 index 000000000..89228a8ad --- /dev/null +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_failed_recon_ids.sql @@ -0,0 +1,15 @@ +/* --title 'Number of Distinct Recon IDs per Target Table Failed' --width 6 */ +SELECT + main.recon_id AS rec_id, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table, + DATE(main.start_ts) AS start_date +FROM + remorph.reconcile.main main + INNER JOIN remorph.reconcile.metrics metrics +ON main.recon_table_id = metrics.recon_table_id +WHERE + metrics.run_metrics.status = FALSE +ORDER BY + metrics.inserted_ts DESC, + main.recon_id, + main.target_table.table_name diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_total_failed_runs.sql similarity index 89% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_total_failed_runs.sql index fc50593b0..71fa10faa 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_0_total_failed_runs.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_total_failed_runs.sql @@ -1,6 +1,6 @@ /* --title 'Total number of runs failed' --width 2 */ SELECT - main.recon_id AS recon_id, + main.recon_id AS rec_id, DATE(main.start_ts) AS start_date FROM remorph.reconcile.main main diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql similarity index 85% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql index f48947de5..db447c680 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_1_failed_targets.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql @@ -1,6 +1,6 @@ /* --title 'Unique target tables failed' --width 2 */ SELECT - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table, DATE(main.start_ts) AS start_date FROM remorph.reconcile.main main diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql similarity index 85% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql index 552f77bad..ac03bce80 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/12_3_successful_targets.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql @@ -1,6 +1,6 @@ /* --title 'Unique target tables successful' --width 2 */ SELECT - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table, DATE(main.start_ts) AS start_date FROM remorph.reconcile.main main diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_missing_mismatch_header.md similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/20_0_aggregate_missing_mismatch_header.md rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/16_0_missing_mismatch_header.md diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_0_mismatched_records.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/14_1_threshold_mismatches.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_missing_in_databricks.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_0_missing_in_databricks.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_0_missing_in_databricks.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_1_missing_in_source.sql similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_missing_in_source.sql rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/18_1_missing_in_source.sql diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml index 3893f2521..73fcad40b 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml @@ -1,19 +1,317 @@ display_name: "Reconciliation Metrics" tiles: - 08_0_recon_details_pivot: + 05_0_summary_table: + overrides: + spec: + withRowNumber: true + encodings: + columns: + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: recon_id + title: recon_id + type: string + cellFormat: + default: + foregroundColor: + rules: + - if: + column: status + fn: '=' + literal: 'true' + value: + foregroundColor: '#3BD973' + - if: + column: status + fn: '=' + literal: 'false' + value: + foregroundColor: '#E92828' + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_type + title: source_type + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: report_type + title: report_type + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_catalog + title: source_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_schema + title: source_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table_name + title: source_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table + title: source_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_catalog + title: target_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_schema + title: target_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table_name + title: target_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table + title: target_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: boolean + fieldName: status + title: status + type: boolean + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: exception + title: exception + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: missing_in_source + title: missing_in_source + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: missing_in_target + title: missing_in_target + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: absolute_mismatch + title: absolute_mismatch + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: threshold_mismatch + title: threshold_mismatch + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: mismatch_columns + title: mismatch_columns + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: schema_comparison + title: schema_comparison + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: executed_by + title: executed_by + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: datetime + fieldName: start_ts + title: start_ts + type: datetime + dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS' + - booleanValues: + - 'false' + - 'true' + displayAs: datetime + fieldName: end_ts + title: end_ts + type: datetime + dateTimeFormat: 'YYYY-MM-DD HH:mm:ss.SSS' + 07_0_schema_details_table: + overrides: + spec: + withRowNumber: true + encodings: + columns: + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: recon_id + title: recon_id + type: string + cellFormat: + default: + foregroundColor: + rules: + - if: + column: is_valid + fn: '=' + literal: 'false' + value: + foregroundColor: '#E92828' + - if: + column: is_valid + fn: '=' + literal: 'true' + value: + foregroundColor: '#3BD973' + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_catalog + title: source_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_schema + title: source_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table_name + title: source_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_table + title: source_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_catalog + title: target_catalog + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_schema + title: target_schema + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table_name + title: target_table_name + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: target_table + title: target_table + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_column + title: source_column + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: source_datatype + title: source_datatype + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: databricks_column + title: databricks_column + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: string + fieldName: databricks_datatype + title: databricks_datatype + type: string + - booleanValues: + - 'false' + - 'true' + displayAs: boolean + fieldName: is_valid + title: is_valid + type: boolean + 11_0_recon_details_pivot: overrides: spec: version: 3 widgetType: pivot encodings: rows: - - fieldName: recon_id + - fieldName: dd_recon_id displayName: recon_id - - fieldName: source_table + - fieldName: dd_source_table displayName: source_table - - fieldName: target_table + - fieldName: dd_target_table displayName: target_table - - fieldName: recon_type + - fieldName: dd_recon_type displayName: recon_type - fieldName: rn displayName: rn @@ -24,93 +322,93 @@ tiles: fieldName: value cellType: text displayName: value - 11_0_failed_recon_ids: + 14_0_failed_recon_ids: overrides: spec: version: 3 widgetType: bar encodings: x: - fieldName: target_table + fieldName: t_table scale: type: categorical sort: by: y-reversed - displayName: target_table + displayName: Target table 'y': - fieldName: countdistinct(recon_id) + fieldName: countdistinct(rec_id) scale: type: quantitative - displayName: Count of Unique recon_id + displayName: Count of Unique Recon Ids label: show: true queries: - name: main_query query: - datasetName: 11_0_failed_recon_ids + datasetName: 14_0_failed_recon_ids fields: - - name: target_table - expression: '`target_table`' - - name: countdistinct(recon_id) - expression: COUNT(DISTINCT `recon_id`) + - name: t_table + expression: '`t_table`' + - name: countdistinct(rec_id) + expression: COUNT(DISTINCT `rec_id`) disaggregated: false - 12_0_total_failed_runs: + 15_0_total_failed_runs: overrides: spec: version: 2 widgetType: counter encodings: value: - fieldName: countdistinct(recon_id) - displayName: countdistinct(recon_id) + fieldName: countdistinct(rec_id) + displayName: countdistinct(rec_id) queries: - name: main_query query: - datasetName: 12_0_total_failed_runs + datasetName: 15_0_total_failed_runs fields: - - name: countdistinct(recon_id) - expression: 'COUNT(DISTINCT `recon_id`)' + - name: countdistinct(rec_id) + expression: 'COUNT(DISTINCT `rec_id`)' disaggregated: false - 12_1_failed_targets: + 15_1_failed_targets: overrides: spec: version: 2 widgetType: counter encodings: value: - fieldName: countdistinct(target_table) - displayName: countdistinct(target_table) + fieldName: countdistinct(t_table) + displayName: countdistinct(t_table) queries: - name: main_query query: - datasetName: 12_1_failed_targets + datasetName: 15_1_failed_targets fields: - - name: countdistinct(target_table) - expression: 'COUNT(DISTINCT `target_table`)' + - name: countdistinct(t_table) + expression: 'COUNT(DISTINCT `t_table`)' disaggregated: false - 12_3_successful_targets: + 15_2_successful_targets: overrides: spec: version: 2 widgetType: counter encodings: value: - fieldName: countdistinct(target_table) - displayName: countdistinct(target_table) + fieldName: countdistinct(t_table) + displayName: countdistinct(t_table) queries: - name: main_query query: - datasetName: 12_3_successful_targets + datasetName: 15_2_successful_targets fields: - - name: countdistinct(target_table) - expression: 'COUNT(DISTINCT `target_table`)' + - name: countdistinct(t_table) + expression: 'COUNT(DISTINCT `t_table`)' disaggregated: false - 14_0_mismatched_records: + 17_0_mismatched_records: overrides: queries: - name: main_query query: - datasetName: 14_0_mismatched_records + datasetName: 17_0_mismatched_records fields: - name: target_table expression: '`target_table`' @@ -140,12 +438,12 @@ tiles: displayName: target_table label: show: false - 14_1_threshold_mismatches: + 17_1_threshold_mismatches: overrides: queries: - name: main_query query: - datasetName: 14_1_threshold_mismatches + datasetName: 17_1_threshold_mismatches fields: - name: target_table expression: '`target_table`' @@ -175,12 +473,12 @@ tiles: displayName: target_table label: show: false - 15_0_missing_in_databricks: + 18_0_missing_in_databricks: overrides: queries: - name: main_query query: - datasetName: 15_0_missing_in_databricks + datasetName: 18_0_missing_in_databricks fields: - name: target_table expression: '`target_table`' @@ -210,12 +508,12 @@ tiles: displayName: target_table label: show: false - 15_1_missing_in_source: + 18_1_missing_in_source: overrides: queries: - name: main_query query: - datasetName: 15_1_missing_in_source + datasetName: 18_1_missing_in_source fields: - name: target_table expression: '`target_table`' @@ -245,108 +543,3 @@ tiles: displayName: target_table label: show: false - 21_0_aggr_mismatched_records: - overrides: - queries: - - name: main_query - query: - datasetName: 21_0_aggr_mismatched_records - fields: - - name: target_table - expression: '`target_table`' - - name: hourly(start_ts) - expression: 'DATE_TRUNC("HOUR", `start_ts`)' - - name: mismatch - expression: '`mismatch`' - disaggregated: true - spec: - version: 3 - widgetType: area - encodings: - x: - fieldName: hourly(start_ts) - scale: - type: temporal - displayName: start_ts - 'y': - fieldName: mismatch - scale: - type: quantitative - displayName: mismatch - color: - fieldName: target_table - scale: - type: categorical - displayName: target_table - label: - show: false - 22_0_aggr_missing_in_databricks: - overrides: - queries: - - name: main_query - query: - datasetName: 22_0_aggr_missing_in_databricks - fields: - - name: target_table - expression: '`target_table`' - - name: hourly(start_ts) - expression: 'DATE_TRUNC("HOUR", `start_ts`)' - - name: missing_in_target - expression: '`missing_in_target`' - disaggregated: true - spec: - version: 3 - widgetType: area - encodings: - x: - fieldName: hourly(start_ts) - scale: - type: temporal - displayName: start_ts - 'y': - fieldName: missing_in_target - scale: - type: quantitative - displayName: missing_in_target - color: - fieldName: target_table - scale: - type: categorical - displayName: target_table - label: - show: false - 22_1_aggr_missing_in_source: - overrides: - queries: - - name: main_query - query: - datasetName: 22_1_aggr_missing_in_source - fields: - - name: target_table - expression: '`target_table`' - - name: hourly(start_ts) - expression: 'DATE_TRUNC("HOUR", `start_ts`)' - - name: missing_in_source - expression: '`missing_in_source`' - disaggregated: true - spec: - version: 3 - widgetType: area - encodings: - x: - fieldName: hourly(start_ts) - scale: - type: temporal - displayName: start_ts - 'y': - fieldName: missing_in_source - scale: - type: quantitative - displayName: missing_in_source - color: - fieldName: target_table - scale: - type: categorical - displayName: target_table - label: - show: false \ No newline at end of file From 9d27325daed5794422135ff9fcdadbdcac92fee0 Mon Sep 17 00:00:00 2001 From: Bishwajit Date: Mon, 30 Sep 2024 17:04:31 +0530 Subject: [PATCH 10/10] Resolve review comments --- .../labs/remorph/deployment/dashboard.py | 144 +++++++++++------- .../labs/remorph/deployment/installation.py | 8 +- .../labs/remorph/deployment/recon.py | 43 ++---- .../01_0_recon_id.filter.yml | 3 +- .../01_1_executed_by.filter.yml} | 3 +- ....filter.yml => 01_2_started_at.filter.yml} | 3 +- .../02_0_source_type.filter.yml} | 3 +- ...ilter.yml => 02_1_source_table.filter.yml} | 3 +- ...ilter.yml => 02_2_target_table.filter.yml} | 3 +- .../dashboard.yml | 8 +- .../01_0_recon_id.filter.yml | 3 +- ...filter.yml => 01_1_report_type.filter.yml} | 3 +- .../01_2_executed_by.filter.yml} | 3 +- .../02_0_source_type.filter.yml} | 3 +- ...ilter.yml => 02_1_source_table.filter.yml} | 3 +- ...ilter.yml => 02_2_target_table.filter.yml} | 3 +- ....filter.yml => 03_0_started_at.filter.yml} | 0 .../05_0_summary_table.sql | 14 +- .../07_0_schema_details_table.sql | 14 +- .../11_0_recon_details_pivot.sql | 16 +- .../15_1_failed_targets.sql | 2 +- .../15_2_successful_targets.sql | 2 +- .../17_0_mismatched_records.sql | 2 +- .../17_1_threshold_mismatches.sql | 2 +- .../reconciliation_metrics/dashboard.yml | 8 +- .../dashboards/queries/01_queries.sql | 2 +- tests/unit/deployment/test_dashboard.py | 57 +++++-- tests/unit/deployment/test_recon.py | 6 +- 28 files changed, 214 insertions(+), 150 deletions(-) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/03_1_executed_by.filter.yml => aggregate_reconciliation_metrics/01_1_executed_by.filter.yml} (60%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/{03_1_started_at.filter.yml => 01_2_started_at.filter.yml} (54%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{reconciliation_metrics/02_1_source_type.filter.yml => aggregate_reconciliation_metrics/02_0_source_type.filter.yml} (60%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/{01_1_source_table.filter.yml => 02_1_source_table.filter.yml} (56%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/{02_0_target_table.filter.yml => 02_2_target_table.filter.yml} (56%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{03_0_report_type.filter.yml => 01_1_report_type.filter.yml} (60%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{aggregate_reconciliation_metrics/03_0_executed_by.filter.yml => reconciliation_metrics/01_2_executed_by.filter.yml} (60%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/{aggregate_reconciliation_metrics/02_1_source_type.filter.yml => reconciliation_metrics/02_0_source_type.filter.yml} (60%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{01_1_source_table.filter.yml => 02_1_source_table.filter.yml} (64%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{02_0_target_table.filter.yml => 02_2_target_table.filter.yml} (64%) rename src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/{04_0_started_at.filter.yml => 03_0_started_at.filter.yml} (100%) diff --git a/src/databricks/labs/remorph/deployment/dashboard.py b/src/databricks/labs/remorph/deployment/dashboard.py index 884a738e2..5b76f09b8 100644 --- a/src/databricks/labs/remorph/deployment/dashboard.py +++ b/src/databricks/labs/remorph/deployment/dashboard.py @@ -4,20 +4,24 @@ from databricks.labs.blueprint.installation import Installation from databricks.labs.blueprint.installer import InstallState +from databricks.labs.lsql.dashboards import DashboardMetadata, Dashboards from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import ( + InvalidParameterValue, + NotFound, + DeadlineExceeded, + InternalError, + ResourceAlreadyExists, +) from databricks.sdk.retries import retried -from databricks.sdk.errors import InvalidParameterValue, NotFound, DatabricksError, ResourceAlreadyExists - -from databricks.sdk.service.dashboards import LifecycleState -from databricks.labs.lsql.dashboards import DashboardMetadata, Dashboards +from databricks.sdk.service.dashboards import LifecycleState, Dashboard -from databricks.labs.remorph.config import ReconcileMetadataConfig +from databricks.labs.remorph.config import ReconcileConfig, ReconcileMetadataConfig logger = logging.getLogger(__name__) class DashboardDeployment: - _UPLOAD_TIMEOUT = timedelta(seconds=30) def __init__( self, @@ -29,80 +33,108 @@ def __init__( self._installation = installation self._install_state = install_state - def _handle_existing_dashboard(self, dashboard_id: str, display_name: str, parent_path: str) -> str | None: - try: - dashboard = self._ws.lakeview.get(dashboard_id) - - if dashboard.lifecycle_state is None: - raise NotFound(f"Dashboard life cycle state: {display_name} ({dashboard_id})") - if dashboard.lifecycle_state == LifecycleState.TRASHED: - logger.info(f"Recreating trashed dashboard: {display_name} ({dashboard_id})") - return None # Recreate the dashboard if it is trashed (manually) - - except (NotFound, InvalidParameterValue): - logger.info(f"Recovering invalid dashboard: {display_name} ({dashboard_id})") - try: - dashboard_path = f"{parent_path}/{display_name}.lvdash.json" - self._ws.workspace.delete(dashboard_path) # Cannot recreate dashboard if file still exists - logger.debug(f"Deleted dangling dashboard {display_name} ({dashboard_id}): {dashboard_path}") - except NotFound: - pass - return None # Recreate the dashboard if it's reference is corrupted (manually) - return dashboard_id # Update the existing dashboard - def deploy( self, - name: str, folder: Path, - config: ReconcileMetadataConfig, + config: ReconcileConfig, ): - """Create a dashboard from Queries inside folder""" - logger.info(f"Deploying dashboard {name} from {folder}") + """ + Create dashboards from Dashboard metadata files. + The given folder is expected to contain subfolders each containing metadata for individual dashboards. + + :param folder: Path to the base folder. + :param config: Configuration for reconciliation. + """ + logger.info(f"Deploying dashboards from base folder {folder}") parent_path = f"{self._installation.install_folder()}/dashboards" try: self._ws.workspace.mkdirs(parent_path) except ResourceAlreadyExists: - pass - + logger.info(f"Dashboard parent path already exists: {parent_path}") + + valid_dashboard_refs = set() + for dashboard_folder in folder.iterdir(): + if not dashboard_folder.is_dir(): + continue + valid_dashboard_refs.add(self._dashboard_reference(dashboard_folder)) + dashboard = self._update_or_create_dashboard(dashboard_folder, parent_path, config.metadata_config) + logger.info( + f"Dashboard deployed with URL: {self._ws.config.host}/sql/dashboardsv3/{dashboard.dashboard_id}" + ) + self._install_state.save() + + self._remove_deprecated_dashboards(valid_dashboard_refs) + + def _dashboard_reference(self, folder: Path) -> str: + return f"{folder.stem}".lower() + + # InternalError and DeadlineExceeded are retried because of Lakeview internal issues + # These issues have been reported to and are resolved by the Lakeview team + # Keeping the retry for resilience + @retried(on=[InternalError, DeadlineExceeded], timeout=timedelta(minutes=3)) + def _update_or_create_dashboard( + self, + folder: Path, + ws_parent_path: str, + config: ReconcileMetadataConfig, + ) -> Dashboard: + logging.info(f"Reading dashboard folder {folder}") metadata = DashboardMetadata.from_path(folder).replace_database( catalog=config.catalog, - catalog_to_replace="remorph_catalog", + catalog_to_replace="remorph", database=config.schema, - database_to_replace="remorph_schema", + database_to_replace="reconcile", ) metadata.display_name = self._name_with_prefix(metadata.display_name) - - dashboard_id = self._install_state.dashboards.get(name) + reference = self._dashboard_reference(folder) + dashboard_id = self._install_state.dashboards.get(reference) if dashboard_id is not None: - dashboard_id = self._handle_existing_dashboard(dashboard_id, metadata.display_name, parent_path) - - # dashboard_data = self._substitute_params(dashboard_file, parameters or {}) - dashboard_id = self._update_or_create_dashboard(name, dashboard_id, metadata, parent_path) - logger.info(f"Dashboard deployed with dashboard_id {dashboard_id}") - logger.info(f"Dashboard URL: {self._ws.config.host}/sql/dashboardsv3/{dashboard_id}") - self._install_state.save() - - @retried(on=[DatabricksError], timeout=_UPLOAD_TIMEOUT) - def _update_or_create_dashboard( - self, - name: str, - dashboard_id: str, - metadata: DashboardMetadata, - parent_path: str, - ) -> str: + try: + dashboard_id = self._handle_existing_dashboard(dashboard_id, metadata.display_name) + except (NotFound, InvalidParameterValue): + logger.info(f"Recovering invalid dashboard: {metadata.display_name} ({dashboard_id})") + try: + dashboard_path = f"{ws_parent_path}/{metadata.display_name}.lvdash.json" + self._ws.workspace.delete(dashboard_path) # Cannot recreate dashboard if file still exists + logger.debug( + f"Deleted dangling dashboard {metadata.display_name} ({dashboard_id}): {dashboard_path}" + ) + except NotFound: + pass + dashboard_id = None # Recreate the dashboard if it's reference is corrupted (manually) dashboard = Dashboards(self._ws).create_dashboard( metadata, dashboard_id=dashboard_id, - parent_path=parent_path, + parent_path=ws_parent_path, warehouse_id=self._ws.config.warehouse_id, publish=True, ) assert dashboard.dashboard_id is not None - self._install_state.dashboards[name] = dashboard.dashboard_id - return dashboard.dashboard_id + self._install_state.dashboards[reference] = dashboard.dashboard_id + return dashboard def _name_with_prefix(self, name: str) -> str: prefix = self._installation.product() return f"[{prefix.upper()}] {name}" + + def _handle_existing_dashboard(self, dashboard_id: str, display_name: str) -> str | None: + dashboard = self._ws.lakeview.get(dashboard_id) + if dashboard.lifecycle_state is None: + raise NotFound(f"Dashboard life cycle state: {display_name} ({dashboard_id})") + if dashboard.lifecycle_state == LifecycleState.TRASHED: + logger.info(f"Recreating trashed dashboard: {display_name} ({dashboard_id})") + return None # Recreate the dashboard if it is trashed (manually) + return dashboard_id # Update the existing dashboard + + def _remove_deprecated_dashboards(self, valid_dashboard_refs: set[str]): + for ref, dashboard_id in self._install_state.dashboards.items(): + if ref not in valid_dashboard_refs: + try: + logger.info(f"Removing dashboard_id={dashboard_id}, as it is no longer needed.") + del self._install_state.dashboards[ref] + self._ws.lakeview.trash(dashboard_id) + except (InvalidParameterValue, NotFound): + logger.warning(f"Dashboard `{dashboard_id}` doesn't exist anymore for some reason.") + continue diff --git a/src/databricks/labs/remorph/deployment/installation.py b/src/databricks/labs/remorph/deployment/installation.py index 9ef6cf068..35aa6b200 100644 --- a/src/databricks/labs/remorph/deployment/installation.py +++ b/src/databricks/labs/remorph/deployment/installation.py @@ -2,16 +2,14 @@ from databricks.labs.blueprint.installation import Installation from databricks.labs.blueprint.tui import Prompts +from databricks.labs.blueprint.upgrades import Upgrades +from databricks.labs.blueprint.wheels import WheelsV2 from databricks.sdk import WorkspaceClient from databricks.sdk.errors import NotFound +from databricks.sdk.errors.platform import InvalidParameterValue from databricks.labs.remorph.config import RemorphConfigs from databricks.labs.remorph.deployment.recon import ReconDeployment -from databricks.labs.blueprint.wheels import WheelsV2 - -from databricks.sdk.errors.platform import InvalidParameterValue -from databricks.labs.blueprint.upgrades import Upgrades - logger = logging.getLogger("databricks.labs.remorph.install") diff --git a/src/databricks/labs/remorph/deployment/recon.py b/src/databricks/labs/remorph/deployment/recon.py index e6449df73..a9db3b569 100644 --- a/src/databricks/labs/remorph/deployment/recon.py +++ b/src/databricks/labs/remorph/deployment/recon.py @@ -4,9 +4,10 @@ from databricks.labs.blueprint.installation import Installation from databricks.labs.blueprint.installer import InstallState from databricks.labs.blueprint.wheels import ProductInfo +from databricks.labs.blueprint.wheels import find_project_root from databricks.sdk import WorkspaceClient from databricks.sdk.errors import InvalidParameterValue, NotFound -from databricks.labs.blueprint.wheels import find_project_root + import databricks.labs.remorph.resources from databricks.labs.remorph.config import ReconcileConfig from databricks.labs.remorph.deployment.dashboard import DashboardDeployment @@ -17,7 +18,6 @@ _RECON_PREFIX = "Reconciliation" RECON_JOB_NAME = f"{_RECON_PREFIX} Runner" -RECON_METRICS_DASHBOARD_NAME = f"{_RECON_PREFIX} Metrics" class ReconDeployment: @@ -41,7 +41,7 @@ def __init__( def install(self, recon_config: ReconcileConfig | None, wheel_paths: list[str]): if not recon_config: - logger.warning("Recon Config is empty") + logger.warning("Recon Config is empty.") return logger.info("Installing reconcile components.") self._deploy_tables(recon_config) @@ -88,44 +88,21 @@ def _deploy_tables(self, recon_config: ReconcileConfig): def _deploy_dashboards(self, recon_config: ReconcileConfig): logger.info("Deploying reconciliation dashboards.") - self._deploy_recon_metrics_dashboard(RECON_METRICS_DASHBOARD_NAME, recon_config) - for dashboard_name, dashboard_id in self._get_deprecated_dashboards(): - try: - logger.info(f"Removing dashboard_id={dashboard_id}, as it is no longer needed.") - del self._install_state.dashboards[dashboard_name] - self._ws.lakeview.trash(dashboard_id) - except (InvalidParameterValue, NotFound): - logger.warning(f"Dashboard `{dashboard_name}` doesn't exist anymore for some reason.") - continue - - def _deploy_recon_metrics_dashboard(self, name: str, recon_config: ReconcileConfig): - queries_folder = find_project_root(__file__) / "src/databricks/labs/remorph/resources/reconcile/dashboards" - logger.info(f"Creating Reconciliation Dashboard `{name}`") - self._dashboard_deployer.deploy(name, queries_folder, recon_config.metadata_config) + dashboard_base_dir = find_project_root(__file__) / "src/databricks/labs/remorph/resources/reconcile/dashboards" + self._dashboard_deployer.deploy(dashboard_base_dir, recon_config) def _get_dashboards(self) -> list[tuple[str, str]]: - return [ - (dashboard_name, dashboard_id) - for dashboard_name, dashboard_id in self._install_state.dashboards.items() - if dashboard_name.startswith(_RECON_PREFIX) - ] - - def _get_deprecated_dashboards(self) -> list[tuple[str, str]]: - return [ - (dashboard_name, dashboard_id) - for dashboard_name, dashboard_id in self._install_state.dashboards.items() - if dashboard_name.startswith(_RECON_PREFIX) and dashboard_name != RECON_METRICS_DASHBOARD_NAME - ] + return list(self._install_state.dashboards.items()) def _remove_dashboards(self): logger.info("Removing reconciliation dashboards.") - for dashboard_name, dashboard_id in self._get_dashboards(): + for dashboard_ref, dashboard_id in self._get_dashboards(): try: - logger.info(f"Removing dashboard {dashboard_name} with dashboard_id={dashboard_id}.") - del self._install_state.dashboards[dashboard_name] + logger.info(f"Removing dashboard with id={dashboard_id}.") + del self._install_state.dashboards[dashboard_ref] self._ws.lakeview.trash(dashboard_id) except (InvalidParameterValue, NotFound): - logger.warning(f"Dashboard `{dashboard_name}` doesn't exist anymore for some reason.") + logger.warning(f"Dashboard with id={dashboard_id} doesn't exist anymore for some reason.") continue def _deploy_jobs(self, recon_config: ReconcileConfig, remorph_wheel_path: str): diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml index 3673bee42..609dd9aca 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_0_recon_id.filter.yml @@ -2,4 +2,5 @@ columns: - recon_id - dd_recon_id type: MULTI_SELECT -title: Recon Id \ No newline at end of file +title: Recon Id +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_executed_by.filter.yml similarity index 60% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_executed_by.filter.yml index f0ccee6e7..7a3b36f6c 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_1_executed_by.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_executed_by.filter.yml @@ -1,4 +1,5 @@ columns: - executed_by type: MULTI_SELECT -title: Executed by \ No newline at end of file +title: Executed by +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_2_started_at.filter.yml similarity index 54% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_2_started_at.filter.yml index 85fe8d11d..3d492465c 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_1_started_at.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_2_started_at.filter.yml @@ -1,4 +1,5 @@ columns: - start_ts title: Started At -type: DATE_RANGE_PICKER \ No newline at end of file +type: DATE_RANGE_PICKER +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_source_type.filter.yml similarity index 60% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_source_type.filter.yml index cf97f4070..427a7b01e 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_type.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_source_type.filter.yml @@ -1,4 +1,5 @@ columns: - source_type type: MULTI_SELECT -title: Source Type \ No newline at end of file +title: Source Type +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_table.filter.yml similarity index 56% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_table.filter.yml index a91c3228b..bd7632fec 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/01_1_source_table.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_table.filter.yml @@ -1,4 +1,5 @@ columns: - source_table type: MULTI_SELECT -title: Source Table Name \ No newline at end of file +title: Source Table Name +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_2_target_table.filter.yml similarity index 56% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_2_target_table.filter.yml index da5c628fe..8cec7edfd 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_0_target_table.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_2_target_table.filter.yml @@ -1,4 +1,5 @@ columns: - target_table type: MULTI_SELECT -title: Target Table Name \ No newline at end of file +title: Target Table Name +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml index 6802a3a71..bc24691ea 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/dashboard.yml @@ -109,10 +109,10 @@ tiles: - booleanValues: - 'false' - 'true' - displayAs: boolean + displayAs: string fieldName: status title: status - type: boolean + type: string - booleanValues: - 'false' - 'true' @@ -252,10 +252,10 @@ tiles: - booleanValues: - 'false' - 'true' - displayAs: boolean + displayAs: string fieldName: status title: status - type: boolean + type: string 10_0_aggr_mismatched_records: overrides: queries: diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml index 3673bee42..609dd9aca 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_0_recon_id.filter.yml @@ -2,4 +2,5 @@ columns: - recon_id - dd_recon_id type: MULTI_SELECT -title: Recon Id \ No newline at end of file +title: Recon Id +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_report_type.filter.yml similarity index 60% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_report_type.filter.yml index 7e24cf414..bdb852bae 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_report_type.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_report_type.filter.yml @@ -1,4 +1,5 @@ columns: - report_type type: MULTI_SELECT -title: Report Type \ No newline at end of file +title: Report Type +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_executed_by.filter.yml similarity index 60% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_executed_by.filter.yml index f0ccee6e7..7a3b36f6c 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/03_0_executed_by.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_2_executed_by.filter.yml @@ -1,4 +1,5 @@ columns: - executed_by type: MULTI_SELECT -title: Executed by \ No newline at end of file +title: Executed by +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml similarity index 60% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml index cf97f4070..427a7b01e 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/aggregate_reconciliation_metrics/02_1_source_type.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_source_type.filter.yml @@ -1,4 +1,5 @@ columns: - source_type type: MULTI_SELECT -title: Source Type \ No newline at end of file +title: Source Type +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_table.filter.yml similarity index 64% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_table.filter.yml index 58ed8848c..8e6c442f8 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/01_1_source_table.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_1_source_table.filter.yml @@ -2,4 +2,5 @@ columns: - source_table - dd_source_table type: MULTI_SELECT -title: Source Table Name \ No newline at end of file +title: Source Table Name +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_target_table.filter.yml similarity index 64% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_target_table.filter.yml index 337adb9c8..202ae3631 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_0_target_table.filter.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/02_2_target_table.filter.yml @@ -2,4 +2,5 @@ columns: - target_table - dd_target_table type: MULTI_SELECT -title: Target Table Name \ No newline at end of file +title: Target Table Name +width: 2 \ No newline at end of file diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml similarity index 100% rename from src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/04_0_started_at.filter.yml rename to src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/03_0_started_at.filter.yml diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql index dcca05eba..4183d241a 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/05_0_summary_table.sql @@ -5,10 +5,16 @@ SELECT main.recon_id, main.source_table.`catalog` AS source_catalog, main.source_table.`schema` AS source_schema, main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) - ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) - END AS source_table, + IF( + ISNULL(source_catalog), + CONCAT_WS('.', source_schema, source_table_name), + CONCAT_WS( + '.', + source_catalog, + source_schema, + source_table_name + ) + ) AS source_table, main.target_table.`catalog` AS target_catalog, main.target_table.`schema` AS target_schema, main.target_table.table_name AS target_table_name, diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql index 731ef4d57..9f7cc115b 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/07_0_schema_details_table.sql @@ -14,10 +14,16 @@ SELECT main.source_table.`catalog` AS source_catalog, main.source_table.`schema` AS source_schema, main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) - ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) - END AS source_table, + IF( + ISNULL(source_catalog), + CONCAT_WS('.', source_schema, source_table_name), + CONCAT_WS( + '.', + source_catalog, + source_schema, + source_table_name + ) + ) AS source_table, main.target_table.`catalog` AS target_catalog, main.target_table.`schema` AS target_schema, main.target_table.table_name AS target_table_name, diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql index 6c928aca1..6edad4d9a 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/11_0_recon_details_pivot.sql @@ -1,4 +1,4 @@ -/* --height 6 --width 6 */ +/* --title 'Recon Details Drill Down' --height 6 --width 6 */ WITH tmp AS ( SELECT recon_table_id, @@ -16,10 +16,16 @@ SELECT main.source_table.`catalog` AS source_catalog, main.source_table.`schema` AS source_schema, main.source_table.table_name AS source_table_name, - CASE - WHEN COALESCE(main.source_table.catalog, '') <> '' THEN CONCAT(main.source_table.catalog, '.', main.source_table.schema, '.', main.source_table.table_name) - ELSE CONCAT(main.source_table.schema, '.', main.source_table.table_name) - END AS dd_source_table, + IF( + ISNULL(source_catalog), + CONCAT_WS('.', source_schema, source_table_name), + CONCAT_WS( + '.', + source_catalog, + source_schema, + source_table_name + ) + ) AS dd_source_table, main.target_table.`catalog` AS target_catalog, main.target_table.`schema` AS target_schema, main.target_table.table_name AS target_table_name, diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql index db447c680..ff16c4355 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_1_failed_targets.sql @@ -1,6 +1,6 @@ /* --title 'Unique target tables failed' --width 2 */ SELECT - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table, + CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS t_table, DATE(main.start_ts) AS start_date FROM remorph.reconcile.main main diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql index ac03bce80..2d2a6eb19 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/15_2_successful_targets.sql @@ -1,6 +1,6 @@ /* --title 'Unique target tables successful' --width 2 */ SELECT - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS t_table, + CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS t_table, DATE(main.start_ts) AS start_date FROM remorph.reconcile.main main diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql index 9f4f45ae8..09eea3885 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_0_mismatched_records.sql @@ -1,7 +1,7 @@ /* --title 'Mismatched Records' --width 3 */ SELECT main.recon_id, - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS target_table, metrics.recon_metrics.column_comparison.absolute_mismatch AS absolute_mismatch, main.start_ts AS start_ts FROM diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql index f621e754c..781fa2c10 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/17_1_threshold_mismatches.sql @@ -1,7 +1,7 @@ /* --title 'Threshold Mismatches' --width 3 */ SELECT main.recon_id, - CONCAT(main.target_table.catalog, '.', main.target_table.schema, '.', main.target_table.table_name) AS target_table, + CONCAT_WS('.', main.target_table.catalog, main.target_table.schema, main.target_table.table_name) AS target_table, metrics.recon_metrics.column_comparison.threshold_mismatch AS threshold_mismatch, main.start_ts AS start_ts FROM diff --git a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml index 73fcad40b..699575c3c 100644 --- a/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml +++ b/src/databricks/labs/remorph/resources/reconcile/dashboards/reconciliation_metrics/dashboard.yml @@ -102,10 +102,10 @@ tiles: - booleanValues: - 'false' - 'true' - displayAs: boolean + displayAs: string fieldName: status title: status - type: boolean + type: string - booleanValues: - 'false' - 'true' @@ -294,10 +294,10 @@ tiles: - booleanValues: - 'false' - 'true' - displayAs: boolean + displayAs: string fieldName: is_valid title: is_valid - type: boolean + type: string 11_0_recon_details_pivot: overrides: spec: diff --git a/tests/resources/dashboards/queries/01_queries.sql b/tests/resources/dashboards/queries/01_queries.sql index 376b0e3aa..6326979d6 100644 --- a/tests/resources/dashboards/queries/01_queries.sql +++ b/tests/resources/dashboards/queries/01_queries.sql @@ -5,4 +5,4 @@ main.report_type, main.source_table.`catalog` as source_catalog, main.source_table.`schema` as source_schema, main.source_table.table_name as source_table_name -FROM remorph_catalog.remorph_schema.main main +FROM remorph.reconcile.main main diff --git a/tests/unit/deployment/test_dashboard.py b/tests/unit/deployment/test_dashboard.py index 2129be0be..46c027d49 100644 --- a/tests/unit/deployment/test_dashboard.py +++ b/tests/unit/deployment/test_dashboard.py @@ -10,7 +10,7 @@ from databricks.sdk.service.dashboards import Dashboard from databricks.sdk.service.dashboards import LifecycleState -from databricks.labs.remorph.config import ReconcileMetadataConfig +from databricks.labs.remorph.config import ReconcileMetadataConfig, ReconcileConfig, DatabaseConfig from databricks.labs.remorph.deployment.dashboard import DashboardDeployment @@ -29,7 +29,7 @@ def test_deploy_dashboard(): main.source_table.`schema` AS source_schema, main.source_table.table_name AS source_table_name\nFROM remorph.reconcile.main AS main""".strip() - dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards") dashboard = Dashboard( dashboard_id="9c1fbf4ad3449be67d6cb64c8acc730b", display_name="Remorph-Reconciliation", @@ -37,18 +37,28 @@ def test_deploy_dashboard(): ws.lakeview.create.return_value = dashboard installation = MockInstallation(is_global=False) install_state = InstallState.from_installation(installation) - name = "Remorph-Reconciliation" dashboard_publisher = DashboardDeployment(ws, installation, install_state) - dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) + reconcile_config = ReconcileConfig( + data_source="oracle", + report_type="all", + secret_scope="remorph_oracle69", + database_config=DatabaseConfig( + source_schema="tpch_sf100069", + target_catalog="tpch69", + target_schema="1000gb69", + ), + metadata_config=ReconcileMetadataConfig(), + ) + dashboard_publisher.deploy(dashboard_folder, reconcile_config) _, kwargs = ws.lakeview.create.call_args query = _get_dashboard_query(kwargs) assert query == expected_query - assert install_state.dashboards[name] == dashboard.dashboard_id + assert install_state.dashboards["queries"] == dashboard.dashboard_id @pytest.mark.parametrize("exception", [InvalidParameterValue, NotFound]) def test_recovery_invalid_dashboard(caplog, exception): - dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards") ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" @@ -58,19 +68,30 @@ def test_recovery_invalid_dashboard(caplog, exception): ) ws.lakeview.create.return_value = dashboard ws.lakeview.get.side_effect = exception - name = "Remorph-Reconciliation" + # name = "Remorph-Reconciliation" installation = MockInstallation( { "state.json": { - "resources": {"dashboards": {name: "8c1fbf4ad3449be67d6cb64c8acc730b"}}, + "resources": {"dashboards": {"queries": "8c1fbf4ad3449be67d6cb64c8acc730b"}}, "version": 1, }, } ) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) + reconcile_config = ReconcileConfig( + data_source="oracle", + report_type="all", + secret_scope="remorph_oracle66", + database_config=DatabaseConfig( + source_schema="tpch_sf100066", + target_catalog="tpch66", + target_schema="1000gb66", + ), + metadata_config=ReconcileMetadataConfig(), + ) with caplog.at_level(logging.DEBUG, logger="databricks.labs.remorph.deployment.dashboard"): - dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) + dashboard_publisher.deploy(dashboard_folder, reconcile_config) assert "Recovering invalid dashboard" in caplog.text assert "Deleted dangling dashboard" in caplog.text ws.workspace.delete.assert_called() @@ -79,7 +100,7 @@ def test_recovery_invalid_dashboard(caplog, exception): def test_recovery_trashed_dashboard(caplog): - dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards/queries") + dashboard_folder = Path(__file__).parent / Path("../../resources/dashboards") ws = create_autospec(WorkspaceClient) dashboard_id = "9c1fbf4ad3449be67d6cb64c8acc730b" @@ -89,19 +110,29 @@ def test_recovery_trashed_dashboard(caplog): ) ws.lakeview.create.return_value = dashboard ws.lakeview.get.return_value = Dashboard(lifecycle_state=LifecycleState.TRASHED) - name = "Remorph-Reconciliation" installation = MockInstallation( { "state.json": { - "resources": {"dashboards": {name: "8c1fbf4ad3449be67d6cb64c8acc730b"}}, + "resources": {"dashboards": {"queries": "8c1fbf4ad3449be67d6cb64c8acc730b"}}, "version": 1, }, } ) install_state = InstallState.from_installation(installation) dashboard_publisher = DashboardDeployment(ws, installation, install_state) + reconcile_config = ReconcileConfig( + data_source="oracle", + report_type="all", + secret_scope="remorph_oracle77", + database_config=DatabaseConfig( + source_schema="tpch_sf100077", + target_catalog="tpch77", + target_schema="1000gb77", + ), + metadata_config=ReconcileMetadataConfig(), + ) with caplog.at_level(logging.DEBUG, logger="databricks.labs.remorph.deployment.dashboard"): - dashboard_publisher.deploy(name, dashboard_folder, ReconcileMetadataConfig()) + dashboard_publisher.deploy(dashboard_folder, reconcile_config) assert "Recreating trashed dashboard" in caplog.text ws.lakeview.create.assert_called() ws.lakeview.update.assert_not_called() diff --git a/tests/unit/deployment/test_recon.py b/tests/unit/deployment/test_recon.py index ea90d6be0..4f8a457eb 100644 --- a/tests/unit/deployment/test_recon.py +++ b/tests/unit/deployment/test_recon.py @@ -116,9 +116,6 @@ def raise_invalid_parameter_err_for_job(jid: str): assert "Reconciliation Deprecated Job 1" not in install_state.jobs assert "Reconciliation Deprecated Job 2" not in install_state.jobs assert "Some other Job" in install_state.jobs - assert "Reconciliation Deprecated Dashboard 1" not in install_state.dashboards - assert "Reconciliation Deprecated Dashboard 2" not in install_state.dashboards - assert "Some other Dashboard" in install_state.dashboards def test_uninstall_missing_config(ws): @@ -211,5 +208,4 @@ def raise_invalid_parameter_err_for_job(jid: str): assert "Reconciliation Runner" not in install_state.jobs assert "Some other Job" in install_state.jobs - assert "Reconciliation Metrics" not in install_state.dashboards - assert "Some other Dashboard" in install_state.dashboards + assert len(install_state.dashboards.keys()) == 0