Skip to content

Commit

Permalink
[dc] Updates log format in azure_log gov collection
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrey Fedorov authored and sfc-gh-afedorov committed Nov 22, 2019
1 parent 199e962 commit adc2d73
Showing 1 changed file with 3 additions and 16 deletions.
19 changes: 3 additions & 16 deletions src/connectors/azure_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,21 +307,6 @@ def connect(connection_name, options):
sql=refresh_task_sql,
)

select_statement_sql = {
'reg': (
f"SELECT value "
f"FROM data.{base_name}_external "
f"WHERE timestamp_part >= DATEADD(HOUR, -2, CURRENT_TIMESTAMP())"
),
'gov': (
f"SELECT value FROM ("
f" SELECT value AS a "
f" FROM data.{base_name}_external"
f" WHERE timestamp_part >= DATEADD(HOUR, -2, CURRENT_TIMESTAMP())"
f"), LATERAL FLATTEN (INPUT => a:records)"
),
}

insert_task_sql = {
'operation': f"""
INSERT (
Expand Down Expand Up @@ -401,7 +386,9 @@ def connect(connection_name, options):
ingest_task_sql = f"""
MERGE INTO data.{base_name}_connection a
USING (
{select_statement_sql[cloud_type]}
SELECT value
FROM data.{base_name}_external
WHERE timestamp_part >= DATEADD(HOUR, -2, CURRENT_TIMESTAMP)
) b
ON a.raw = b.value
WHEN NOT MATCHED THEN
Expand Down

0 comments on commit adc2d73

Please sign in to comment.