From f53d0242c4e7a7ebc1ab46a1948b616437b6a0db Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:49:59 -0800 Subject: [PATCH 01/15] started --- CHANGELOG.md | 4 + dbt_project.yml | 2 +- models/src_zendesk.yml | 2 +- models/stg_zendesk.yml | 135 ++++++++++++++++-- models/stg_zendesk__brand.sql | 9 +- models/stg_zendesk__daylight_time.sql | 8 +- models/stg_zendesk__domain_name.sql | 9 +- models/stg_zendesk__group.sql | 12 +- models/stg_zendesk__organization.sql | 8 +- models/stg_zendesk__organization_tag.sql | 9 +- models/stg_zendesk__schedule.sql | 7 +- models/stg_zendesk__schedule_holiday.sql | 10 +- models/stg_zendesk__ticket.sql | 8 +- models/stg_zendesk__ticket_comment.sql | 9 +- models/stg_zendesk__ticket_field_history.sql | 9 +- models/stg_zendesk__ticket_form_history.sql | 11 +- models/stg_zendesk__ticket_schedule.sql | 9 +- models/stg_zendesk__ticket_tag.sql | 13 +- models/stg_zendesk__time_zone.sql | 8 +- models/stg_zendesk__user.sql | 9 +- models/stg_zendesk__user_tag.sql | 9 +- models/tmp/stg_zendesk__brand_tmp.sql | 14 +- models/tmp/stg_zendesk__daylight_time_tmp.sql | 14 +- models/tmp/stg_zendesk__domain_name_tmp.sql | 14 +- models/tmp/stg_zendesk__group_tmp.sql | 14 +- .../tmp/stg_zendesk__organization_tag_tmp.sql | 14 +- models/tmp/stg_zendesk__organization_tmp.sql | 14 +- .../tmp/stg_zendesk__schedule_holiday_tmp.sql | 14 +- models/tmp/stg_zendesk__schedule_tmp.sql | 14 +- .../tmp/stg_zendesk__ticket_comment_tmp.sql | 14 +- .../stg_zendesk__ticket_field_history_tmp.sql | 14 +- .../stg_zendesk__ticket_form_history_tmp.sql | 14 +- .../tmp/stg_zendesk__ticket_schedule_tmp.sql | 33 ++--- models/tmp/stg_zendesk__ticket_tag_tmp.sql | 14 +- models/tmp/stg_zendesk__ticket_tmp.sql | 14 +- models/tmp/stg_zendesk__time_zone_tmp.sql | 14 +- models/tmp/stg_zendesk__user_tag_tmp.sql | 14 +- models/tmp/stg_zendesk__user_tmp.sql | 14 +- packages.yml | 11 +- 39 files changed, 474 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890e2d4..0276670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# dbt_zendesk_source v0.11.0 + +Unioning.... + # dbt_zendesk_source v0.10.0 [PR #42](https://github.com/fivetran/dbt_zendesk_source/pull/42) introduces the following updates: diff --git a/dbt_project.yml b/dbt_project.yml index bc1d385..681439e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'zendesk_source' -version: '0.10.0' +version: '0.11.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: zendesk_source: diff --git a/models/src_zendesk.yml b/models/src_zendesk.yml index eadd496..cc5ef4e 100644 --- a/models/src_zendesk.yml +++ b/models/src_zendesk.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: zendesk - schema: "{{var ('zendesk_schema', 'zendesk')}}" + schema: "{{ var('zendesk_schema', 'zendesk') if var('zendesk_union_schemas', []) == [] else var('zendesk_union_schemas', [])[0] }}" database: "{% if target.type != 'spark'%}{{ var('zendesk_database', target.database) }}{% endif %}" loader: fivetran loaded_at_field: _fivetran_synced diff --git a/models/stg_zendesk.yml b/models/stg_zendesk.yml index 3214663..939b525 100644 --- a/models/stg_zendesk.yml +++ b/models/stg_zendesk.yml @@ -5,11 +5,15 @@ models: description: > Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_id + - source_relation columns: - name: ticket_id description: Automatically assigned when the ticket is created tests: - - unique - not_null - name: url description: The API url of this ticket @@ -58,17 +62,25 @@ models: - name: source_to_address description: The address of the source the ticket was created from - name: source_to_name - description: The name of the source the ticket was created from + description: The name of the source the ticket was created from + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__brand description: > Brands are your customer-facing identities. They might represent multiple products or services, or they might literally be multiple brands owned and represented by your company. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - brand_id + - source_relation columns: - name: brand_id description: The ID automatically assigned when the brand is created tests: - - unique - not_null - name: brand_url description: The url of the brand @@ -78,6 +90,10 @@ models: description: The subdomain of the brand - name: active description: If the brand is set as active + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__domain_name description: Domain names associated with an organization. An organization may have multiple domain names. @@ -90,6 +106,10 @@ models: description: The name of the domain associated with the organization - name: index description: Index number of the domain name associated with the organization + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__group description: > @@ -97,14 +117,22 @@ models: element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being assigned to a Group. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - group_id + - source_relation columns: - name: group_id description: Automatically assigned when creating groups tests: - - unique - not_null - name: name description: The name of the group + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__organization_tag description: The tags associated with an organization. An organization may have multiple tags. @@ -113,6 +141,10 @@ models: description: Reference to the organization - name: tag description: Tag associated with the organization + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__organization description: > @@ -120,24 +152,36 @@ models: organizations. You can manually assign customers to an organization or automatically assign them to an organization by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or to send email notifications. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - organization_id + - source_relation columns: - name: organization_id description: Automatically assigned when the organization is created tests: - - unique - not_null - name: name description: A unique name for the organization - name: details description: Any details obout the organization, such as the address + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_comment description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_comment_id + - source_relation columns: - name: ticket_comment_id description: Automatically assigned when the comment is created tests: - - unique - not_null - name: body description: The comment string @@ -155,6 +199,10 @@ models: description: Boolean field indicating if the comment is a twitter tweet - name: is_voice_comment description: Boolean field indicating if the comment is a voice comment + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__user_tag description: Table containing all tags associated with a user. Only present if your account has user tagging enabled. @@ -163,14 +211,22 @@ models: description: Reference to the user - name: tag description: Tag associated with the user + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__user description: Zendesk has three types of users, end-users (your customers), agents, and administrators. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - user_id + - source_relation columns: - name: user_id description: Automatically assigned when the user is created tests: - - unique - not_null - name: email description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address @@ -188,12 +244,23 @@ models: description: The user's time zone. See Time Zone - name: ticket_restriction description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__schedule description: The support schedules created with different business hours and holidays. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - schedule_id + - source_relation columns: - name: schedule_id description: ID automatically assigned to the schedule upon creation + tests: + - not_null - name: schedule_name description: Name of the schedule - name: created_at @@ -204,7 +271,11 @@ models: description: End time of the schedule, in the schedule's time zone. - name: time_zone description: Timezone in which the schedule operates. - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. + - name: stg_zendesk__ticket_schedule description: The schedules applied to tickets through a trigger. columns: @@ -214,7 +285,11 @@ models: description: The time the schedule was assigned to the ticket - name: schedule_id description: The ID of the schedule applied to the ticket - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. + - name: stg_zendesk__ticket_form_history description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. columns: @@ -232,6 +307,10 @@ models: description: If the form is set as active - name: name description: The name of the form + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_tag description: > @@ -242,7 +321,10 @@ models: description: The ID of the ticket associated with the tag - name: tags description: The tag, or word(s), associated with the ticket - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_field_history description: All fields and field values associated with tickets. @@ -259,6 +341,10 @@ models: description: The value of the field - name: user_id description: The id of the user who made the update + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__daylight_time description: > @@ -267,7 +353,8 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - time_zone - - year + - year + - source_relation columns: - name: daylight_end_utc description: UTC timestamp of when Daylight Time ended in this year. @@ -281,33 +368,53 @@ models: description: Year in which daylight savings occurred. - name: daylight_offset_minutes description: Number of **minutes** added during Daylight Savings Time. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__time_zone description: Offsets (from UTC) for each timezone. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - time_zone + - source_relation columns: - name: time_zone description: Name of the time zone. tests: - - unique - not_null - name: standard_offset description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. - name: standard_offset_minutes description: Standard offset of the timezone (non-daylight savings hours) in minutes. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__schedule_holiday description: Information about holidays for each specified schedule. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - holiday_id + - source_relation columns: - name: end_date_at description: ISO 8601 representation of the holiday end date. - name: holiday_id description: The ID of the scheduled holiday. tests: - - unique - not_null - name: holiday_name description: Name of the holiday. - name: schedule_id description: The ID of the schedule. - name: start_date_at - description: ISO 8601 representation of the holiday start date. \ No newline at end of file + description: ISO 8601 representation of the holiday start date. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. \ No newline at end of file diff --git a/models/stg_zendesk__brand.sql b/models/stg_zendesk__brand.sql index 48ca08a..90d7f8f 100644 --- a/models/stg_zendesk__brand.sql +++ b/models/stg_zendesk__brand.sql @@ -21,6 +21,11 @@ fields as ( staging_columns=get_brand_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} from base ), @@ -32,7 +37,9 @@ final as ( brand_url, name, subdomain, - active as is_active + active as is_active, + source_relation + from fields where not coalesce(_fivetran_deleted, false) ) diff --git a/models/stg_zendesk__daylight_time.sql b/models/stg_zendesk__daylight_time.sql index bd132b8..a0c38a8 100644 --- a/models/stg_zendesk__daylight_time.sql +++ b/models/stg_zendesk__daylight_time.sql @@ -24,6 +24,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -35,7 +40,8 @@ final as ( daylight_start_utc, time_zone, year, - daylight_offset * 60 as daylight_offset_minutes + daylight_offset * 60 as daylight_offset_minutes, + source_relation from fields ) diff --git a/models/stg_zendesk__domain_name.sql b/models/stg_zendesk__domain_name.sql index f48e154..69a3924 100644 --- a/models/stg_zendesk__domain_name.sql +++ b/models/stg_zendesk__domain_name.sql @@ -23,6 +23,11 @@ fields as ( staging_columns=get_domain_name_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} from base ), @@ -32,7 +37,9 @@ final as ( select organization_id, domain_name, - index + index, + source_relation + from fields ) diff --git a/models/stg_zendesk__group.sql b/models/stg_zendesk__group.sql index c85af24..37a90b3 100644 --- a/models/stg_zendesk__group.sql +++ b/models/stg_zendesk__group.sql @@ -21,7 +21,12 @@ fields as ( staging_columns=get_group_columns() ) }} - + + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -29,9 +34,10 @@ final as ( select id as group_id, - name + name, + source_relation + from fields - where not coalesce(_fivetran_deleted, false) ) diff --git a/models/stg_zendesk__organization.sql b/models/stg_zendesk__organization.sql index 31886d5..1a84813 100644 --- a/models/stg_zendesk__organization.sql +++ b/models/stg_zendesk__organization.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -33,7 +38,8 @@ final as ( updated_at, details, name, - external_id + external_id, + source_relation from fields ) diff --git a/models/stg_zendesk__organization_tag.sql b/models/stg_zendesk__organization_tag.sql index a69ff86..3e37c5e 100644 --- a/models/stg_zendesk__organization_tag.sql +++ b/models/stg_zendesk__organization_tag.sql @@ -24,6 +24,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -36,7 +41,9 @@ final as ( {% else %} tag {% endif %} - as tags + as tags, + source_relation + from fields ) diff --git a/models/stg_zendesk__schedule.sql b/models/stg_zendesk__schedule.sql index 6ab434b..4b74dda 100644 --- a/models/stg_zendesk__schedule.sql +++ b/models/stg_zendesk__schedule.sql @@ -24,6 +24,10 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} from base ), @@ -35,7 +39,8 @@ final as ( start_time, name as schedule_name, created_at, - time_zone + time_zone, + source_relation from fields where not coalesce(_fivetran_deleted, false) diff --git a/models/stg_zendesk__schedule_holiday.sql b/models/stg_zendesk__schedule_holiday.sql index cc14f94..361aca7 100644 --- a/models/stg_zendesk__schedule_holiday.sql +++ b/models/stg_zendesk__schedule_holiday.sql @@ -16,6 +16,12 @@ fields as ( staging_columns=get_schedule_holiday_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -28,7 +34,9 @@ final as ( cast(id as {{ dbt.type_string() }} ) as holiday_id, name as holiday_name, cast(schedule_id as {{ dbt.type_string() }} ) as schedule_id, - cast(start_date as {{ dbt.type_timestamp() }} ) as holiday_start_date_at + cast(start_date as {{ dbt.type_timestamp() }} ) as holiday_start_date_at, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket.sql b/models/stg_zendesk__ticket.sql index cbe76cb..ce700f3 100644 --- a/models/stg_zendesk__ticket.sql +++ b/models/stg_zendesk__ticket.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + --The below script allows for pass through columns. {% if var('zendesk__ticket_passthrough_columns',[]) != [] %} , @@ -67,7 +72,8 @@ final as ( via_source_from_title as source_from_title, via_source_rel as source_rel, via_source_to_address as source_to_address, - via_source_to_name as source_to_name + via_source_to_name as source_to_name, + source_relation --The below script allows for pass through columns. {% if var('zendesk__ticket_passthrough_columns',[]) != [] %} diff --git a/models/stg_zendesk__ticket_comment.sql b/models/stg_zendesk__ticket_comment.sql index ba8e9db..79ed94b 100644 --- a/models/stg_zendesk__ticket_comment.sql +++ b/models/stg_zendesk__ticket_comment.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -41,7 +46,9 @@ final as ( user_id, facebook_comment as is_facebook_comment, tweet as is_tweet, - voice_comment as is_voice_comment + voice_comment as is_voice_comment, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index 1d0829b..7cc332c 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -38,7 +43,9 @@ final as ( lead(updated) over (partition by ticket_id, field_name order by updated) as valid_ending_at, {% endif %} value, - user_id + user_id, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket_form_history.sql b/models/stg_zendesk__ticket_form_history.sql index 07342bc..c7b00ab 100644 --- a/models/stg_zendesk__ticket_form_history.sql +++ b/models/stg_zendesk__ticket_form_history.sql @@ -23,7 +23,12 @@ fields as ( staging_columns=get_ticket_form_history_columns() ) }} - + + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -40,7 +45,9 @@ final as ( {% endif %} display_name, active as is_active, - name + name, + source_relation + from fields where not coalesce(_fivetran_deleted, false) diff --git a/models/stg_zendesk__ticket_schedule.sql b/models/stg_zendesk__ticket_schedule.sql index ab81fc8..6307ccf 100644 --- a/models/stg_zendesk__ticket_schedule.sql +++ b/models/stg_zendesk__ticket_schedule.sql @@ -24,6 +24,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -36,7 +41,9 @@ final as ( {% else -%} created_at, {% endif %} - cast(schedule_id as {{ dbt.type_string() }}) as schedule_id --need to convert from numeric to string for downstream models to work properly + cast(schedule_id as {{ dbt.type_string() }}) as schedule_id, --need to convert from numeric to string for downstream models to work properly + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket_tag.sql b/models/stg_zendesk__ticket_tag.sql index d44c37b..0684f01 100644 --- a/models/stg_zendesk__ticket_tag.sql +++ b/models/stg_zendesk__ticket_tag.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -30,10 +35,12 @@ final as ( select ticket_id, {% if target.type == 'redshift' %} - "tag" as tags + "tag" {% else %} - tag as tags - {% endif %} + tag + {% endif %} as tags, + source_relation + from fields ) diff --git a/models/stg_zendesk__time_zone.sql b/models/stg_zendesk__time_zone.sql index 76c10fa..51d84ce 100644 --- a/models/stg_zendesk__time_zone.sql +++ b/models/stg_zendesk__time_zone.sql @@ -18,6 +18,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -30,7 +35,8 @@ final as ( -- let's convert it to an integer value of minutes cast( {{ dbt.split_part(string_text='standard_offset', delimiter_text="':'", part_number=1) }} as {{ dbt.type_int() }} ) * 60 + (cast( {{ dbt.split_part(string_text='standard_offset', delimiter_text="':'", part_number=2) }} as {{ dbt.type_int() }} ) * - (case when standard_offset like '-%' then -1 else 1 end) ) as standard_offset_minutes + (case when standard_offset like '-%' then -1 else 1 end) ) as standard_offset_minutes, + source_relation from fields ) diff --git a/models/stg_zendesk__user.sql b/models/stg_zendesk__user.sql index 4aa19cd..67018b9 100644 --- a/models/stg_zendesk__user.sql +++ b/models/stg_zendesk__user.sql @@ -22,6 +22,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -48,7 +53,9 @@ final as ( time_zone, locale, active as is_active, - suspended as is_suspended + suspended as is_suspended, + source_relation + from fields ) diff --git a/models/stg_zendesk__user_tag.sql b/models/stg_zendesk__user_tag.sql index fc194bc..702b0be 100644 --- a/models/stg_zendesk__user_tag.sql +++ b/models/stg_zendesk__user_tag.sql @@ -24,6 +24,11 @@ fields as ( ) }} + {{ fivetran_utils.source_relation( + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases') + }} + from base ), @@ -36,7 +41,9 @@ final as ( {% else %} tag {% endif %} - as tags + as tags, + source_relation + from fields ) diff --git a/models/tmp/stg_zendesk__brand_tmp.sql b/models/tmp/stg_zendesk__brand_tmp.sql index 3dbc310..a4f6e9f 100644 --- a/models/tmp/stg_zendesk__brand_tmp.sql +++ b/models/tmp/stg_zendesk__brand_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk','brand')) }} -from {{ source('zendesk','brand') }} as brand_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='brand', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='brand', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__daylight_time_tmp.sql b/models/tmp/stg_zendesk__daylight_time_tmp.sql index 63313a1..40b5eff 100644 --- a/models/tmp/stg_zendesk__daylight_time_tmp.sql +++ b/models/tmp/stg_zendesk__daylight_time_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'daylight_time')) }} -from {{ source('zendesk', 'daylight_time') }} as daylight_time_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='daylight_time', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='daylight_time', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__domain_name_tmp.sql b/models/tmp/stg_zendesk__domain_name_tmp.sql index 719ad1b..c1d8225 100644 --- a/models/tmp/stg_zendesk__domain_name_tmp.sql +++ b/models/tmp/stg_zendesk__domain_name_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_domain_names variable within your dbt_project.yml file to False. {{ config(enabled=var('using_domain_names', True)) }} -select {{ dbt_utils.star(source('zendesk', 'domain_name')) }} -from {{ source('zendesk', 'domain_name') }} as domain_name_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='domain_name', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='domain_name', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__group_tmp.sql b/models/tmp/stg_zendesk__group_tmp.sql index b38eb34..75316a6 100644 --- a/models/tmp/stg_zendesk__group_tmp.sql +++ b/models/tmp/stg_zendesk__group_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk','group')) }} -from {{ source('zendesk','group') }} as group_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='group', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='group', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tag_tmp.sql b/models/tmp/stg_zendesk__organization_tag_tmp.sql index a824a3e..a8a7f60 100644 --- a/models/tmp/stg_zendesk__organization_tag_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tag_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_organization_tags variable within your dbt_project.yml file to False. {{ config(enabled=var('using_organization_tags', True)) }} -select {{ dbt_utils.star(source('zendesk','organization_tag')) }} -from {{ source('zendesk','organization_tag') }} as organization_tag_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='organization_tag', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='organization_tag', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tmp.sql b/models/tmp/stg_zendesk__organization_tmp.sql index 48aa3c8..4bb4016 100644 --- a/models/tmp/stg_zendesk__organization_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk', 'organization')) }} -from {{ source('zendesk','organization') }} as organization_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='organization', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='organization', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql index 1fae51d..fdf826b 100644 --- a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'schedule_holiday')) }} -from {{ source('zendesk', 'schedule_holiday') }} as schedule_holiday_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='schedule_holiday', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='schedule_holiday', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_tmp.sql b/models/tmp/stg_zendesk__schedule_tmp.sql index bdd21e7..47f44e0 100644 --- a/models/tmp/stg_zendesk__schedule_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'schedule')) }} -from {{ source('zendesk', 'schedule') }} as schedule_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='schedule', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='schedule', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_comment_tmp.sql b/models/tmp/stg_zendesk__ticket_comment_tmp.sql index 84d6c6a..b74d250 100644 --- a/models/tmp/stg_zendesk__ticket_comment_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_comment_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_comment')) }} -from {{ source('zendesk', 'ticket_comment') }} as ticket_comment_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket_comment', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket_comment', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql index 6b61ed8..9a66b99 100644 --- a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_field_history')) }} -from {{ source('zendesk', 'ticket_field_history') }} as ticket_field_history_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket_field_history', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket_field_history', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql index 60342b8..1026cae 100644 --- a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_ticket_form_history variable within your dbt_project.yml file to False. {{ config(enabled=var('using_ticket_form_history', True)) }} -select {{ dbt_utils.star(source('zendesk', 'ticket_form_history')) }} -from {{ source('zendesk', 'ticket_form_history') }} as ticket_form_history_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket_form_history', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket_form_history', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql index 2345b52..6e85076 100644 --- a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql @@ -1,24 +1,15 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -{%- set source_relation = adapter.get_relation( - database=source('zendesk', 'ticket_schedule').database, - schema=source('zendesk', 'ticket_schedule').schema, - identifier=source('zendesk', 'ticket_schedule').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} - -select {{ dbt_utils.star(source('zendesk', 'ticket_schedule')) }} -from {{ source('zendesk', 'ticket_schedule') }} as ticket_schedule_table - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_timestamp() }}) as created_at, - cast(null as {{ dbt.type_int() }}) as schedule_id, - cast(null as {{ dbt.type_int() }}) as ticket_id - -{% endif %} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket_schedule', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket_schedule', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tag_tmp.sql b/models/tmp/stg_zendesk__ticket_tag_tmp.sql index c2a3ace..b408936 100644 --- a/models/tmp/stg_zendesk__ticket_tag_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tag_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_tag')) }} -from {{ source('zendesk', 'ticket_tag') }} as ticket_tag_table +{{ + fivetran_utils.union_data( + table_identifier='ticket_tag', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket_tag', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tmp.sql b/models/tmp/stg_zendesk__ticket_tmp.sql index 21c9bab..f986522 100644 --- a/models/tmp/stg_zendesk__ticket_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket')) }} -from {{ source('zendesk', 'ticket') }} as ticket_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='ticket', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__time_zone_tmp.sql b/models/tmp/stg_zendesk__time_zone_tmp.sql index 17c58a2..dd509d7 100644 --- a/models/tmp/stg_zendesk__time_zone_tmp.sql +++ b/models/tmp/stg_zendesk__time_zone_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'time_zone')) }} -from {{ source('zendesk', 'time_zone') }} as time_zone_table +{{ + fivetran_utils.union_data( + table_identifier='time_zone', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='time_zone', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tag_tmp.sql b/models/tmp/stg_zendesk__user_tag_tmp.sql index 94f7784..6b2922b 100644 --- a/models/tmp/stg_zendesk__user_tag_tmp.sql +++ b/models/tmp/stg_zendesk__user_tag_tmp.sql @@ -1,5 +1,15 @@ --To disable this model, set the using_user_tags variable within your dbt_project.yml file to False. {{ config(enabled=var('using_user_tags', True)) }} -select {{ dbt_utils.star(source('zendesk','user_tag')) }} -from {{ source('zendesk','user_tag') }} as user_tag_table +{{ + fivetran_utils.union_data( + table_identifier='user_tag', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='user_tag', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tmp.sql b/models/tmp/stg_zendesk__user_tmp.sql index 7e996ea..9d72a6d 100644 --- a/models/tmp/stg_zendesk__user_tmp.sql +++ b/models/tmp/stg_zendesk__user_tmp.sql @@ -1,2 +1,12 @@ -select {{ dbt_utils.star(source('zendesk','user')) }} -from {{ source('zendesk','user') }} as user_table \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='user', + database_variable='zendesk_database', + schema_variable='zendesk_schema', + default_database=target.database, + default_schema='zendesk', + default_variable='user', + union_schema_variable='zendesk_union_schemas', + union_database_variable='zendesk_union_databases' + ) +}} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 908f471..9b0c6db 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,10 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +# - local: ../../dbt_fivetran_utils +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/enhance-union-data + warn-unpinned: false + - package: dbt-labs/spark_utils - version: [">=0.3.0", "<0.4.0"] + version: [">=0.3.0", "<0.4.0"] \ No newline at end of file From e4f50e0f80c3103fb903ac340a7ceb242f9e1210 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:56:13 -0800 Subject: [PATCH 02/15] test and docs --- README.md | 26 ++++++++++++++++++++++++++ models/stg_zendesk.yml | 7 +------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8123722..0935024 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ packages: version: [">=0.10.0", "<0.11.0"] ``` ## Step 3: Define database and schema variables +### Option 1: Single connector By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -54,6 +55,31 @@ vars: zendesk_database: your_destination_name zendesk_schema: your_schema_name ``` + +### Option 2: Union multiple connectors +If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + zendesk_union_schemas: ['zendesk_usa','zendesk_canada'] # use this if the data is in different schemas/datasets of the same database/project + zendesk_union_databases: ['zendesk_usa','zendesk_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +#### Recommended: Incorporate unioned sources into DAG +By default, this package defines one source, called `zendesk`. This will point to the data in the first entry provided to `zendesk_union_schemas` or `zendesk_union_databases` (so `zendesk_usa` in the above example). + +To properly incorporate all of your Zendesk connectors into your project's DAG: +1. Define your sources in a `.yml` file in your project. Please copy and paste the table and column configs from our [pre-defined](models/src_zendesk.yml) zendesk source. +2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to true, like such: +```yml +# dbt_project.yml +vars: + zendesk_source: + has_defined_sources: true +``` + ## Step 4: Disable models for non-existent sources This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: ```yml diff --git a/models/stg_zendesk.yml b/models/stg_zendesk.yml index 939b525..5d0132b 100644 --- a/models/stg_zendesk.yml +++ b/models/stg_zendesk.yml @@ -250,12 +250,7 @@ models: Empty string if you are not. - name: stg_zendesk__schedule - description: The support schedules created with different business hours and holidays. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - schedule_id - - source_relation + description: The support schedules created with different business hours and holidays. columns: - name: schedule_id description: ID automatically assigned to the schedule upon creation From 3c2b3944691fcd67cd1cbe1dbc93cfa2623ecf12 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:48:05 -0800 Subject: [PATCH 03/15] Docs and sources --- README.md | 341 ++++++++++++++++++++++++++++++++++++++++- models/src_zendesk.yml | 6 +- 2 files changed, 343 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0935024..352f186 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,347 @@ vars: ``` #### Recommended: Incorporate unioned sources into DAG -By default, this package defines one source, called `zendesk`. This will point to the data in the first entry provided to `zendesk_union_schemas` or `zendesk_union_databases` (so `zendesk_usa` in the above example). +By default, this package defines one single-connector source, called `zendesk`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Zendesk sources, though the package will run successfully. To properly incorporate all of your Zendesk connectors into your project's DAG: -1. Define your sources in a `.yml` file in your project. Please copy and paste the table and column configs from our [pre-defined](models/src_zendesk.yml) zendesk source. +1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. + +
Expand for source configuration template

+ +```yml +sources: + - name: + schema: + database: + loader: fivetran + loaded_at_field: _fivetran_synced + + freshness: + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} + + tables: &zendesk_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ + - name: ticket + description: > + Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can + originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. + columns: + - name: id + description: Automatically assigned when the ticket is created + - name: url + description: The API url of this ticket + - name: assignee_id + description: The agent currently assigned to the ticket + - name: brand_id + description: Enterprise only. The id of the brand this ticket is associated with + - name: created_at + description: When this record was created + - name: type + description: The type of this ticket, possible values are problem, incident, question or task + - name: subject + description: The value of the subject field for this ticket + - name: description + description: Read-only first comment on the ticket + - name: priority + description: The urgency with which the ticket should be addressed, possible values are urgent, high, normal and low + - name: status + description: The state of the ticket, possible values are new, open, pending, hold, solved and closed + - name: recipient + description: The original recipient e-mail address of the ticket + - name: requester_id + description: The user who requested this ticket + - name: submitter_id + description: The user who submitted the ticket. The submitter always becomes the author of the first comment on the ticket + - name: organization_id + description: The organization of the requester + - name: group_id + description: The group this ticket is assigned to + - name: due_at + description: If this is a ticket of type "task" it has a due date. Due date format uses ISO 8601 format. + - name: ticket_form_id + description: Enterprise only. The id of the ticket form to render for the ticket + - name: is_public + description: Is true if any comments are public, false otherwise + - name: updated_at + description: When this record last got updated + - name: via_channel + description: The channel the ticket was created from + - name: via_source_from_id + description: The channel the ticket was created from + - name: via_source_from_title + description: The channel the ticket was created from + - name: via_source_rel + description: The rel the ticket was created from + - name: via_source_to_address + description: The address of the source the ticket was created from + - name: via_source_to_name + description: The name of the source the ticket was created from + + - name: brand + description: > + Brands are your customer-facing identities. They might represent multiple products or services, or they + might literally be multiple brands owned and represented by your company. + columns: + - name: id + description: The ID automatically assigned when the brand is created + - name: brand_url + description: The url of the brand + - name: name + description: The name of the brand + - name: subdomain + description: The subdomain of the brand + - name: active + description: If the brand is set as active + + - name: domain_name + description: Domain names associated with an organization. An organization may have multiple domain names. + config: + enabled: "{{ var('using_domain_names', true) }}" + columns: + - name: organization_id + description: Reference to the organization + - name: domain_name + description: The name of the domain associated with the organization + - name: index + description: Index number of the domain name associated with the organization + + - name: group + identifier: > + {% if target.type == 'snowflake' %}"GROUP"{% else %}group{% endif %} + description: > + When support requests arrive in Zendesk Support, they can be assigned to a Group. Groups serve as the core + element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group + only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being + assigned to a Group. + freshness: null + columns: + - name: id + description: Automatically assigned when creating groups + - name: name + description: The name of the group + + - name: organization_tag + description: The tags associated with an organization. An organization may have multiple tags. + config: + enabled: "{{ var('using_organization_tags', true) }}" + columns: + - name: organization_id + description: Reference to the organization + - name: tag + description: Tag associated with the organization + + - name: organization + description: > + Just as agents can be segmented into groups in Zendesk Support, your customers (end-users) can be segmented into + organizations. You can manually assign customers to an organization or automatically assign them to an organization + by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or + to send email notifications. + freshness: null + columns: + - name: id + description: Automatically assigned when the organization is created + - name: name + description: A unique name for the organization + - name: details + description: Any details obout the organization, such as the address + - name: url + description: The API url of this organization + - name: external_id + description: A unique external id to associate organizations to an external record + - name: created_at + description: The time the organization was created + - name: updated_at + description: The time of the last update of the organization + - name: domain_names + description: An array of domain names associated with this organization + - name: details + description: Any details obout the organization, such as the address + - name: notes + description: Any notes you have about the organization + - name: group_id + description: New tickets from users in this organization are automatically put in this group + - name: shared_tickets + description: End users in this organization are able to see each other's tickets + - name: shared_comments + description: End users in this organization are able to see each other's comments on tickets + - name: tags + description: The tags of the organization + - name: organization_fields + description: Custom fields for this organization + + - name: ticket_comment + description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. + columns: + - name: id + description: Automatically assigned when the comment is created + - name: body + description: The comment string + - name: created + description: The time the comment was created + - name: public + description: Boolean field indicating if the comment is public (true), or if it is an internal note (false) + - name: ticket_id + description: The ticket id associated with this comment + - name: user_id + description: The id of the comment author + - name: facebook_comment + description: Boolean field indicating if the comment is a facebook comment + - name: tweet + description: Boolean field indicating if the comment is a twitter tweet + - name: voice_comment + description: Boolean field indicating if the comment is a voice comment + + - name: user_tag + description: Table containing all tags associated with a user. Only present if your account has user tagging enabled. + config: + enabled: "{{ var('using_user_tags', true) }}" + columns: + - name: user_id + description: Reference to the user + - name: tag + description: Tag associated with the user + + - name: user + description: Zendesk has three types of users, end-users (your customers), agents, and administrators. + freshness: null + columns: + - name: id + description: Automatically assigned when the user is created + - name: email + description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address + - name: name + description: The user's name + - name: active + description: false if the user has been deleted + - name: created_at + description: The time the user was created + - name: organization_id + description: The id of the user's organization. If the user has more than one organization memberships, the id of the user's default organization + - name: role + description: The user's role. Possible values are "end-user", "agent", or "admin" + - name: time_zone + description: The user's time zone. See Time Zone + - name: ticket_restriction + description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null + + - name: schedule + description: The support schedules created with different business hours and holidays. + freshness: null + config: + enabled: "{{ var('using_schedules', true) }}" + columns: + - name: id + description: ID automatically assigned to the schedule upon creation + - name: name + description: Name of the schedule + - name: created_at + description: Time the schedule was created + - name: start_time + description: Start time of the schedule, in the schedule's time zone. + - name: end_time + description: End time of the schedule, in the schedule's time zone. + - name: time_zone + description: Timezone in which the schedule operates. + + - name: ticket_schedule + description: The schedules applied to tickets through a trigger. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket assigned to the schedule + - name: created_at + description: The time the schedule was assigned to the ticket + - name: schedule_id + description: The ID of the schedule applied to the ticket + + - name: ticket_form_history + description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. + config: + enabled: "{{ var('using_ticket_form_history', true) }}" + columns: + - name: id + description: Automatically assigned when creating ticket form + - name: created_at + description: The time the ticket form was created + - name: updated_at + description: The time of the last update of the ticket form + - name: display_name + description: The name of the form that is displayed to an end user + - name: active + description: If the form is set as active + - name: name + description: The name of the form + + - name: ticket_tag + description: > + Tags are words, or combinations of words, you can use to add more context to tickets. The table lists all + tags currently associated with a ticket. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket associated with the tag + - name: tags + description: The tag, or word(s), associated with the ticket + + - name: ticket_field_history + description: All fields and field values associated with tickets. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket associated with the field + - name: field_name + description: The name of the ticket field + - name: updated + description: The time the ticket field value was created + - name: value + description: The value of the field + - name: user_id + description: The id of the user who made the update + + - name: daylight_time + description: > + Appropriate offsets (from UTC) for timezones that engage or have engaged with Daylight Savings at some point since 1970. + freshness: null + columns: + - name: daylight_end_utc + description: UTC timestamp of when Daylight Time ended in this year. + - name: daylight_offset + description: Number of **hours** added during Daylight Savings Time. + - name: daylight_start_utc + description: UTC timestamp of when Daylight Time began in this year. + - name: time_zone + description: Name of the timezone. + - name: year + description: Year in which daylight savings occurred. + + - name: time_zone + description: Offsets (from UTC) for each timezone. + freshness: null + columns: + - name: time_zone + description: Name of the time zone. + - name: standard_offset + description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. + + - name: schedule_holiday + description: Information about holidays for each specified schedule. + freshness: null + config: + enabled: "{{ var('using_schedules', true) }}" + columns: + - name: end_date + description: ISO 8601 representation of the holiday end date. + - name: id + description: The ID of the scheduled holiday. + - name: name + description: Name of the holiday. + - name: schedule_id + description: The ID of the schedule. + - name: start_date + description: ISO 8601 representation of the holiday start date. +``` +

+ 2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to true, like such: ```yml # dbt_project.yml diff --git a/models/src_zendesk.yml b/models/src_zendesk.yml index cc5ef4e..ee7275f 100644 --- a/models/src_zendesk.yml +++ b/models/src_zendesk.yml @@ -2,16 +2,18 @@ version: 2 sources: - name: zendesk - schema: "{{ var('zendesk_schema', 'zendesk') if var('zendesk_union_schemas', []) == [] else var('zendesk_union_schemas', [])[0] }}" + schema: "{{ var('zendesk_schema', 'zendesk') }}" database: "{% if target.type != 'spark'%}{{ var('zendesk_database', target.database) }}{% endif %}" loader: fivetran loaded_at_field: _fivetran_synced + + config: + enabled: "{{ var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] }}" freshness: warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: - name: ticket identifier: "{{ var('zendesk_ticket_identifier', 'ticket')}}" From 457d9773607b60afd0c8356c9213064202aa4b7e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:01:15 -0800 Subject: [PATCH 04/15] missed window ffunction --- README.md | 4 +++- models/stg_zendesk__ticket_field_history.sql | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 352f186..1f870a5 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,9 @@ vars: ``` ## Step 4: Disable models for non-existent sources -This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: +This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. + +By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: ```yml vars: using_schedules: False #Disable if you are not using schedules diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index 7cc332c..101a032 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -37,10 +37,10 @@ final as ( field_name, {% if target.type == 'redshift' -%} cast(updated as timestamp without time zone) as valid_starting_at, - cast(lead(updated) over (partition by ticket_id, field_name order by updated) as timestamp without time zone) as valid_ending_at, + cast(lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as timestamp without time zone) as valid_ending_at, {% else -%} updated as valid_starting_at, - lead(updated) over (partition by ticket_id, field_name order by updated) as valid_ending_at, + lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as valid_ending_at, {% endif %} value, user_id, From 78fd1b41864006603f2cc4e1a36b566da7277e00 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:56:07 -0800 Subject: [PATCH 05/15] readme docs for unioning sources --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1f870a5..fbefcbf 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ To properly incorporate all of your Zendesk connectors into your project's DAG:
Expand for source configuration template

+> If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. + ```yml sources: - name: From cf909d9b5b2a034d3dc3f0392671b271415f01e3 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:07:58 -0800 Subject: [PATCH 06/15] docs --- CHANGELOG.md | 3 ++- README.md | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0276670..f786081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # dbt_zendesk_source v0.11.0 -Unioning.... +## 🎉 Feature Update 🎉 +This release supports runnning the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). # dbt_zendesk_source v0.10.0 [PR #42](https://github.com/fivetran/dbt_zendesk_source/pull/42) introduces the following updates: diff --git a/README.md b/README.md index fbefcbf..1516e19 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ packages: - package: fivetran/zendesk_source version: [">=0.10.0", "<0.11.0"] ``` + ## Step 3: Define database and schema variables ### Option 1: Single connector By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file: @@ -55,6 +56,7 @@ vars: zendesk_database: your_destination_name zendesk_schema: your_schema_name ``` +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. ### Option 2: Union multiple connectors If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: @@ -75,7 +77,7 @@ To properly incorporate all of your Zendesk connectors into your project's DAG:

Expand for source configuration template

-> If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. ```yml sources: From 25137a48290b13fb44142bd443ef16e74fd9a7a7 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:55:32 -0800 Subject: [PATCH 07/15] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f786081..ed753dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_zendesk_source v0.11.0 ## 🎉 Feature Update 🎉 -This release supports runnning the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). +This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). # dbt_zendesk_source v0.10.0 [PR #42](https://github.com/fivetran/dbt_zendesk_source/pull/42) introduces the following updates: From abb7bf20433c26f449b8f6daa7f631fb2065f8d3 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 27 Dec 2023 10:58:26 -0800 Subject: [PATCH 08/15] update package versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1516e19..d14cb2e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Include the following zendesk_source package version in your `packages.yml` file ```yaml packages: - package: fivetran/zendesk_source - version: [">=0.10.0", "<0.11.0"] + version: [">=0.11.0", "<0.12.0"] ``` ## Step 3: Define database and schema variables From 55e4b44007302c1964d9d152fb87c16aae09d913 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:32:30 -0800 Subject: [PATCH 09/15] joe feedback --- CHANGELOG.md | 5 ++++- models/stg_zendesk__ticket_field_history.sql | 9 ++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed753dc..00b1b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # dbt_zendesk_source v0.11.0 ## 🎉 Feature Update 🎉 -This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). +- This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). + +## 🔧 Under the Hood 🔧 +- Swaps Redshift-speciifc casting logic in the `stg_zendesk__ticket_field_history` model with the cross-database `dbt.type_timestamp()` macro ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). # dbt_zendesk_source v0.10.0 [PR #42](https://github.com/fivetran/dbt_zendesk_source/pull/42) introduces the following updates: diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index 101a032..00d2939 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -35,13 +35,8 @@ final as ( select ticket_id, field_name, - {% if target.type == 'redshift' -%} - cast(updated as timestamp without time zone) as valid_starting_at, - cast(lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as timestamp without time zone) as valid_ending_at, - {% else -%} - updated as valid_starting_at, - lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as valid_ending_at, - {% endif %} + cast(updated as {{ dbt.type_timestamp() }}) as valid_starting_at, + cast(lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as {{ dbt.type_timestamp() }}) as valid_ending_at, value, user_id, source_relation From 363962650939a5be08dd0dd1ad2b74cb644e03f5 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:24:54 -0300 Subject: [PATCH 10/15] readme tweaks following hubspot --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d14cb2e..9f91d0a 100644 --- a/README.md +++ b/README.md @@ -422,7 +422,9 @@ vars: ``` ## Step 4: Disable models for non-existent sources -This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ + +This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: ```yml @@ -462,6 +464,8 @@ vars: zendesk__identifier: your_table_name ``` +This solution is most relevant to users running the package on a single connector. + ### 🚨 Snowflake Users If you do **not** use the default all-caps naming conventions for Snowflake, you may need to provide the case-sensitive spelling of your source tables that are also Snowflake reserved words. From e76b12b41d20f5c429e682fdae07f9473dce8283 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:16:48 -0400 Subject: [PATCH 11/15] update readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c388973..64732d7 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ packages: ``` ## Step 3: Define database and schema variables -### Option 1: Single connector +### Option 1: Single connector 💃 By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -58,7 +58,7 @@ vars: ``` > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors +### Option 2: Union multiple connectors 👯 If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml @@ -422,7 +422,7 @@ vars: ``` ## Step 4: Disable models for non-existent sources -> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. @@ -437,6 +437,7 @@ vars: ``` ## (Optional) Step 5: Additional configurations +

Expand/collapse configurations ### Add passthrough columns This package includes all source columns defined in the staging models. However, the `stg_zendesk__ticket` model allows for additional columns to be added using a pass-through column variable. This is extremely useful if you'd like to include custom fields to the package. @@ -467,7 +468,7 @@ models: +schema: my_new_schema_name # leave blank for just the target_schema ``` -### Change the source table references +### Change the source table references (only if using a single connector) If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [dbt_project.yml](https://github.com/fivetran/dbt_zendesk_source/blob/main/dbt_project.yml) variable declarations to see the expected names. @@ -476,8 +477,6 @@ vars: zendesk__identifier: your_table_name ``` -This solution is most relevant to users running the package on a single connector. - ### 🚨 Snowflake Users If you do **not** use the default all-caps naming conventions for Snowflake, you may need to provide the case-sensitive spelling of your source tables that are also Snowflake reserved words. @@ -488,6 +487,7 @@ vars: zendesk_group_identifier: "Group" # as an example, must include the double-quotes and correct case! ``` +
## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Coreâ„¢
Expand to view details From 80dbfa4be7c6c4706eb29ce9797e4b2901212397 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:54:58 -0500 Subject: [PATCH 12/15] swap source definition template --- README.md | 340 ++---------------------------------------------------- 1 file changed, 10 insertions(+), 330 deletions(-) diff --git a/README.md b/README.md index 64732d7..deacfb0 100644 --- a/README.md +++ b/README.md @@ -73,347 +73,27 @@ vars: By default, this package defines one single-connector source, called `zendesk`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Zendesk sources, though the package will run successfully. To properly incorporate all of your Zendesk connectors into your project's DAG: -1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. - -
Expand for source configuration template

- -> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_zendesk.yml` [file](https://github.com/fivetran/dbt_zendesk_source/blob/main/models/src_zendesk.yml#L15-L351). ```yml +# a .yml file in your root project sources: - - name: - schema: - database: + - name: # ex: zendesk_usa + schema: # one of var('zendesk_union_schemas') if unioning schemas, otherwise just 'zendesk' + database: # one of var('zendesk_union_databases') if unioning databases, otherwise whatever DB your zendesk schemas all live in loader: fivetran loaded_at_field: _fivetran_synced - freshness: + freshness: # feel free to adjust to your liking warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: &zendesk_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - - name: ticket - description: > - Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can - originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. - columns: - - name: id - description: Automatically assigned when the ticket is created - - name: url - description: The API url of this ticket - - name: assignee_id - description: The agent currently assigned to the ticket - - name: brand_id - description: Enterprise only. The id of the brand this ticket is associated with - - name: created_at - description: When this record was created - - name: type - description: The type of this ticket, possible values are problem, incident, question or task - - name: subject - description: The value of the subject field for this ticket - - name: description - description: Read-only first comment on the ticket - - name: priority - description: The urgency with which the ticket should be addressed, possible values are urgent, high, normal and low - - name: status - description: The state of the ticket, possible values are new, open, pending, hold, solved and closed - - name: recipient - description: The original recipient e-mail address of the ticket - - name: requester_id - description: The user who requested this ticket - - name: submitter_id - description: The user who submitted the ticket. The submitter always becomes the author of the first comment on the ticket - - name: organization_id - description: The organization of the requester - - name: group_id - description: The group this ticket is assigned to - - name: due_at - description: If this is a ticket of type "task" it has a due date. Due date format uses ISO 8601 format. - - name: ticket_form_id - description: Enterprise only. The id of the ticket form to render for the ticket - - name: is_public - description: Is true if any comments are public, false otherwise - - name: updated_at - description: When this record last got updated - - name: via_channel - description: The channel the ticket was created from - - name: via_source_from_id - description: The channel the ticket was created from - - name: via_source_from_title - description: The channel the ticket was created from - - name: via_source_rel - description: The rel the ticket was created from - - name: via_source_to_address - description: The address of the source the ticket was created from - - name: via_source_to_name - description: The name of the source the ticket was created from - - - name: brand - description: > - Brands are your customer-facing identities. They might represent multiple products or services, or they - might literally be multiple brands owned and represented by your company. - columns: - - name: id - description: The ID automatically assigned when the brand is created - - name: brand_url - description: The url of the brand - - name: name - description: The name of the brand - - name: subdomain - description: The subdomain of the brand - - name: active - description: If the brand is set as active - - - name: domain_name - description: Domain names associated with an organization. An organization may have multiple domain names. - config: - enabled: "{{ var('using_domain_names', true) }}" - columns: - - name: organization_id - description: Reference to the organization - - name: domain_name - description: The name of the domain associated with the organization - - name: index - description: Index number of the domain name associated with the organization - - - name: group - identifier: > - {% if target.type == 'snowflake' %}"GROUP"{% else %}group{% endif %} - description: > - When support requests arrive in Zendesk Support, they can be assigned to a Group. Groups serve as the core - element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group - only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being - assigned to a Group. - freshness: null - columns: - - name: id - description: Automatically assigned when creating groups - - name: name - description: The name of the group - - - name: organization_tag - description: The tags associated with an organization. An organization may have multiple tags. - config: - enabled: "{{ var('using_organization_tags', true) }}" - columns: - - name: organization_id - description: Reference to the organization - - name: tag - description: Tag associated with the organization - - - name: organization - description: > - Just as agents can be segmented into groups in Zendesk Support, your customers (end-users) can be segmented into - organizations. You can manually assign customers to an organization or automatically assign them to an organization - by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or - to send email notifications. - freshness: null - columns: - - name: id - description: Automatically assigned when the organization is created - - name: name - description: A unique name for the organization - - name: details - description: Any details obout the organization, such as the address - - name: url - description: The API url of this organization - - name: external_id - description: A unique external id to associate organizations to an external record - - name: created_at - description: The time the organization was created - - name: updated_at - description: The time of the last update of the organization - - name: domain_names - description: An array of domain names associated with this organization - - name: details - description: Any details obout the organization, such as the address - - name: notes - description: Any notes you have about the organization - - name: group_id - description: New tickets from users in this organization are automatically put in this group - - name: shared_tickets - description: End users in this organization are able to see each other's tickets - - name: shared_comments - description: End users in this organization are able to see each other's comments on tickets - - name: tags - description: The tags of the organization - - name: organization_fields - description: Custom fields for this organization - - - name: ticket_comment - description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. - columns: - - name: id - description: Automatically assigned when the comment is created - - name: body - description: The comment string - - name: created - description: The time the comment was created - - name: public - description: Boolean field indicating if the comment is public (true), or if it is an internal note (false) - - name: ticket_id - description: The ticket id associated with this comment - - name: user_id - description: The id of the comment author - - name: facebook_comment - description: Boolean field indicating if the comment is a facebook comment - - name: tweet - description: Boolean field indicating if the comment is a twitter tweet - - name: voice_comment - description: Boolean field indicating if the comment is a voice comment - - - name: user_tag - description: Table containing all tags associated with a user. Only present if your account has user tagging enabled. - config: - enabled: "{{ var('using_user_tags', true) }}" - columns: - - name: user_id - description: Reference to the user - - name: tag - description: Tag associated with the user - - - name: user - description: Zendesk has three types of users, end-users (your customers), agents, and administrators. - freshness: null - columns: - - name: id - description: Automatically assigned when the user is created - - name: email - description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address - - name: name - description: The user's name - - name: active - description: false if the user has been deleted - - name: created_at - description: The time the user was created - - name: organization_id - description: The id of the user's organization. If the user has more than one organization memberships, the id of the user's default organization - - name: role - description: The user's role. Possible values are "end-user", "agent", or "admin" - - name: time_zone - description: The user's time zone. See Time Zone - - name: ticket_restriction - description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null - - - name: schedule - description: The support schedules created with different business hours and holidays. - freshness: null - config: - enabled: "{{ var('using_schedules', true) }}" - columns: - - name: id - description: ID automatically assigned to the schedule upon creation - - name: name - description: Name of the schedule - - name: created_at - description: Time the schedule was created - - name: start_time - description: Start time of the schedule, in the schedule's time zone. - - name: end_time - description: End time of the schedule, in the schedule's time zone. - - name: time_zone - description: Timezone in which the schedule operates. - - - name: ticket_schedule - description: The schedules applied to tickets through a trigger. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket assigned to the schedule - - name: created_at - description: The time the schedule was assigned to the ticket - - name: schedule_id - description: The ID of the schedule applied to the ticket - - - name: ticket_form_history - description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. - config: - enabled: "{{ var('using_ticket_form_history', true) }}" - columns: - - name: id - description: Automatically assigned when creating ticket form - - name: created_at - description: The time the ticket form was created - - name: updated_at - description: The time of the last update of the ticket form - - name: display_name - description: The name of the form that is displayed to an end user - - name: active - description: If the form is set as active - - name: name - description: The name of the form - - - name: ticket_tag - description: > - Tags are words, or combinations of words, you can use to add more context to tickets. The table lists all - tags currently associated with a ticket. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket associated with the tag - - name: tags - description: The tag, or word(s), associated with the ticket - - - name: ticket_field_history - description: All fields and field values associated with tickets. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket associated with the field - - name: field_name - description: The name of the ticket field - - name: updated - description: The time the ticket field value was created - - name: value - description: The value of the field - - name: user_id - description: The id of the user who made the update - - - name: daylight_time - description: > - Appropriate offsets (from UTC) for timezones that engage or have engaged with Daylight Savings at some point since 1970. - freshness: null - columns: - - name: daylight_end_utc - description: UTC timestamp of when Daylight Time ended in this year. - - name: daylight_offset - description: Number of **hours** added during Daylight Savings Time. - - name: daylight_start_utc - description: UTC timestamp of when Daylight Time began in this year. - - name: time_zone - description: Name of the timezone. - - name: year - description: Year in which daylight savings occurred. - - - name: time_zone - description: Offsets (from UTC) for each timezone. - freshness: null - columns: - - name: time_zone - description: Name of the time zone. - - name: standard_offset - description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. - - - name: schedule_holiday - description: Information about holidays for each specified schedule. - freshness: null - config: - enabled: "{{ var('using_schedules', true) }}" - columns: - - name: end_date - description: ISO 8601 representation of the holiday end date. - - name: id - description: The ID of the scheduled holiday. - - name: name - description: Name of the holiday. - - name: schedule_id - description: The ID of the schedule. - - name: start_date - description: ISO 8601 representation of the holiday start date. + tables: # copy and paste from models/src_zendesk.yml ``` -

-2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to true, like such: +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. + +2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to `True`, like such: ```yml # dbt_project.yml vars: From 8be3f396f484895848715818f3801663e8cf7ea2 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:24:50 -0800 Subject: [PATCH 13/15] working --- CHANGELOG.md | 4 +- macros/union/union_zendesk_connections.sql | 63 +++++++++ macros/union/zendesk_source_relation.sql | 15 ++ macros/union/zendesk_union_relations.sql | 131 ++++++++++++++++++ models/src_zendesk.yml | 2 +- models/stg_zendesk__audit_log.sql | 9 ++ models/stg_zendesk__brand.sql | 10 +- models/stg_zendesk__daylight_time.sql | 10 +- models/stg_zendesk__domain_name.sql | 10 +- models/stg_zendesk__group.sql | 10 +- models/stg_zendesk__organization.sql | 10 +- models/stg_zendesk__organization_tag.sql | 10 +- models/stg_zendesk__schedule.sql | 11 +- models/stg_zendesk__schedule_holiday.sql | 10 +- models/stg_zendesk__ticket.sql | 10 +- models/stg_zendesk__ticket_comment.sql | 10 +- models/stg_zendesk__ticket_field_history.sql | 10 +- models/stg_zendesk__ticket_form_history.sql | 10 +- models/stg_zendesk__ticket_schedule.sql | 11 +- models/stg_zendesk__ticket_tag.sql | 10 +- models/stg_zendesk__time_zone.sql | 10 +- models/stg_zendesk__user.sql | 10 +- models/stg_zendesk__user_tag.sql | 10 +- models/tmp/stg_zendesk__audit_log_tmp.sql | 9 +- models/tmp/stg_zendesk__brand_tmp.sql | 13 +- models/tmp/stg_zendesk__daylight_time_tmp.sql | 13 +- models/tmp/stg_zendesk__domain_name_tmp.sql | 13 +- models/tmp/stg_zendesk__group_tmp.sql | 13 +- .../tmp/stg_zendesk__organization_tag_tmp.sql | 13 +- models/tmp/stg_zendesk__organization_tmp.sql | 13 +- .../tmp/stg_zendesk__schedule_holiday_tmp.sql | 13 +- models/tmp/stg_zendesk__schedule_tmp.sql | 13 +- .../tmp/stg_zendesk__ticket_comment_tmp.sql | 13 +- .../stg_zendesk__ticket_field_history_tmp.sql | 13 +- .../stg_zendesk__ticket_form_history_tmp.sql | 13 +- .../tmp/stg_zendesk__ticket_schedule_tmp.sql | 13 +- models/tmp/stg_zendesk__ticket_tag_tmp.sql | 13 +- models/tmp/stg_zendesk__ticket_tmp.sql | 13 +- models/tmp/stg_zendesk__time_zone_tmp.sql | 13 +- models/tmp/stg_zendesk__user_tag_tmp.sql | 14 +- models/tmp/stg_zendesk__user_tmp.sql | 13 +- 41 files changed, 415 insertions(+), 212 deletions(-) create mode 100644 macros/union/union_zendesk_connections.sql create mode 100644 macros/union/zendesk_source_relation.sql create mode 100644 macros/union/zendesk_union_relations.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index fab7804..829e46b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,8 @@ # dbt_zendesk_source v0.14.0 -## 🎉 Feature Update 🎉 +## Feature Update: Run Package on Unioned Connectors - This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). -## 🔧 Under the Hood 🔧 -- Swaps Redshift-speciifc casting logic in the `stg_zendesk__ticket_field_history` model with the cross-database `dbt.type_timestamp()` macro ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). # dbt_zendesk_source v0.13.0 [PR #55](https://github.com/fivetran/dbt_zendesk_source/pull/55) includes the following updates: diff --git a/macros/union/union_zendesk_connections.sql b/macros/union/union_zendesk_connections.sql new file mode 100644 index 0000000..195a7ee --- /dev/null +++ b/macros/union/union_zendesk_connections.sql @@ -0,0 +1,63 @@ +{% macro union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} + +{{ adapter.dispatch('union_zendesk_connections', 'zendesk_source') (connection_dictionary, single_source_name, single_table_name) }} + +{%- endmacro %} + +{% macro default__union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} + +{% if connection_dictionary %} +{# For unioning #} + {%- set relations = [] -%} + {%- for connection in connection_dictionary -%} + + {%- set relation=source(connection.name, single_table_name) if var('has_defined_sources', false) + else adapter.get_relation( + database=connection.database if connection.database else target.database, + schema=connection.schema if connection.schema else single_source_name, + identifier=single_table_name + ) + -%} + + {%- if relation is not none -%} + {%- do relations.append(relation) -%} + {%- endif -%} + + {%- endfor -%} + + {%- if relations != [] -%} + {{ zendesk_source.zendesk_union_relations(relations) }} + {%- else -%} + {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} + {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name ~ "." ~ single_table_name ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} + {% endif -%} + select + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + limit 0 + {%- endif -%} + +{% else %} +{# Not unioning #} + {%- set relation=adapter.get_relation( + database=source(single_source_name, single_table_name).database, + schema=source(single_source_name, single_table_name).schema, + identifier=source(single_source_name, single_table_name).identifier + ) -%} + + {%- if relation is not none -%} + select + {{ dbt_utils.star(from=source(single_source_name, single_table_name)) }} + from {{ source(single_source_name, single_table_name) }} as source_table + + {% else %} + {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} + {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name|upper ~ "." ~ single_table_name|upper ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} + {% endif -%} + + select + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + limit 0 + {%- endif -%} +{% endif -%} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/union/zendesk_source_relation.sql b/macros/union/zendesk_source_relation.sql new file mode 100644 index 0000000..a5bab64 --- /dev/null +++ b/macros/union/zendesk_source_relation.sql @@ -0,0 +1,15 @@ +{% macro zendesk_source_relation(connection_dictionary, single_schema, single_database, single_table_identifier) -%} + +{{ adapter.dispatch('zendesk_source_relation', 'zendesk_source') (connection_dictionary, single_schema, single_database, single_table_identifier) }} + +{%- endmacro %} + +{% macro default__zendesk_source_relation(connection_dictionary, single_schema, single_database, single_table_identifier) -%} + +{% if connection_dictionary %} +, _dbt_source_relation as source_relation +{% else %} +, '{{ single_database }}' || '.'|| '{{ single_schema }}' || '.'|| '{{ single_table_identifier }}' as source_relation +{% endif %} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/union/zendesk_union_relations.sql b/macros/union/zendesk_union_relations.sql new file mode 100644 index 0000000..75a8085 --- /dev/null +++ b/macros/union/zendesk_union_relations.sql @@ -0,0 +1,131 @@ +{# Adapted from dbt_utils.union_relations() #} + +{%- macro zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} + {{ return(adapter.dispatch('zendesk_union_relations', 'zendesk_source')(relations, aliases, column_override, include, exclude, source_column_name, where)) }} +{% endmacro %} + +{%- macro default__zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} + + {%- if exclude and include -%} + {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} + {%- endif -%} + + {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} + {%- if not execute %} + {{ return('') }} + {% endif -%} + + {%- set column_override = column_override if column_override is not none else {} -%} + + {%- set relation_columns = {} -%} + {%- set column_superset = {} -%} + {%- set all_excludes = [] -%} + {%- set all_includes = [] -%} + + {%- if exclude -%} + {%- for exc in exclude -%} + {%- do all_excludes.append(exc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- if include -%} + {%- for inc in include -%} + {%- do all_includes.append(inc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- for relation in relations -%} + + {%- do relation_columns.update({relation: []}) -%} + + {%- do dbt_utils._is_relation(relation, 'zendesk_union_relations') -%} + {%- do dbt_utils._is_ephemeral(relation, 'zendesk_union_relations') -%} + {%- set cols = adapter.get_columns_in_relation(relation) -%} + {%- for col in cols -%} + + {#- If an exclude list was provided and the column is in the list, do nothing -#} + {%- if exclude and col.column | lower in all_excludes -%} + + {#- If an include list was provided and the column is not in the list, do nothing -#} + {%- elif include and col.column | lower not in all_includes -%} + + {#- Otherwise add the column to the column superset -#} + {%- else -%} + + {#- update the list of columns in this relation -#} + {%- do relation_columns[relation].append(col.column) -%} + + {%- if col.column in column_superset -%} + + {%- set stored = column_superset[col.column] -%} + {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} + + {%- do column_superset.update({col.column: col}) -%} + + {%- endif %} + + {%- else -%} + + {%- do column_superset.update({col.column: col}) -%} + + {%- endif -%} + + {%- endif -%} + + {%- endfor -%} + {%- endfor -%} + + {%- set ordered_column_names = column_superset.keys() -%} + {%- set dbt_command = flags.WHICH -%} + + + {% if dbt_command in ['run', 'build'] %} + {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} + {%- set relations_string -%} + {%- for relation in relations -%} + {{ relation.name }} + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endset -%} + + {%- set error_message -%} + There were no columns found to union for relations {{ relations_string }} + {%- endset -%} + + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} + {%- endif -%} + + {%- for relation in relations %} + + ( + select + + {%- if source_column_name is not none %} + cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, + {%- endif %} + + {% for col_name in ordered_column_names -%} + + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + + {%- endfor %} + + {# This alias is the only addition made to thr dbt_utils.union_relations() code. Avoids errors if the table is named a reserved keyword #} + from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }} + + {% if where -%} + where {{ where }} + {%- endif %} + ) + + {% if not loop.last -%} + union all + {% endif -%} + + {%- endfor -%} + +{%- endmacro -%} \ No newline at end of file diff --git a/models/src_zendesk.yml b/models/src_zendesk.yml index 9efa277..03ffc74 100644 --- a/models/src_zendesk.yml +++ b/models/src_zendesk.yml @@ -8,7 +8,7 @@ sources: loaded_at_field: _fivetran_synced config: - enabled: "{{ var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] }}" + enabled: "{{ var('zendesk_sources', []) == [] }}" freshness: warn_after: {count: 72, period: hour} diff --git a/models/stg_zendesk__audit_log.sql b/models/stg_zendesk__audit_log.sql index b119847..ab9f22f 100644 --- a/models/stg_zendesk__audit_log.sql +++ b/models/stg_zendesk__audit_log.sql @@ -22,6 +22,15 @@ fields as ( staging_columns=get_audit_log_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_audit_log_identifier", "audit_log") + ) + }} from base ), diff --git a/models/stg_zendesk__brand.sql b/models/stg_zendesk__brand.sql index 90d7f8f..98f0cad 100644 --- a/models/stg_zendesk__brand.sql +++ b/models/stg_zendesk__brand.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_brand_identifier", "brand") + ) }} from base diff --git a/models/stg_zendesk__daylight_time.sql b/models/stg_zendesk__daylight_time.sql index a0c38a8..206953d 100644 --- a/models/stg_zendesk__daylight_time.sql +++ b/models/stg_zendesk__daylight_time.sql @@ -24,9 +24,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_daylight_time_identifier", "daylight_time") + ) }} from base diff --git a/models/stg_zendesk__domain_name.sql b/models/stg_zendesk__domain_name.sql index 69a3924..34263e8 100644 --- a/models/stg_zendesk__domain_name.sql +++ b/models/stg_zendesk__domain_name.sql @@ -24,9 +24,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_domain_name_identifier", "domain_name") + ) }} from base diff --git a/models/stg_zendesk__group.sql b/models/stg_zendesk__group.sql index 37a90b3..3b5f2c7 100644 --- a/models/stg_zendesk__group.sql +++ b/models/stg_zendesk__group.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_group_identifier", "group") + ) }} from base diff --git a/models/stg_zendesk__organization.sql b/models/stg_zendesk__organization.sql index 5b110ee..90bbe5d 100644 --- a/models/stg_zendesk__organization.sql +++ b/models/stg_zendesk__organization.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_organization_identifier", "organization") + ) }} from base diff --git a/models/stg_zendesk__organization_tag.sql b/models/stg_zendesk__organization_tag.sql index 3e37c5e..3beee31 100644 --- a/models/stg_zendesk__organization_tag.sql +++ b/models/stg_zendesk__organization_tag.sql @@ -24,9 +24,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_organization_tag_identifier", "organization_tag") + ) }} from base diff --git a/models/stg_zendesk__schedule.sql b/models/stg_zendesk__schedule.sql index 4b74dda..6a060f0 100644 --- a/models/stg_zendesk__schedule.sql +++ b/models/stg_zendesk__schedule.sql @@ -24,10 +24,15 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_schedule_identifier", "schedule") + ) }} + from base ), diff --git a/models/stg_zendesk__schedule_holiday.sql b/models/stg_zendesk__schedule_holiday.sql index aa6daa0..122970c 100644 --- a/models/stg_zendesk__schedule_holiday.sql +++ b/models/stg_zendesk__schedule_holiday.sql @@ -17,9 +17,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_schedule_holiday_identifier", "schedule_holiday") + ) }} from base diff --git a/models/stg_zendesk__ticket.sql b/models/stg_zendesk__ticket.sql index f1295cc..fbb0ade 100644 --- a/models/stg_zendesk__ticket.sql +++ b/models/stg_zendesk__ticket.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_identifier", "ticket") + ) }} from base diff --git a/models/stg_zendesk__ticket_comment.sql b/models/stg_zendesk__ticket_comment.sql index ebd0ecf..3ea6993 100644 --- a/models/stg_zendesk__ticket_comment.sql +++ b/models/stg_zendesk__ticket_comment.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_comment_identifier", "ticket_comment") + ) }} from base diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index 00d2939..3842948 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_field_history_identifier", "ticket_field_history") + ) }} from base diff --git a/models/stg_zendesk__ticket_form_history.sql b/models/stg_zendesk__ticket_form_history.sql index 22a5d85..a73b456 100644 --- a/models/stg_zendesk__ticket_form_history.sql +++ b/models/stg_zendesk__ticket_form_history.sql @@ -24,9 +24,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_form_history_identifier", "ticket_form_history") + ) }} from base diff --git a/models/stg_zendesk__ticket_schedule.sql b/models/stg_zendesk__ticket_schedule.sql index f5fd78f..4977dbe 100644 --- a/models/stg_zendesk__ticket_schedule.sql +++ b/models/stg_zendesk__ticket_schedule.sql @@ -24,11 +24,14 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_schedule_identifier", "ticket_schedule") + ) }} - from base ), diff --git a/models/stg_zendesk__ticket_tag.sql b/models/stg_zendesk__ticket_tag.sql index 0684f01..9336128 100644 --- a/models/stg_zendesk__ticket_tag.sql +++ b/models/stg_zendesk__ticket_tag.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_ticket_tag_identifier", "ticket_tag") + ) }} from base diff --git a/models/stg_zendesk__time_zone.sql b/models/stg_zendesk__time_zone.sql index fab3fe3..f0a85a0 100644 --- a/models/stg_zendesk__time_zone.sql +++ b/models/stg_zendesk__time_zone.sql @@ -18,9 +18,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_time_zone_identifier", "time_zone") + ) }} from base diff --git a/models/stg_zendesk__user.sql b/models/stg_zendesk__user.sql index a565ccd..50e6954 100644 --- a/models/stg_zendesk__user.sql +++ b/models/stg_zendesk__user.sql @@ -22,9 +22,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_user_identifier", "user") + ) }} from base diff --git a/models/stg_zendesk__user_tag.sql b/models/stg_zendesk__user_tag.sql index 702b0be..bb3f480 100644 --- a/models/stg_zendesk__user_tag.sql +++ b/models/stg_zendesk__user_tag.sql @@ -24,9 +24,13 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases') + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database), + single_table_identifier=var("zendesk_user_tag_identifier", "user_tag") + ) }} from base diff --git a/models/tmp/stg_zendesk__audit_log_tmp.sql b/models/tmp/stg_zendesk__audit_log_tmp.sql index fbf42ee..065ec3e 100644 --- a/models/tmp/stg_zendesk__audit_log_tmp.sql +++ b/models/tmp/stg_zendesk__audit_log_tmp.sql @@ -1,4 +1,9 @@ {{ config(enabled=var('using_schedules', True) and var('using_schedule_histories', False)) }} -select {{ dbt_utils.star(source('zendesk','audit_log')) }} -from {{ source('zendesk','audit_log') }} as audit_log_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='audit_log' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__brand_tmp.sql b/models/tmp/stg_zendesk__brand_tmp.sql index a4f6e9f..8b9a773 100644 --- a/models/tmp/stg_zendesk__brand_tmp.sql +++ b/models/tmp/stg_zendesk__brand_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='brand', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='brand', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='brand' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__daylight_time_tmp.sql b/models/tmp/stg_zendesk__daylight_time_tmp.sql index 40b5eff..0c0171f 100644 --- a/models/tmp/stg_zendesk__daylight_time_tmp.sql +++ b/models/tmp/stg_zendesk__daylight_time_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_schedules', True)) }} {{ - fivetran_utils.union_data( - table_identifier='daylight_time', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='daylight_time', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='daylight_time' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__domain_name_tmp.sql b/models/tmp/stg_zendesk__domain_name_tmp.sql index c1d8225..ce31773 100644 --- a/models/tmp/stg_zendesk__domain_name_tmp.sql +++ b/models/tmp/stg_zendesk__domain_name_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_domain_names', True)) }} {{ - fivetran_utils.union_data( - table_identifier='domain_name', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='domain_name', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='domain_name' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__group_tmp.sql b/models/tmp/stg_zendesk__group_tmp.sql index 75316a6..00a55ec 100644 --- a/models/tmp/stg_zendesk__group_tmp.sql +++ b/models/tmp/stg_zendesk__group_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='group', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='group', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='group' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tag_tmp.sql b/models/tmp/stg_zendesk__organization_tag_tmp.sql index a8a7f60..e814ec2 100644 --- a/models/tmp/stg_zendesk__organization_tag_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tag_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_organization_tags', True)) }} {{ - fivetran_utils.union_data( - table_identifier='organization_tag', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='organization_tag', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='organization_tag' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tmp.sql b/models/tmp/stg_zendesk__organization_tmp.sql index 4bb4016..4f88156 100644 --- a/models/tmp/stg_zendesk__organization_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='organization', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='organization', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='organization' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql index 0fa38b1..0387678 100644 --- a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_schedules', True) and var('using_holidays', True)) }} {{ - fivetran_utils.union_data( - table_identifier='schedule_holiday', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='schedule_holiday', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='schedule_holiday' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_tmp.sql b/models/tmp/stg_zendesk__schedule_tmp.sql index 47f44e0..7da0928 100644 --- a/models/tmp/stg_zendesk__schedule_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_schedules', True)) }} {{ - fivetran_utils.union_data( - table_identifier='schedule', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='schedule', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='schedule' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_comment_tmp.sql b/models/tmp/stg_zendesk__ticket_comment_tmp.sql index b74d250..bf8db6f 100644 --- a/models/tmp/stg_zendesk__ticket_comment_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_comment_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='ticket_comment', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket_comment', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_comment' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql index 9a66b99..b8221c6 100644 --- a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='ticket_field_history', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket_field_history', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_field_history' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql index 1026cae..8e704b9 100644 --- a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_ticket_form_history', True)) }} {{ - fivetran_utils.union_data( - table_identifier='ticket_form_history', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket_form_history', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_form_history' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql index 6e85076..54ce83f 100644 --- a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_schedules', True)) }} {{ - fivetran_utils.union_data( - table_identifier='ticket_schedule', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket_schedule', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_schedule' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tag_tmp.sql b/models/tmp/stg_zendesk__ticket_tag_tmp.sql index b408936..7c2e618 100644 --- a/models/tmp/stg_zendesk__ticket_tag_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tag_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='ticket_tag', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket_tag', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_tag' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tmp.sql b/models/tmp/stg_zendesk__ticket_tmp.sql index f986522..3db4852 100644 --- a/models/tmp/stg_zendesk__ticket_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='ticket', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='ticket', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__time_zone_tmp.sql b/models/tmp/stg_zendesk__time_zone_tmp.sql index dd509d7..76aec90 100644 --- a/models/tmp/stg_zendesk__time_zone_tmp.sql +++ b/models/tmp/stg_zendesk__time_zone_tmp.sql @@ -2,14 +2,9 @@ {{ config(enabled=var('using_schedules', True)) }} {{ - fivetran_utils.union_data( - table_identifier='time_zone', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='time_zone', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='time_zone' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tag_tmp.sql b/models/tmp/stg_zendesk__user_tag_tmp.sql index 6b2922b..47a6d42 100644 --- a/models/tmp/stg_zendesk__user_tag_tmp.sql +++ b/models/tmp/stg_zendesk__user_tag_tmp.sql @@ -1,15 +1,9 @@ --To disable this model, set the using_user_tags variable within your dbt_project.yml file to False. {{ config(enabled=var('using_user_tags', True)) }} - {{ - fivetran_utils.union_data( - table_identifier='user_tag', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='user_tag', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='user_tag' ) }} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tmp.sql b/models/tmp/stg_zendesk__user_tmp.sql index 9d72a6d..7dfad55 100644 --- a/models/tmp/stg_zendesk__user_tmp.sql +++ b/models/tmp/stg_zendesk__user_tmp.sql @@ -1,12 +1,7 @@ {{ - fivetran_utils.union_data( - table_identifier='user', - database_variable='zendesk_database', - schema_variable='zendesk_schema', - default_database=target.database, - default_schema='zendesk', - default_variable='user', - union_schema_variable='zendesk_union_schemas', - union_database_variable='zendesk_union_databases' + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='user' ) }} \ No newline at end of file From a14fc9fdf20b5132388eac47b102521bdec7a7a1 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:02:28 -0800 Subject: [PATCH 14/15] fix source relation --- macros/union/zendesk_source_relation.sql | 4 ++-- models/stg_zendesk__audit_log.sql | 3 +-- models/stg_zendesk__brand.sql | 3 +-- models/stg_zendesk__daylight_time.sql | 3 +-- models/stg_zendesk__domain_name.sql | 3 +-- models/stg_zendesk__group.sql | 3 +-- models/stg_zendesk__organization.sql | 3 +-- models/stg_zendesk__organization_tag.sql | 3 +-- models/stg_zendesk__schedule.sql | 3 +-- models/stg_zendesk__schedule_holiday.sql | 3 +-- models/stg_zendesk__ticket.sql | 3 +-- models/stg_zendesk__ticket_comment.sql | 3 +-- models/stg_zendesk__ticket_field_history.sql | 3 +-- models/stg_zendesk__ticket_form_history.sql | 3 +-- models/stg_zendesk__ticket_schedule.sql | 3 +-- models/stg_zendesk__ticket_tag.sql | 3 +-- models/stg_zendesk__time_zone.sql | 3 +-- models/stg_zendesk__user.sql | 3 +-- models/stg_zendesk__user_tag.sql | 3 +-- 19 files changed, 20 insertions(+), 38 deletions(-) diff --git a/macros/union/zendesk_source_relation.sql b/macros/union/zendesk_source_relation.sql index a5bab64..76ba91c 100644 --- a/macros/union/zendesk_source_relation.sql +++ b/macros/union/zendesk_source_relation.sql @@ -1,4 +1,4 @@ -{% macro zendesk_source_relation(connection_dictionary, single_schema, single_database, single_table_identifier) -%} +{% macro zendesk_source_relation(connection_dictionary, single_schema, single_database) -%} {{ adapter.dispatch('zendesk_source_relation', 'zendesk_source') (connection_dictionary, single_schema, single_database, single_table_identifier) }} @@ -9,7 +9,7 @@ {% if connection_dictionary %} , _dbt_source_relation as source_relation {% else %} -, '{{ single_database }}' || '.'|| '{{ single_schema }}' || '.'|| '{{ single_table_identifier }}' as source_relation +, '{{ single_database }}' || '.'|| '{{ single_schema }}' as source_relation {% endif %} {%- endmacro %} \ No newline at end of file diff --git a/models/stg_zendesk__audit_log.sql b/models/stg_zendesk__audit_log.sql index ab9f22f..020a277 100644 --- a/models/stg_zendesk__audit_log.sql +++ b/models/stg_zendesk__audit_log.sql @@ -27,8 +27,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_audit_log_identifier", "audit_log") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__brand.sql b/models/stg_zendesk__brand.sql index 98f0cad..968fe14 100644 --- a/models/stg_zendesk__brand.sql +++ b/models/stg_zendesk__brand.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_brand_identifier", "brand") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__daylight_time.sql b/models/stg_zendesk__daylight_time.sql index 206953d..03d97c3 100644 --- a/models/stg_zendesk__daylight_time.sql +++ b/models/stg_zendesk__daylight_time.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_daylight_time_identifier", "daylight_time") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__domain_name.sql b/models/stg_zendesk__domain_name.sql index 34263e8..a853d55 100644 --- a/models/stg_zendesk__domain_name.sql +++ b/models/stg_zendesk__domain_name.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_domain_name_identifier", "domain_name") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__group.sql b/models/stg_zendesk__group.sql index 3b5f2c7..62a32fd 100644 --- a/models/stg_zendesk__group.sql +++ b/models/stg_zendesk__group.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_group_identifier", "group") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__organization.sql b/models/stg_zendesk__organization.sql index 90bbe5d..aeba102 100644 --- a/models/stg_zendesk__organization.sql +++ b/models/stg_zendesk__organization.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_organization_identifier", "organization") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__organization_tag.sql b/models/stg_zendesk__organization_tag.sql index 3beee31..1fb922e 100644 --- a/models/stg_zendesk__organization_tag.sql +++ b/models/stg_zendesk__organization_tag.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_organization_tag_identifier", "organization_tag") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__schedule.sql b/models/stg_zendesk__schedule.sql index 6a060f0..3f327ef 100644 --- a/models/stg_zendesk__schedule.sql +++ b/models/stg_zendesk__schedule.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_schedule_identifier", "schedule") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__schedule_holiday.sql b/models/stg_zendesk__schedule_holiday.sql index 122970c..d6c07cf 100644 --- a/models/stg_zendesk__schedule_holiday.sql +++ b/models/stg_zendesk__schedule_holiday.sql @@ -21,8 +21,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_schedule_holiday_identifier", "schedule_holiday") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__ticket.sql b/models/stg_zendesk__ticket.sql index fbb0ade..4c4e615 100644 --- a/models/stg_zendesk__ticket.sql +++ b/models/stg_zendesk__ticket.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_identifier", "ticket") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__ticket_comment.sql b/models/stg_zendesk__ticket_comment.sql index 3ea6993..1676b6c 100644 --- a/models/stg_zendesk__ticket_comment.sql +++ b/models/stg_zendesk__ticket_comment.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_comment_identifier", "ticket_comment") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index 3842948..b2d639a 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_field_history_identifier", "ticket_field_history") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__ticket_form_history.sql b/models/stg_zendesk__ticket_form_history.sql index a73b456..a643a19 100644 --- a/models/stg_zendesk__ticket_form_history.sql +++ b/models/stg_zendesk__ticket_form_history.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_form_history_identifier", "ticket_form_history") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__ticket_schedule.sql b/models/stg_zendesk__ticket_schedule.sql index 4977dbe..9771ff1 100644 --- a/models/stg_zendesk__ticket_schedule.sql +++ b/models/stg_zendesk__ticket_schedule.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_schedule_identifier", "ticket_schedule") + single_database=var('zendesk_schema', target.database) ) }} from base diff --git a/models/stg_zendesk__ticket_tag.sql b/models/stg_zendesk__ticket_tag.sql index 9336128..2aca06b 100644 --- a/models/stg_zendesk__ticket_tag.sql +++ b/models/stg_zendesk__ticket_tag.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_ticket_tag_identifier", "ticket_tag") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__time_zone.sql b/models/stg_zendesk__time_zone.sql index f0a85a0..df21ac3 100644 --- a/models/stg_zendesk__time_zone.sql +++ b/models/stg_zendesk__time_zone.sql @@ -22,8 +22,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_time_zone_identifier", "time_zone") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__user.sql b/models/stg_zendesk__user.sql index 50e6954..384ad69 100644 --- a/models/stg_zendesk__user.sql +++ b/models/stg_zendesk__user.sql @@ -26,8 +26,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_user_identifier", "user") + single_database=var('zendesk_schema', target.database) ) }} diff --git a/models/stg_zendesk__user_tag.sql b/models/stg_zendesk__user_tag.sql index bb3f480..c5c0e6e 100644 --- a/models/stg_zendesk__user_tag.sql +++ b/models/stg_zendesk__user_tag.sql @@ -28,8 +28,7 @@ fields as ( zendesk_source.zendesk_source_relation( connection_dictionary=var('zendesk_sources', []), single_schema=var('zendesk_schema', 'zendesk'), - single_database=var('zendesk_schema', target.database), - single_table_identifier=var("zendesk_user_tag_identifier", "user_tag") + single_database=var('zendesk_schema', target.database) ) }} From af6cad5f40ceaee12509fc92e732b4e28ff14bd6 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:34:45 -0800 Subject: [PATCH 15/15] fix --- macros/union/zendesk_source_relation.sql | 4 ++-- macros/union/zendesk_union_relations.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/union/zendesk_source_relation.sql b/macros/union/zendesk_source_relation.sql index 76ba91c..bb41a69 100644 --- a/macros/union/zendesk_source_relation.sql +++ b/macros/union/zendesk_source_relation.sql @@ -1,10 +1,10 @@ {% macro zendesk_source_relation(connection_dictionary, single_schema, single_database) -%} -{{ adapter.dispatch('zendesk_source_relation', 'zendesk_source') (connection_dictionary, single_schema, single_database, single_table_identifier) }} +{{ adapter.dispatch('zendesk_source_relation', 'zendesk_source') (connection_dictionary, single_schema, single_database) }} {%- endmacro %} -{% macro default__zendesk_source_relation(connection_dictionary, single_schema, single_database, single_table_identifier) -%} +{% macro default__zendesk_source_relation(connection_dictionary, single_schema, single_database) -%} {% if connection_dictionary %} , _dbt_source_relation as source_relation diff --git a/macros/union/zendesk_union_relations.sql b/macros/union/zendesk_union_relations.sql index 75a8085..8a791e3 100644 --- a/macros/union/zendesk_union_relations.sql +++ b/macros/union/zendesk_union_relations.sql @@ -102,7 +102,7 @@ select {%- if source_column_name is not none %} - cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, + cast({{ dbt.string_literal(relation.database ~ '.' ~ relation.schema) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, {%- endif %} {% for col_name in ordered_column_names -%}