diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d611e..829e46b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_zendesk_source v0.14.0 + +## Feature Update: Run Package on Unioned Connectors +- This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk_source/pull/44)). + # dbt_zendesk_source v0.13.0 [PR #55](https://github.com/fivetran/dbt_zendesk_source/pull/55) includes the following updates: diff --git a/README.md b/README.md index 05f5fb3..aa64e2f 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,10 @@ Include the following zendesk_source package version in your `packages.yml` file ```yaml packages: - package: fivetran/zendesk_source - version: [">=0.13.0", "<0.14.0"] + version: [">=0.14.0", "<0.15.0"] ``` ### Step 3: Define database and schema variables +#### Option 1: Single connector By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk Support data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -54,8 +55,57 @@ vars: zendesk_database: your_destination_name zendesk_schema: your_schema_name ``` -### Step 4: Enable/Disable models for non-existent sources -This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable: +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. + +#### Option 2: Union multiple connectors +If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + zendesk_union_schemas: ['zendesk_usa','zendesk_canada'] # use this if the data is in different schemas/datasets of the same database/project + zendesk_union_databases: ['zendesk_usa','zendesk_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +##### Recommended: Incorporate unioned sources into DAG +By default, this package defines one single-connector source, called `zendesk`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Zendesk sources, though the package will run successfully. + +To properly incorporate all of your Zendesk connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_zendesk.yml` [file](https://github.com/fivetran/dbt_zendesk_source/blob/main/models/src_zendesk.yml#L15-L351). + +```yml +# a .yml file in your root project +sources: + - name: # ex: zendesk_usa + schema: # one of var('zendesk_union_schemas') if unioning schemas, otherwise just 'zendesk' + database: # one of var('zendesk_union_databases') if unioning databases, otherwise whatever DB your zendesk schemas all live in + loader: fivetran + loaded_at_field: _fivetran_synced + + freshness: # feel free to adjust to your liking + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} + + tables: # copy and paste from models/src_zendesk.yml +``` + +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. + +2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to `True`, like such: +```yml +# dbt_project.yml +vars: + zendesk_source: + has_defined_sources: true +``` + +### Step 4: Disable models for non-existent sources +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ + +This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. + +By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable: ```yml vars: using_schedule_histories: True #Enable if you are using audit_logs for schedule histories @@ -117,8 +167,8 @@ models: +schema: my_new_schema_name # leave blank for just the target_schema ``` -#### Change the source table references -If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: +### Change the source table references (only if using a single connector) +If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable. This is not available when running the package on multiple unioned connectors. > IMPORTANT: See this project's [dbt_project.yml](https://github.com/fivetran/dbt_zendesk_source/blob/main/dbt_project.yml) variable declarations to see the expected names. ```yml diff --git a/dbt_project.yml b/dbt_project.yml index 61044c7..f46ffa6 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,8 @@ config-version: 2 name: 'zendesk_source' -version: '0.13.0' +version: '0.14.0' require-dbt-version: [">=1.3.0", "<2.0.0"] + models: zendesk_source: materialized: table diff --git a/macros/union/union_zendesk_connections.sql b/macros/union/union_zendesk_connections.sql new file mode 100644 index 0000000..195a7ee --- /dev/null +++ b/macros/union/union_zendesk_connections.sql @@ -0,0 +1,63 @@ +{% macro union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} + +{{ adapter.dispatch('union_zendesk_connections', 'zendesk_source') (connection_dictionary, single_source_name, single_table_name) }} + +{%- endmacro %} + +{% macro default__union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} + +{% if connection_dictionary %} +{# For unioning #} + {%- set relations = [] -%} + {%- for connection in connection_dictionary -%} + + {%- set relation=source(connection.name, single_table_name) if var('has_defined_sources', false) + else adapter.get_relation( + database=connection.database if connection.database else target.database, + schema=connection.schema if connection.schema else single_source_name, + identifier=single_table_name + ) + -%} + + {%- if relation is not none -%} + {%- do relations.append(relation) -%} + {%- endif -%} + + {%- endfor -%} + + {%- if relations != [] -%} + {{ zendesk_source.zendesk_union_relations(relations) }} + {%- else -%} + {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} + {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name ~ "." ~ single_table_name ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} + {% endif -%} + select + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + limit 0 + {%- endif -%} + +{% else %} +{# Not unioning #} + {%- set relation=adapter.get_relation( + database=source(single_source_name, single_table_name).database, + schema=source(single_source_name, single_table_name).schema, + identifier=source(single_source_name, single_table_name).identifier + ) -%} + + {%- if relation is not none -%} + select + {{ dbt_utils.star(from=source(single_source_name, single_table_name)) }} + from {{ source(single_source_name, single_table_name) }} as source_table + + {% else %} + {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} + {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name|upper ~ "." ~ single_table_name|upper ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} + {% endif -%} + + select + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + limit 0 + {%- endif -%} +{% endif -%} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/union/zendesk_source_relation.sql b/macros/union/zendesk_source_relation.sql new file mode 100644 index 0000000..bb41a69 --- /dev/null +++ b/macros/union/zendesk_source_relation.sql @@ -0,0 +1,15 @@ +{% macro zendesk_source_relation(connection_dictionary, single_schema, single_database) -%} + +{{ adapter.dispatch('zendesk_source_relation', 'zendesk_source') (connection_dictionary, single_schema, single_database) }} + +{%- endmacro %} + +{% macro default__zendesk_source_relation(connection_dictionary, single_schema, single_database) -%} + +{% if connection_dictionary %} +, _dbt_source_relation as source_relation +{% else %} +, '{{ single_database }}' || '.'|| '{{ single_schema }}' as source_relation +{% endif %} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/union/zendesk_union_relations.sql b/macros/union/zendesk_union_relations.sql new file mode 100644 index 0000000..8a791e3 --- /dev/null +++ b/macros/union/zendesk_union_relations.sql @@ -0,0 +1,131 @@ +{# Adapted from dbt_utils.union_relations() #} + +{%- macro zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} + {{ return(adapter.dispatch('zendesk_union_relations', 'zendesk_source')(relations, aliases, column_override, include, exclude, source_column_name, where)) }} +{% endmacro %} + +{%- macro default__zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} + + {%- if exclude and include -%} + {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} + {%- endif -%} + + {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} + {%- if not execute %} + {{ return('') }} + {% endif -%} + + {%- set column_override = column_override if column_override is not none else {} -%} + + {%- set relation_columns = {} -%} + {%- set column_superset = {} -%} + {%- set all_excludes = [] -%} + {%- set all_includes = [] -%} + + {%- if exclude -%} + {%- for exc in exclude -%} + {%- do all_excludes.append(exc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- if include -%} + {%- for inc in include -%} + {%- do all_includes.append(inc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- for relation in relations -%} + + {%- do relation_columns.update({relation: []}) -%} + + {%- do dbt_utils._is_relation(relation, 'zendesk_union_relations') -%} + {%- do dbt_utils._is_ephemeral(relation, 'zendesk_union_relations') -%} + {%- set cols = adapter.get_columns_in_relation(relation) -%} + {%- for col in cols -%} + + {#- If an exclude list was provided and the column is in the list, do nothing -#} + {%- if exclude and col.column | lower in all_excludes -%} + + {#- If an include list was provided and the column is not in the list, do nothing -#} + {%- elif include and col.column | lower not in all_includes -%} + + {#- Otherwise add the column to the column superset -#} + {%- else -%} + + {#- update the list of columns in this relation -#} + {%- do relation_columns[relation].append(col.column) -%} + + {%- if col.column in column_superset -%} + + {%- set stored = column_superset[col.column] -%} + {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} + + {%- do column_superset.update({col.column: col}) -%} + + {%- endif %} + + {%- else -%} + + {%- do column_superset.update({col.column: col}) -%} + + {%- endif -%} + + {%- endif -%} + + {%- endfor -%} + {%- endfor -%} + + {%- set ordered_column_names = column_superset.keys() -%} + {%- set dbt_command = flags.WHICH -%} + + + {% if dbt_command in ['run', 'build'] %} + {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} + {%- set relations_string -%} + {%- for relation in relations -%} + {{ relation.name }} + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endset -%} + + {%- set error_message -%} + There were no columns found to union for relations {{ relations_string }} + {%- endset -%} + + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} + {%- endif -%} + + {%- for relation in relations %} + + ( + select + + {%- if source_column_name is not none %} + cast({{ dbt.string_literal(relation.database ~ '.' ~ relation.schema) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, + {%- endif %} + + {% for col_name in ordered_column_names -%} + + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + + {%- endfor %} + + {# This alias is the only addition made to thr dbt_utils.union_relations() code. Avoids errors if the table is named a reserved keyword #} + from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }} + + {% if where -%} + where {{ where }} + {%- endif %} + ) + + {% if not loop.last -%} + union all + {% endif -%} + + {%- endfor -%} + +{%- endmacro -%} \ No newline at end of file diff --git a/models/src_zendesk.yml b/models/src_zendesk.yml index 6bafaf1..03ffc74 100644 --- a/models/src_zendesk.yml +++ b/models/src_zendesk.yml @@ -2,16 +2,18 @@ version: 2 sources: - name: zendesk - schema: "{{var ('zendesk_schema', 'zendesk')}}" + schema: "{{ var('zendesk_schema', 'zendesk') }}" database: "{% if target.type != 'spark'%}{{ var('zendesk_database', target.database) }}{% endif %}" loader: fivetran loaded_at_field: _fivetran_synced + + config: + enabled: "{{ var('zendesk_sources', []) == [] }}" freshness: warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: - name: audit_log identifier: "{{ var('zendesk_audit_log_identifier', 'audit_log')}}" diff --git a/models/stg_zendesk.yml b/models/stg_zendesk.yml index 8d861e2..bf1b735 100644 --- a/models/stg_zendesk.yml +++ b/models/stg_zendesk.yml @@ -31,11 +31,15 @@ models: description: > Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_id + - source_relation columns: - name: ticket_id description: Automatically assigned when the ticket is created tests: - - unique - not_null - name: url description: The API url of this ticket @@ -84,7 +88,11 @@ models: - name: source_to_address description: The address of the source the ticket was created from - name: source_to_name - description: The name of the source the ticket was created from + description: The name of the source the ticket was created from + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: _fivetran_deleted description: Boolean created by Fivetran to indicate whether the record has been deleted. @@ -92,11 +100,15 @@ models: description: > Brands are your customer-facing identities. They might represent multiple products or services, or they might literally be multiple brands owned and represented by your company. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - brand_id + - source_relation columns: - name: brand_id description: The ID automatically assigned when the brand is created tests: - - unique - not_null - name: brand_url description: The url of the brand @@ -106,6 +118,10 @@ models: description: The subdomain of the brand - name: active description: If the brand is set as active + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__domain_name description: Domain names associated with an organization. An organization may have multiple domain names. @@ -118,6 +134,10 @@ models: description: The name of the domain associated with the organization - name: index description: Index number of the domain name associated with the organization + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__group description: > @@ -125,14 +145,22 @@ models: element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being assigned to a Group. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - group_id + - source_relation columns: - name: group_id description: Automatically assigned when creating groups tests: - - unique - not_null - name: name description: The name of the group + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__organization_tag description: The tags associated with an organization. An organization may have multiple tags. @@ -141,6 +169,10 @@ models: description: Reference to the organization - name: tag description: Tag associated with the organization + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__organization description: > @@ -148,24 +180,36 @@ models: organizations. You can manually assign customers to an organization or automatically assign them to an organization by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or to send email notifications. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - organization_id + - source_relation columns: - name: organization_id description: Automatically assigned when the organization is created tests: - - unique - not_null - name: name description: A unique name for the organization - name: details description: Any details obout the organization, such as the address + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_comment description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_comment_id + - source_relation columns: - name: ticket_comment_id description: Automatically assigned when the comment is created tests: - - unique - not_null - name: body description: The comment string @@ -183,6 +227,10 @@ models: description: Boolean field indicating if the comment is a twitter tweet - name: is_voice_comment description: Boolean field indicating if the comment is a voice comment + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: _fivetran_deleted description: Boolean created by Fivetran to indicate whether the record has been deleted. @@ -193,6 +241,10 @@ models: description: Reference to the user - name: tag description: Tag associated with the user + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__user description: Zendesk Support has three types of users, end-users (your customers), agents, and administrators. @@ -200,7 +252,6 @@ models: - name: user_id description: Automatically assigned when the user is created tests: - - unique - not_null - name: email description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address @@ -220,6 +271,10 @@ models: description: The user's time zone. See Time Zone - name: ticket_restriction description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: is_suspended description: Boolean representing whether the user has been suspended, meaning that they can no longer sign in and any new support requests you receive from them are sent to the suspended tickets queue. - name: external_id @@ -234,10 +289,12 @@ models: description: Boolean created by Fivetran to indicate whether the record has been deleted. - name: stg_zendesk__schedule - description: The support schedules created with different business hours and holidays. + description: The support schedules created with different business hours and holidays. columns: - name: schedule_id description: ID automatically assigned to the schedule upon creation + tests: + - not_null - name: schedule_name description: Name of the schedule - name: created_at @@ -248,7 +305,11 @@ models: description: End time of the schedule, in the schedule's time zone. - name: time_zone description: Timezone in which the schedule operates. - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. + - name: stg_zendesk__ticket_schedule description: The schedules applied to tickets through a trigger. columns: @@ -258,7 +319,11 @@ models: description: The time the schedule was assigned to the ticket - name: schedule_id description: The ID of the schedule applied to the ticket - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. + - name: stg_zendesk__ticket_form_history description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. columns: @@ -276,6 +341,10 @@ models: description: If the form is set as active - name: name description: The name of the form + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_tag description: > @@ -286,7 +355,10 @@ models: description: The ID of the ticket associated with the tag - name: tags description: The tag, or word(s), associated with the ticket - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__ticket_field_history description: All fields and field values associated with tickets. @@ -303,6 +375,10 @@ models: description: The value of the field - name: user_id description: The id of the user who made the update + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__daylight_time description: > @@ -311,7 +387,8 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - time_zone - - year + - year + - source_relation columns: - name: daylight_end_utc description: UTC timestamp of when Daylight Time ended in this year. @@ -325,33 +402,53 @@ models: description: Year in which daylight savings occurred. - name: daylight_offset_minutes description: Number of **minutes** added during Daylight Savings Time. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__time_zone description: Offsets (from UTC) for each timezone. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - time_zone + - source_relation columns: - name: time_zone description: Name of the time zone. tests: - - unique - not_null - name: standard_offset description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. - name: standard_offset_minutes description: Standard offset of the timezone (non-daylight savings hours) in minutes. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: stg_zendesk__schedule_holiday description: Information about holidays for each specified schedule. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - holiday_id + - source_relation columns: - name: end_date_at description: ISO 8601 representation of the holiday end date. - name: holiday_id description: The ID of the scheduled holiday. tests: - - unique - not_null - name: holiday_name description: Name of the holiday. - name: schedule_id description: The ID of the schedule. - name: start_date_at - description: ISO 8601 representation of the holiday start date. \ No newline at end of file + description: ISO 8601 representation of the holiday start date. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. \ No newline at end of file diff --git a/models/stg_zendesk__audit_log.sql b/models/stg_zendesk__audit_log.sql index b119847..020a277 100644 --- a/models/stg_zendesk__audit_log.sql +++ b/models/stg_zendesk__audit_log.sql @@ -22,6 +22,14 @@ fields as ( staging_columns=get_audit_log_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} from base ), diff --git a/models/stg_zendesk__brand.sql b/models/stg_zendesk__brand.sql index 48ca08a..968fe14 100644 --- a/models/stg_zendesk__brand.sql +++ b/models/stg_zendesk__brand.sql @@ -21,6 +21,14 @@ fields as ( staging_columns=get_brand_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} from base ), @@ -32,7 +40,9 @@ final as ( brand_url, name, subdomain, - active as is_active + active as is_active, + source_relation + from fields where not coalesce(_fivetran_deleted, false) ) diff --git a/models/stg_zendesk__daylight_time.sql b/models/stg_zendesk__daylight_time.sql index bd132b8..03d97c3 100644 --- a/models/stg_zendesk__daylight_time.sql +++ b/models/stg_zendesk__daylight_time.sql @@ -24,6 +24,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -35,7 +43,8 @@ final as ( daylight_start_utc, time_zone, year, - daylight_offset * 60 as daylight_offset_minutes + daylight_offset * 60 as daylight_offset_minutes, + source_relation from fields ) diff --git a/models/stg_zendesk__domain_name.sql b/models/stg_zendesk__domain_name.sql index f48e154..a853d55 100644 --- a/models/stg_zendesk__domain_name.sql +++ b/models/stg_zendesk__domain_name.sql @@ -23,6 +23,14 @@ fields as ( staging_columns=get_domain_name_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} from base ), @@ -32,7 +40,9 @@ final as ( select organization_id, domain_name, - index + index, + source_relation + from fields ) diff --git a/models/stg_zendesk__group.sql b/models/stg_zendesk__group.sql index c85af24..62a32fd 100644 --- a/models/stg_zendesk__group.sql +++ b/models/stg_zendesk__group.sql @@ -21,7 +21,15 @@ fields as ( staging_columns=get_group_columns() ) }} - + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -29,9 +37,10 @@ final as ( select id as group_id, - name + name, + source_relation + from fields - where not coalesce(_fivetran_deleted, false) ) diff --git a/models/stg_zendesk__organization.sql b/models/stg_zendesk__organization.sql index db6827a..aeba102 100644 --- a/models/stg_zendesk__organization.sql +++ b/models/stg_zendesk__organization.sql @@ -22,6 +22,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -33,7 +41,8 @@ final as ( updated_at, details, name, - external_id + external_id, + source_relation {{ fivetran_utils.fill_pass_through_columns('zendesk__organization_passthrough_columns') }} diff --git a/models/stg_zendesk__organization_tag.sql b/models/stg_zendesk__organization_tag.sql index a69ff86..1fb922e 100644 --- a/models/stg_zendesk__organization_tag.sql +++ b/models/stg_zendesk__organization_tag.sql @@ -24,6 +24,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -36,7 +44,9 @@ final as ( {% else %} tag {% endif %} - as tags + as tags, + source_relation + from fields ) diff --git a/models/stg_zendesk__schedule.sql b/models/stg_zendesk__schedule.sql index 6ab434b..3f327ef 100644 --- a/models/stg_zendesk__schedule.sql +++ b/models/stg_zendesk__schedule.sql @@ -24,6 +24,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -35,7 +43,8 @@ final as ( start_time, name as schedule_name, created_at, - time_zone + time_zone, + source_relation from fields where not coalesce(_fivetran_deleted, false) diff --git a/models/stg_zendesk__schedule_holiday.sql b/models/stg_zendesk__schedule_holiday.sql index 22e5b27..d6c07cf 100644 --- a/models/stg_zendesk__schedule_holiday.sql +++ b/models/stg_zendesk__schedule_holiday.sql @@ -16,6 +16,15 @@ fields as ( staging_columns=get_schedule_holiday_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -28,7 +37,9 @@ final as ( cast(id as {{ dbt.type_string() }} ) as holiday_id, name as holiday_name, cast(schedule_id as {{ dbt.type_string() }} ) as schedule_id, - cast(start_date as {{ dbt.type_timestamp() }} ) as holiday_start_date_at + cast(start_date as {{ dbt.type_timestamp() }} ) as holiday_start_date_at, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket.sql b/models/stg_zendesk__ticket.sql index a60a1a2..4c4e615 100644 --- a/models/stg_zendesk__ticket.sql +++ b/models/stg_zendesk__ticket.sql @@ -21,6 +21,14 @@ fields as ( staging_columns=get_ticket_columns() ) }} + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} from base ), @@ -56,7 +64,8 @@ final as ( via_source_from_title as source_from_title, via_source_rel as source_rel, via_source_to_address as source_to_address, - via_source_to_name as source_to_name + via_source_to_name as source_to_name, + source_relation {{ fivetran_utils.fill_pass_through_columns('zendesk__ticket_passthrough_columns') }} diff --git a/models/stg_zendesk__ticket_comment.sql b/models/stg_zendesk__ticket_comment.sql index a7208b8..1676b6c 100644 --- a/models/stg_zendesk__ticket_comment.sql +++ b/models/stg_zendesk__ticket_comment.sql @@ -22,6 +22,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -38,7 +46,9 @@ final as ( user_id, facebook_comment as is_facebook_comment, tweet as is_tweet, - voice_comment as is_voice_comment + voice_comment as is_voice_comment, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket_field_history.sql b/models/stg_zendesk__ticket_field_history.sql index e7f5c3b..b2d639a 100644 --- a/models/stg_zendesk__ticket_field_history.sql +++ b/models/stg_zendesk__ticket_field_history.sql @@ -22,6 +22,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -31,9 +39,11 @@ final as ( ticket_id, field_name, cast(updated as {{ dbt.type_timestamp() }}) as valid_starting_at, - cast(lead(updated) over (partition by ticket_id, field_name order by updated) as {{ dbt.type_timestamp() }}) as valid_ending_at, + cast(lead(updated) over (partition by ticket_id, field_name, source_relation order by updated) as {{ dbt.type_timestamp() }}) as valid_ending_at, value, - user_id + user_id, + source_relation + from fields ) diff --git a/models/stg_zendesk__ticket_form_history.sql b/models/stg_zendesk__ticket_form_history.sql index 9a7a820..a643a19 100644 --- a/models/stg_zendesk__ticket_form_history.sql +++ b/models/stg_zendesk__ticket_form_history.sql @@ -23,7 +23,15 @@ fields as ( staging_columns=get_ticket_form_history_columns() ) }} - + + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -35,7 +43,9 @@ final as ( cast(updated_at as {{ dbt.type_timestamp() }}) as updated_at, display_name, active as is_active, - name + name, + source_relation + from fields where not coalesce(_fivetran_deleted, false) diff --git a/models/stg_zendesk__ticket_schedule.sql b/models/stg_zendesk__ticket_schedule.sql index bba8661..9771ff1 100644 --- a/models/stg_zendesk__ticket_schedule.sql +++ b/models/stg_zendesk__ticket_schedule.sql @@ -24,6 +24,13 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} from base ), @@ -32,7 +39,8 @@ final as ( select ticket_id, cast(created_at as {{ dbt.type_timestamp() }}) as created_at, - cast(schedule_id as {{ dbt.type_string() }}) as schedule_id --need to convert from numeric to string for downstream models to work properly + cast(schedule_id as {{ dbt.type_string() }}) as schedule_id, --need to convert from numeric to string for downstream models to work properly + source_relation from fields ) diff --git a/models/stg_zendesk__ticket_tag.sql b/models/stg_zendesk__ticket_tag.sql index d44c37b..2aca06b 100644 --- a/models/stg_zendesk__ticket_tag.sql +++ b/models/stg_zendesk__ticket_tag.sql @@ -22,6 +22,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -30,10 +38,12 @@ final as ( select ticket_id, {% if target.type == 'redshift' %} - "tag" as tags + "tag" {% else %} - tag as tags - {% endif %} + tag + {% endif %} as tags, + source_relation + from fields ) diff --git a/models/stg_zendesk__time_zone.sql b/models/stg_zendesk__time_zone.sql index ddfa045..df21ac3 100644 --- a/models/stg_zendesk__time_zone.sql +++ b/models/stg_zendesk__time_zone.sql @@ -18,6 +18,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -30,7 +38,8 @@ final as ( -- let's convert it to an integer value of minutes cast( {{ dbt.split_part(string_text='standard_offset', delimiter_text="':'", part_number=1) }} as {{ dbt.type_int() }} ) * 60 + (cast( {{ dbt.split_part(string_text='standard_offset', delimiter_text="':'", part_number=2) }} as {{ dbt.type_int() }} ) * - (case when standard_offset like '-%' then -1 else 1 end) ) as standard_offset_minutes + (case when standard_offset like '-%' then -1 else 1 end) ) as standard_offset_minutes, + source_relation from fields ) diff --git a/models/stg_zendesk__user.sql b/models/stg_zendesk__user.sql index 6760cc5..384ad69 100644 --- a/models/stg_zendesk__user.sql +++ b/models/stg_zendesk__user.sql @@ -22,6 +22,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -51,10 +59,11 @@ final as ( time_zone, locale, active as is_active, - suspended as is_suspended + suspended as is_suspended, + source_relation {{ fivetran_utils.fill_pass_through_columns('zendesk__user_passthrough_columns') }} - + from fields ) diff --git a/models/stg_zendesk__user_tag.sql b/models/stg_zendesk__user_tag.sql index fc194bc..c5c0e6e 100644 --- a/models/stg_zendesk__user_tag.sql +++ b/models/stg_zendesk__user_tag.sql @@ -24,6 +24,14 @@ fields as ( ) }} + {{ + zendesk_source.zendesk_source_relation( + connection_dictionary=var('zendesk_sources', []), + single_schema=var('zendesk_schema', 'zendesk'), + single_database=var('zendesk_schema', target.database) + ) + }} + from base ), @@ -36,7 +44,9 @@ final as ( {% else %} tag {% endif %} - as tags + as tags, + source_relation + from fields ) diff --git a/models/tmp/stg_zendesk__audit_log_tmp.sql b/models/tmp/stg_zendesk__audit_log_tmp.sql index fbf42ee..065ec3e 100644 --- a/models/tmp/stg_zendesk__audit_log_tmp.sql +++ b/models/tmp/stg_zendesk__audit_log_tmp.sql @@ -1,4 +1,9 @@ {{ config(enabled=var('using_schedules', True) and var('using_schedule_histories', False)) }} -select {{ dbt_utils.star(source('zendesk','audit_log')) }} -from {{ source('zendesk','audit_log') }} as audit_log_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='audit_log' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__brand_tmp.sql b/models/tmp/stg_zendesk__brand_tmp.sql index 3dbc310..8b9a773 100644 --- a/models/tmp/stg_zendesk__brand_tmp.sql +++ b/models/tmp/stg_zendesk__brand_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk','brand')) }} -from {{ source('zendesk','brand') }} as brand_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='brand' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__daylight_time_tmp.sql b/models/tmp/stg_zendesk__daylight_time_tmp.sql index 63313a1..0c0171f 100644 --- a/models/tmp/stg_zendesk__daylight_time_tmp.sql +++ b/models/tmp/stg_zendesk__daylight_time_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'daylight_time')) }} -from {{ source('zendesk', 'daylight_time') }} as daylight_time_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='daylight_time' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__domain_name_tmp.sql b/models/tmp/stg_zendesk__domain_name_tmp.sql index 719ad1b..ce31773 100644 --- a/models/tmp/stg_zendesk__domain_name_tmp.sql +++ b/models/tmp/stg_zendesk__domain_name_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_domain_names variable within your dbt_project.yml file to False. {{ config(enabled=var('using_domain_names', True)) }} -select {{ dbt_utils.star(source('zendesk', 'domain_name')) }} -from {{ source('zendesk', 'domain_name') }} as domain_name_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='domain_name' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__group_tmp.sql b/models/tmp/stg_zendesk__group_tmp.sql index b38eb34..00a55ec 100644 --- a/models/tmp/stg_zendesk__group_tmp.sql +++ b/models/tmp/stg_zendesk__group_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk','group')) }} -from {{ source('zendesk','group') }} as group_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='group' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tag_tmp.sql b/models/tmp/stg_zendesk__organization_tag_tmp.sql index a824a3e..e814ec2 100644 --- a/models/tmp/stg_zendesk__organization_tag_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tag_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_organization_tags variable within your dbt_project.yml file to False. {{ config(enabled=var('using_organization_tags', True)) }} -select {{ dbt_utils.star(source('zendesk','organization_tag')) }} -from {{ source('zendesk','organization_tag') }} as organization_tag_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='organization_tag' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__organization_tmp.sql b/models/tmp/stg_zendesk__organization_tmp.sql index 48aa3c8..4f88156 100644 --- a/models/tmp/stg_zendesk__organization_tmp.sql +++ b/models/tmp/stg_zendesk__organization_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk', 'organization')) }} -from {{ source('zendesk','organization') }} as organization_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='organization' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql index 2b9c5ee..0387678 100644 --- a/models/tmp/stg_zendesk__schedule_holiday_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_holiday_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_schedules or using_holidays variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True) and var('using_holidays', True)) }} -select {{ dbt_utils.star(source('zendesk', 'schedule_holiday')) }} -from {{ source('zendesk', 'schedule_holiday') }} as schedule_holiday_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='schedule_holiday' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__schedule_tmp.sql b/models/tmp/stg_zendesk__schedule_tmp.sql index bdd21e7..7da0928 100644 --- a/models/tmp/stg_zendesk__schedule_tmp.sql +++ b/models/tmp/stg_zendesk__schedule_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'schedule')) }} -from {{ source('zendesk', 'schedule') }} as schedule_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='schedule' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_comment_tmp.sql b/models/tmp/stg_zendesk__ticket_comment_tmp.sql index 84d6c6a..bf8db6f 100644 --- a/models/tmp/stg_zendesk__ticket_comment_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_comment_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_comment')) }} -from {{ source('zendesk', 'ticket_comment') }} as ticket_comment_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_comment' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql index 6b61ed8..b8221c6 100644 --- a/models/tmp/stg_zendesk__ticket_field_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_field_history_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_field_history')) }} -from {{ source('zendesk', 'ticket_field_history') }} as ticket_field_history_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_field_history' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql index 60342b8..8e704b9 100644 --- a/models/tmp/stg_zendesk__ticket_form_history_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_form_history_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_ticket_form_history variable within your dbt_project.yml file to False. {{ config(enabled=var('using_ticket_form_history', True)) }} -select {{ dbt_utils.star(source('zendesk', 'ticket_form_history')) }} -from {{ source('zendesk', 'ticket_form_history') }} as ticket_form_history_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_form_history' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql index 2345b52..54ce83f 100644 --- a/models/tmp/stg_zendesk__ticket_schedule_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_schedule_tmp.sql @@ -1,24 +1,10 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -{%- set source_relation = adapter.get_relation( - database=source('zendesk', 'ticket_schedule').database, - schema=source('zendesk', 'ticket_schedule').schema, - identifier=source('zendesk', 'ticket_schedule').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} - -select {{ dbt_utils.star(source('zendesk', 'ticket_schedule')) }} -from {{ source('zendesk', 'ticket_schedule') }} as ticket_schedule_table - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_timestamp() }}) as created_at, - cast(null as {{ dbt.type_int() }}) as schedule_id, - cast(null as {{ dbt.type_int() }}) as ticket_id - -{% endif %} \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_schedule' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tag_tmp.sql b/models/tmp/stg_zendesk__ticket_tag_tmp.sql index c2a3ace..7c2e618 100644 --- a/models/tmp/stg_zendesk__ticket_tag_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tag_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket_tag')) }} -from {{ source('zendesk', 'ticket_tag') }} as ticket_tag_table +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket_tag' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__ticket_tmp.sql b/models/tmp/stg_zendesk__ticket_tmp.sql index 21c9bab..3db4852 100644 --- a/models/tmp/stg_zendesk__ticket_tmp.sql +++ b/models/tmp/stg_zendesk__ticket_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk', 'ticket')) }} -from {{ source('zendesk', 'ticket') }} as ticket_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='ticket' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__time_zone_tmp.sql b/models/tmp/stg_zendesk__time_zone_tmp.sql index 17c58a2..76aec90 100644 --- a/models/tmp/stg_zendesk__time_zone_tmp.sql +++ b/models/tmp/stg_zendesk__time_zone_tmp.sql @@ -1,5 +1,10 @@ --To disable this model, set the using_schedules variable within your dbt_project.yml file to False. {{ config(enabled=var('using_schedules', True)) }} -select {{ dbt_utils.star(source('zendesk', 'time_zone')) }} -from {{ source('zendesk', 'time_zone') }} as time_zone_table +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='time_zone' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tag_tmp.sql b/models/tmp/stg_zendesk__user_tag_tmp.sql index 94f7784..47a6d42 100644 --- a/models/tmp/stg_zendesk__user_tag_tmp.sql +++ b/models/tmp/stg_zendesk__user_tag_tmp.sql @@ -1,5 +1,9 @@ --To disable this model, set the using_user_tags variable within your dbt_project.yml file to False. {{ config(enabled=var('using_user_tags', True)) }} - -select {{ dbt_utils.star(source('zendesk','user_tag')) }} -from {{ source('zendesk','user_tag') }} as user_tag_table +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='user_tag' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_zendesk__user_tmp.sql b/models/tmp/stg_zendesk__user_tmp.sql index 7e996ea..7dfad55 100644 --- a/models/tmp/stg_zendesk__user_tmp.sql +++ b/models/tmp/stg_zendesk__user_tmp.sql @@ -1,2 +1,7 @@ -select {{ dbt_utils.star(source('zendesk','user')) }} -from {{ source('zendesk','user') }} as user_table \ No newline at end of file +{{ + zendesk_source.union_zendesk_connections( + connection_dictionary=var('zendesk_sources'), + single_source_name='zendesk', + single_table_name='user' + ) +}} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 908f471..9b0c6db 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,10 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +# - local: ../../dbt_fivetran_utils +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/enhance-union-data + warn-unpinned: false + - package: dbt-labs/spark_utils - version: [">=0.3.0", "<0.4.0"] + version: [">=0.3.0", "<0.4.0"] \ No newline at end of file