feature/fb-pages-apr-2024-api-update (#8)

* feature/fb-pages-apr-2024-api-update

* regen docs

* update yml

* update seeds

* update databricks

* Update README.md

* Update CHANGELOG.md

* Update packages.yml

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -4,48 +4,26 @@
 **This PR will result in the following new package version:**
 <!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
 
-**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
+<!--- Copy/paste the CHANGELOG for this version below. -->
 
 ## PR Checklist
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt compile
-- [ ] dbt run –full-refresh
-- [ ] dbt run
-- [ ] dbt test
-- [ ] dbt run –vars (if applicable)
+- [ ] dbt run –full-refresh && dbt test
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked and tagged
-- [ ] You are assigned to the corresponding issue and this PR
+- [ ] The appropriate issue has been linked, tagged, and properly assigned
+- [ ] All necessary documentation and version upgrades have been applied
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
+- [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
-Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
-- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
-- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
-- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
 
-### Standard Updates
-Please acknowledge that your PR contains the following standard updates:
-- Package versioning has been appropriately indexed in the following locations:
-    - [ ] indexed within dbt_project.yml
-    - [ ] indexed within integration_tests/dbt_project.yml
-- [ ] CHANGELOG has individual entries for each respective change in this PR
-    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
-- [ ] README updates have been applied (if applicable)
-    <!--- Remember to check the following README locations for common updates. →
-        <!--- Suggested install range (needed for breaking changes) →
-        <!--- Dependency matrix is appropriately updated (if applicable) →
-        <!--- New variable documentation (if applicable) -->
-- [ ] DECISIONLOG updates have been updated (if applicable)
-- [ ] Appropriate yml documentation has been added (if applicable)
-
-### dbt Docs
-Please acknowledge that after the above were all completed the below were applied to your branch:
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-
 ### If you had to summarize this PR in an emoji, which would it be?
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+:dancer:

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

CHANGELOG.md

@@ -1,3 +1,27 @@
+# dbt_social_media_reporting v0.4.0
+[PR #8](https://github.com/fivetran/dbt_social_media_reporting/pull/8) includes the following breaking changes:
+
+## 🚨 Breaking Changes 🚨:
+- This change is made breaking due to changes made in the [dbt_facebook_pages_source](https://github.com/fivetran/dbt_facebook_pages_source) and [dbt_facebook_pages](https://github.com/fivetran/dbt_facebook_pages) packages. Columns have been removed in the source package (see the [dbt_facebook_pages_source v0.3.0 CHANGELOG](https://github.com/fivetran/dbt_facebook_pages_source/blob/main/CHANGELOG.md#dbt_facebook_pages_source-v030) for more details). 
+    - No columns were changed in the end models in this package, however if you use the Facebook Pages staging models independently, you will need to update your downstream use cases accordingly.
+    - Columns removed from staging model `stg_facebook_pages__daily_page_metrics_total`:
+        - `consumptions`
+        - `content_activity`
+        - `engaged_users`
+        - `places_checkin_mobile`
+        - `views_external_referrals`
+        - `views_logged_in_total`
+        - `views_logout`
+    - Columns removed from staging model `stg_facebook_pages__lifetime_post_metrics_total`:
+        - `impressions_fan_paid`
+
+## Documentation Update
+- Updated documentation to reflect the current schema. 
+
+## Under the Hood:
+- Updated the pull request templates.
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+
 # dbt_social_media_reporting v0.3.0
 [PR #7](https://github.com/fivetran/dbt_social_media_reporting/pull/7) includes the following breaking changes:
 ## 🚨 Breaking Changes 🚨:
@@ -72,4 +96,4 @@ Currently, this package supports the following social media connector types:
 * [Facebook Pages](https://github.com/fivetran/dbt_facebook_pages)
 * [Instagram Business](https://github.com/fivetran/dbt_instagram_business)
 * [LinkedIn Company Pages](https://github.com/fivetran/dbt_linkedin_pages)
-* [Twitter Organic](https://github.com/fivetran/dbt_twitter_organic)
+* [Twitter Organic](https://github.com/fivetran/dbt_twitter_organic)

README.md

@@ -163,10 +163,10 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/facebook_pages
-      version: [">=0.2.0", "<0.3.0"]
+      version: [">=0.3.0", "<0.4.0"]
 
     - package: fivetran/facebook_pages_source
-      version: [">=0.2.0", "<0.3.0"]
+      version: [">=0.3.0", "<0.4.0"]
 
     - package: fivetran/instagram_business
       version: [">=0.2.0", "<0.3.0"]
@@ -208,4 +208,4 @@ We highly encourage and welcome contributions to this package. Check out [this p
 # 🏪 Are there any resources available?
 - If you encounter any questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_social_media_reporting/issues/new/choose) section to find the right avenue of support for you.
 - If you would like to provide feedback to the dbt package team at Fivetran, or would like to request a future dbt package to be developed, then feel free to fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).
-- Have questions or want to just say hi? Book a time during our office hours [here](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or send us an email at [email protected].
+- Have questions or want to just say hi? Book a time during our office hours [here](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or send us an email at [email protected].

integration_tests/.gitignore

@@ -4,3 +4,4 @@ dbt_modules/
 dbt_packages/
 logs/
 venv/
+package-lock.yml

integration_tests/ci/sample.profiles.yml

@@ -45,7 +45,7 @@ integration_tests:
       schema: social_media_rollup_integration_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: social_media_rollup_integration_tests

integration_tests/requirements.txt

@@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
-
-oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437
+dbt-databricks>=1.3.0,<2.0.0

integration_tests/seeds/facebook_pages_daily_page_metrics_total_data.csv

@@ -1,7 +1,7 @@
-date,page_id,_fivetran_synced,page_actions_post_reactions_anger_total,page_actions_post_reactions_haha_total,page_actions_post_reactions_like_total,page_actions_post_reactions_love_total,page_actions_post_reactions_sorry_total,page_actions_post_reactions_total,page_actions_post_reactions_wow_total,page_consumptions,page_content_activity,page_engaged_users,page_fan_adds,page_fan_removes,page_fans,page_impressions,page_impressions_nonviral,page_impressions_organic,page_impressions_paid,page_impressions_viral,page_negative_feedback,page_places_checkin_mobile,page_places_checkin_total,page_post_engagements,page_posts_impressions,page_posts_impressions_nonviral,page_posts_impressions_organic,page_posts_impressions_paid,page_posts_impressions_viral,page_total_actions,page_video_complete_views_30_s,page_video_complete_views_30_s_autoplayed,page_video_complete_views_30_s_click_to_play,page_video_complete_views_30_s_organic,page_video_complete_views_30_s_paid,page_video_complete_views_30_s_repeat_views,page_video_repeat_views,page_video_view_time,page_video_views,page_video_views_10_s,page_video_views_10_s_autoplayed,page_video_views_10_s_click_to_play,page_video_views_10_s_organic,page_video_views_10_s_paid,page_video_views_10_s_repeat,page_video_views_autoplayed,page_video_views_click_to_play,page_video_views_organic,page_video_views_paid,page_views_external_referrals,page_views_logged_in_total,page_views_logout,page_views_total,page_fans_online_per_day
-2020-04-10 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,68,0,,0,44,79,90,0,1,256,6696,250,340,6305,90,0,0,0,123,6641,250,336,6305,86,0,6,6,0,1,5,0,7,1362884,78,22,22,0,2,20,0,78,0,3,75,,43,46,81,220
-2020-04-05 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,4,0,0,,0,24,5,26,1,0,256,7691,67,69,7617,2,0,0,0,28,7686,67,69,7617,2,0,9,9,0,1,8,0,26,2711260,188,47,45,2,1,46,3,186,2,2,186,,4,17,12,223
-2020-04-06 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,6,0,0,,0,36,8,39,1,0,257,8342,10,11,8327,1,0,0,0,43,8338,10,11,8327,1,0,7,7,0,0,7,0,16,1624807,119,22,22,0,0,22,0,119,0,0,119,,5,27,22,227
-2020-04-07 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,4,0,0,,0,32,4,30,0,0,257,9590,36,40,9544,4,0,0,0,36,9584,36,40,9544,4,0,15,15,0,2,13,1,24,2027200,133,33,33,0,2,31,2,133,0,2,131,,4,52,49,222
-2020-04-11 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,20,0,,0,37,24,42,1,0,257,4245,217,241,3988,24,0,0,0,58,4227,217,239,3988,22,0,12,12,0,1,11,1,9,1403092,84,23,23,0,2,21,2,84,0,4,80,,10,50,50,224
-2020-04-09 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,3,0,,0,36,3,33,0,0,257,6940,41,44,6847,3,0,0,0,39,6891,41,44,6847,3,0,11,9,2,0,11,0,24,2327161,156,31,29,2,0,31,4,154,2,2,154,,53,47,80,219
+date,page_id,_fivetran_synced,page_actions_post_reactions_anger_total,page_actions_post_reactions_haha_total,page_actions_post_reactions_like_total,page_actions_post_reactions_love_total,page_actions_post_reactions_sorry_total,page_actions_post_reactions_total,page_actions_post_reactions_wow_total,page_fan_adds,page_fan_removes,page_fans,page_impressions,page_impressions_nonviral,page_impressions_organic,page_impressions_paid,page_impressions_viral,page_negative_feedback,page_places_checkin_total,page_post_engagements,page_posts_impressions,page_posts_impressions_nonviral,page_posts_impressions_organic,page_posts_impressions_paid,page_posts_impressions_viral,page_total_actions,page_video_complete_views_30_s,page_video_complete_views_30_s_autoplayed,page_video_complete_views_30_s_click_to_play,page_video_complete_views_30_s_organic,page_video_complete_views_30_s_paid,page_video_complete_views_30_s_repeat_views,page_video_repeat_views,page_video_view_time,page_video_views,page_video_views_10_s,page_video_views_10_s_autoplayed,page_video_views_10_s_click_to_play,page_video_views_10_s_organic,page_video_views_10_s_paid,page_video_views_10_s_repeat,page_video_views_autoplayed,page_video_views_click_to_play,page_video_views_organic,page_video_views_paid,page_views_total,page_fans_online_per_day
+2020-04-10 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,68,0,,0,0,1,256,6696,250,340,6305,90,0,0,123,6641,250,336,6305,86,0,6,6,0,1,5,0,7,1362884,78,22,22,0,2,20,0,78,0,3,75,81,220
+2020-04-05 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,4,0,0,,0,1,0,256,7691,67,69,7617,2,0,0,28,7686,67,69,7617,2,0,9,9,0,1,8,0,26,2711260,188,47,45,2,1,46,3,186,2,2,186,12,223
+2020-04-06 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,6,0,0,,0,1,0,257,8342,10,11,8327,1,0,0,43,8338,10,11,8327,1,0,7,7,0,0,7,0,16,1624807,119,22,22,0,0,22,0,119,0,0,119,22,227
+2020-04-07 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,4,0,0,,0,0,0,257,9590,36,40,9544,4,0,0,36,9584,36,40,9544,4,0,15,15,0,2,13,1,24,2027200,133,33,33,0,2,31,2,133,0,2,131,49,222
+2020-04-11 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,20,0,,0,1,0,257,4245,217,241,3988,24,0,0,58,4227,217,239,3988,22,0,12,12,0,1,11,1,9,1403092,84,23,23,0,2,21,2,84,0,4,80,50,224
+2020-04-09 07:00:00,514435155248879,2021-09-09 15:05:33.416,0,0,0,3,0,,0,0,0,257,6940,41,44,6847,3,0,0,39,6891,41,44,6847,3,0,11,9,2,0,11,0,24,2327161,156,31,29,2,0,31,4,154,2,2,154,80,219