From ce3321f140fd5668974b6692ebb3e7dbf182e4ac Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:00:20 -0600 Subject: [PATCH 01/14] cicd tests --- .github/workflows/aws_dev.yaml | 36 +++++++++++++++++++ .../dask/partition_examples.py | 0 .../databricks/consumer_flow.py | 0 .../dbt/dbt_snowflake_flow.py | 0 .../interactive-workflows/README.md | 0 .../interactive-workflow.py | 0 .../interactive-workflows/marvin_extension.py | 0 .../interactive-workflows/requirements.txt | 0 .../simple_flows/classic_flow.py | 0 {flows => WIP_demos}/simple_flows/hello.py | 0 .../simple_flows}/wave_data.py | 0 .../aws => WIP_demos/simple_flows}/weather.py | 0 flows/aws/datalake/this_is_a_test.py | 0 13 files changed, 36 insertions(+) create mode 100644 .github/workflows/aws_dev.yaml rename {flows => WIP_demos}/dask/partition_examples.py (100%) rename {flows => WIP_demos}/databricks/consumer_flow.py (100%) rename {flows => WIP_demos}/dbt/dbt_snowflake_flow.py (100%) rename {flows => WIP_demos}/machine_learning/interactive-workflows/README.md (100%) rename {flows => WIP_demos}/machine_learning/interactive-workflows/interactive-workflow.py (100%) rename {flows => WIP_demos}/machine_learning/interactive-workflows/marvin_extension.py (100%) rename {flows => WIP_demos}/machine_learning/interactive-workflows/requirements.txt (100%) rename {flows => WIP_demos}/simple_flows/classic_flow.py (100%) rename {flows => WIP_demos}/simple_flows/hello.py (100%) rename {flows/aws => WIP_demos/simple_flows}/wave_data.py (100%) rename {flows/aws => WIP_demos/simple_flows}/weather.py (100%) create mode 100644 flows/aws/datalake/this_is_a_test.py diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml new file mode 100644 index 0000000..9ad3c22 --- /dev/null +++ b/.github/workflows/aws_dev.yaml @@ -0,0 +1,36 @@ +name: Build image and deploy Prefect flow - Project 1 + +env: + PROJECT_NAME: flows/aws + +on: + push: + paths: + - "flows/aws/**" + workflow_dispatch: + +jobs: + deploy: + name: Deploy AWS flows + runs-on: ubuntu-latest + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Filter Paths + uses: dorny/paths-filter@v3 + id: filter + with: + # Changed file can be 'added', 'modified', or 'deleted'. + # By default, the type of change is not considered. + # Optionally, it's possible to specify it using nested + # dictionary, where the type of change composes the key. + # Multiple change types can be specified using `|` as the delimiter. + filters: | + addedOrModified: + - added|modified: '**' + diff --git a/flows/dask/partition_examples.py b/WIP_demos/dask/partition_examples.py similarity index 100% rename from flows/dask/partition_examples.py rename to WIP_demos/dask/partition_examples.py diff --git a/flows/databricks/consumer_flow.py b/WIP_demos/databricks/consumer_flow.py similarity index 100% rename from flows/databricks/consumer_flow.py rename to WIP_demos/databricks/consumer_flow.py diff --git a/flows/dbt/dbt_snowflake_flow.py b/WIP_demos/dbt/dbt_snowflake_flow.py similarity index 100% rename from flows/dbt/dbt_snowflake_flow.py rename to WIP_demos/dbt/dbt_snowflake_flow.py diff --git a/flows/machine_learning/interactive-workflows/README.md b/WIP_demos/machine_learning/interactive-workflows/README.md similarity index 100% rename from flows/machine_learning/interactive-workflows/README.md rename to WIP_demos/machine_learning/interactive-workflows/README.md diff --git a/flows/machine_learning/interactive-workflows/interactive-workflow.py b/WIP_demos/machine_learning/interactive-workflows/interactive-workflow.py similarity index 100% rename from flows/machine_learning/interactive-workflows/interactive-workflow.py rename to WIP_demos/machine_learning/interactive-workflows/interactive-workflow.py diff --git a/flows/machine_learning/interactive-workflows/marvin_extension.py b/WIP_demos/machine_learning/interactive-workflows/marvin_extension.py similarity index 100% rename from flows/machine_learning/interactive-workflows/marvin_extension.py rename to WIP_demos/machine_learning/interactive-workflows/marvin_extension.py diff --git a/flows/machine_learning/interactive-workflows/requirements.txt b/WIP_demos/machine_learning/interactive-workflows/requirements.txt similarity index 100% rename from flows/machine_learning/interactive-workflows/requirements.txt rename to WIP_demos/machine_learning/interactive-workflows/requirements.txt diff --git a/flows/simple_flows/classic_flow.py b/WIP_demos/simple_flows/classic_flow.py similarity index 100% rename from flows/simple_flows/classic_flow.py rename to WIP_demos/simple_flows/classic_flow.py diff --git a/flows/simple_flows/hello.py b/WIP_demos/simple_flows/hello.py similarity index 100% rename from flows/simple_flows/hello.py rename to WIP_demos/simple_flows/hello.py diff --git a/flows/aws/wave_data.py b/WIP_demos/simple_flows/wave_data.py similarity index 100% rename from flows/aws/wave_data.py rename to WIP_demos/simple_flows/wave_data.py diff --git a/flows/aws/weather.py b/WIP_demos/simple_flows/weather.py similarity index 100% rename from flows/aws/weather.py rename to WIP_demos/simple_flows/weather.py diff --git a/flows/aws/datalake/this_is_a_test.py b/flows/aws/datalake/this_is_a_test.py new file mode 100644 index 0000000..e69de29 From 2e1fa0861d14826b1a0c7b0127de2a88fd6048d8 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:03:40 -0600 Subject: [PATCH 02/14] cicd tests2 --- .github/workflows/aws_dev.yaml | 2 +- flows/aws/datalake/{this_is_a_test.py => this_is_a_test2.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename flows/aws/datalake/{this_is_a_test.py => this_is_a_test2.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index 9ad3c22..6f96550 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -30,7 +30,7 @@ jobs: # Optionally, it's possible to specify it using nested # dictionary, where the type of change composes the key. # Multiple change types can be specified using `|` as the delimiter. - filters: | + filters: addedOrModified: - added|modified: '**' diff --git a/flows/aws/datalake/this_is_a_test.py b/flows/aws/datalake/this_is_a_test2.py similarity index 100% rename from flows/aws/datalake/this_is_a_test.py rename to flows/aws/datalake/this_is_a_test2.py From 8f71107f1c71c4c9f20dcabf525408f8bf4a8c95 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:07:35 -0600 Subject: [PATCH 03/14] cicd tests3 --- .github/workflows/aws_dev.yaml | 20 +++++++++---------- ...{this_is_a_test2.py => this_is_a_test3.py} | 0 2 files changed, 10 insertions(+), 10 deletions(-) rename flows/aws/datalake/{this_is_a_test2.py => this_is_a_test3.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index 6f96550..9fa4361 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -23,14 +23,14 @@ jobs: - name: Filter Paths uses: dorny/paths-filter@v3 - id: filter - with: - # Changed file can be 'added', 'modified', or 'deleted'. - # By default, the type of change is not considered. - # Optionally, it's possible to specify it using nested - # dictionary, where the type of change composes the key. - # Multiple change types can be specified using `|` as the delimiter. - filters: - addedOrModified: - - added|modified: '**' + id: filter + with: + # Changed file can be 'added', 'modified', or 'deleted'. + # By default, the type of change is not considered. + # Optionally, it's possible to specify it using nested + # dictionary, where the type of change composes the key. + # Multiple change types can be specified using `|` as the delimiter. + filters: + addedOrModified: + - added|modified: '**' diff --git a/flows/aws/datalake/this_is_a_test2.py b/flows/aws/datalake/this_is_a_test3.py similarity index 100% rename from flows/aws/datalake/this_is_a_test2.py rename to flows/aws/datalake/this_is_a_test3.py From be80abcaf1e4473f53dfc7ceec0b233c6ae91c9a Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:09:20 -0600 Subject: [PATCH 04/14] cicd tests4 --- .github/workflows/aws_dev.yaml | 2 +- flows/aws/datalake/{this_is_a_test3.py => this_is_a_test4.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename flows/aws/datalake/{this_is_a_test3.py => this_is_a_test4.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index 9fa4361..5207317 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -30,7 +30,7 @@ jobs: # Optionally, it's possible to specify it using nested # dictionary, where the type of change composes the key. # Multiple change types can be specified using `|` as the delimiter. - filters: + filters: | addedOrModified: - added|modified: '**' diff --git a/flows/aws/datalake/this_is_a_test3.py b/flows/aws/datalake/this_is_a_test4.py similarity index 100% rename from flows/aws/datalake/this_is_a_test3.py rename to flows/aws/datalake/this_is_a_test4.py From 12b9748951755c5487263dbe510bcc5aab2e4d61 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:11:08 -0600 Subject: [PATCH 05/14] cicd tests5 --- .github/workflows/aws_dev.yaml | 5 +++++ .../aws/datalake/{this_is_a_test4.py => this_is_a_test5.py} | 0 2 files changed, 5 insertions(+) rename flows/aws/datalake/{this_is_a_test4.py => this_is_a_test5.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index 5207317..f7c50ec 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -15,6 +15,11 @@ jobs: runs-on: ubuntu-latest steps: + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/flows/aws/datalake/this_is_a_test4.py b/flows/aws/datalake/this_is_a_test5.py similarity index 100% rename from flows/aws/datalake/this_is_a_test4.py rename to flows/aws/datalake/this_is_a_test5.py From 2d13dfdab35479c4b52dde6c207c0b47ef2def9d Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:11:49 -0600 Subject: [PATCH 06/14] cicd tests5 --- .github/workflows/aws_dev.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index f7c50ec..d910827 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -14,8 +14,6 @@ jobs: name: Deploy AWS flows runs-on: ubuntu-latest - steps: - steps: - name: Checkout uses: actions/checkout@v4 From a0b49a8c9cb76f08d32f479d2b1c4c2dbe89c74f Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:16:59 -0600 Subject: [PATCH 07/14] cicd tests6 --- flows/aws/datalake/{this_is_a_test5.py => this_is_a_test6.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename flows/aws/datalake/{this_is_a_test5.py => this_is_a_test6.py} (100%) diff --git a/flows/aws/datalake/this_is_a_test5.py b/flows/aws/datalake/this_is_a_test6.py similarity index 100% rename from flows/aws/datalake/this_is_a_test5.py rename to flows/aws/datalake/this_is_a_test6.py From 56c402af31aab47ea0e2540ac4dd312d5c563cd7 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:20:53 -0600 Subject: [PATCH 08/14] cicd tests7 --- .github/workflows/aws_dev.yaml | 2 +- flows/aws/datalake/{this_is_a_test6.py => this_is_a_test7.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename flows/aws/datalake/{this_is_a_test6.py => this_is_a_test7.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index d910827..a3c92b8 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -35,5 +35,5 @@ jobs: # Multiple change types can be specified using `|` as the delimiter. filters: | addedOrModified: - - added|modified: '**' + - added|modified: 'flows/aws/**' diff --git a/flows/aws/datalake/this_is_a_test6.py b/flows/aws/datalake/this_is_a_test7.py similarity index 100% rename from flows/aws/datalake/this_is_a_test6.py rename to flows/aws/datalake/this_is_a_test7.py From 34464f298441bb0ed0e006b5e8341c6de3d0069f Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:26:14 -0600 Subject: [PATCH 09/14] cicd tests8 --- .github/workflows/aws_dev.yaml | 4 ++++ flows/aws/datalake/{this_is_a_test7.py => this_is_a_test8.py} | 0 2 files changed, 4 insertions(+) rename flows/aws/datalake/{this_is_a_test7.py => this_is_a_test8.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index a3c92b8..529dd9b 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -36,4 +36,8 @@ jobs: filters: | addedOrModified: - added|modified: 'flows/aws/**' + + - name: Get Working Directory + id: getwd + run: echo {{ steps.filter.outputs.addedOrModified }} diff --git a/flows/aws/datalake/this_is_a_test7.py b/flows/aws/datalake/this_is_a_test8.py similarity index 100% rename from flows/aws/datalake/this_is_a_test7.py rename to flows/aws/datalake/this_is_a_test8.py From 07ac617a79988ef50c4494d48cf12afe3f74274a Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 10:28:05 -0600 Subject: [PATCH 10/14] cicd tests9 --- .github/workflows/aws_dev.yaml | 2 +- flows/aws/datalake/{this_is_a_test8.py => this_is_a_test9.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename flows/aws/datalake/{this_is_a_test8.py => this_is_a_test9.py} (100%) diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml index 529dd9b..fe1ee56 100644 --- a/.github/workflows/aws_dev.yaml +++ b/.github/workflows/aws_dev.yaml @@ -39,5 +39,5 @@ jobs: - name: Get Working Directory id: getwd - run: echo {{ steps.filter.outputs.addedOrModified }} + run: echo ${{ steps.filter.outputs.addedOrModified }} diff --git a/flows/aws/datalake/this_is_a_test8.py b/flows/aws/datalake/this_is_a_test9.py similarity index 100% rename from flows/aws/datalake/this_is_a_test8.py rename to flows/aws/datalake/this_is_a_test9.py From f0aeb138f2a7d1cbf12e9be77a6e12f7e57af60c Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 12:31:02 -0600 Subject: [PATCH 11/14] cicd dev branch --- .github/workflows/aws_datalake.yaml | 6 +++- .github/workflows/aws_dev.yaml | 43 --------------------------- flows/aws/datalake/this_is_a_test9.py | 0 3 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/aws_dev.yaml delete mode 100644 flows/aws/datalake/this_is_a_test9.py diff --git a/.github/workflows/aws_datalake.yaml b/.github/workflows/aws_datalake.yaml index 0108bf7..00d92cc 100644 --- a/.github/workflows/aws_datalake.yaml +++ b/.github/workflows/aws_datalake.yaml @@ -7,6 +7,7 @@ on: push: branches: - main + - dev paths: - "flows/aws/datalake/**" workflow_dispatch: @@ -16,6 +17,9 @@ jobs: name: Deploy AWS datalake flows runs-on: ubuntu-latest + env: + workspace: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }} + steps: - name: Checkout uses: actions/checkout@v4 @@ -45,6 +49,6 @@ jobs: run: | cd flows/aws/datalake pip install -r requirements-ci.txt - prefect cloud workspace set -w sales-engineering/se-datalake + prefect cloud workspace set -w sales-engineering/$workspace python deploy.py diff --git a/.github/workflows/aws_dev.yaml b/.github/workflows/aws_dev.yaml deleted file mode 100644 index fe1ee56..0000000 --- a/.github/workflows/aws_dev.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build image and deploy Prefect flow - Project 1 - -env: - PROJECT_NAME: flows/aws - -on: - push: - paths: - - "flows/aws/**" - workflow_dispatch: - -jobs: - deploy: - name: Deploy AWS flows - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: "pip" - - - name: Filter Paths - uses: dorny/paths-filter@v3 - id: filter - with: - # Changed file can be 'added', 'modified', or 'deleted'. - # By default, the type of change is not considered. - # Optionally, it's possible to specify it using nested - # dictionary, where the type of change composes the key. - # Multiple change types can be specified using `|` as the delimiter. - filters: | - addedOrModified: - - added|modified: 'flows/aws/**' - - - name: Get Working Directory - id: getwd - run: echo ${{ steps.filter.outputs.addedOrModified }} - diff --git a/flows/aws/datalake/this_is_a_test9.py b/flows/aws/datalake/this_is_a_test9.py deleted file mode 100644 index e69de29..0000000 From 1b1991db7c2fb0fd10d18f710f4a54da221d2171 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 12:36:33 -0600 Subject: [PATCH 12/14] cicd dev branch2 --- .github/workflows/aws_datalake.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/aws_datalake.yaml b/.github/workflows/aws_datalake.yaml index 00d92cc..37eb178 100644 --- a/.github/workflows/aws_datalake.yaml +++ b/.github/workflows/aws_datalake.yaml @@ -43,6 +43,7 @@ jobs: - name: Prefect Deploy env: + BRANCH: ${{ github.ref_name }} GITHUB_SHA: ${{ steps.get-commit-hash.outputs.COMMIT_HASH }} PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} ECR_REPO: ${{ secrets.ECR_REPO }} From 03455b18dafc7c0aeca22676137bef22844a28e7 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Thu, 18 Apr 2024 12:43:10 -0600 Subject: [PATCH 13/14] formatting consistency --- .github/workflows/aws_datalake.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aws_datalake.yaml b/.github/workflows/aws_datalake.yaml index 37eb178..a746e4a 100644 --- a/.github/workflows/aws_datalake.yaml +++ b/.github/workflows/aws_datalake.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest env: - workspace: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }} + WORKSPACE: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }} steps: - name: Checkout @@ -50,6 +50,6 @@ jobs: run: | cd flows/aws/datalake pip install -r requirements-ci.txt - prefect cloud workspace set -w sales-engineering/$workspace + prefect cloud workspace set -w sales-engineering/$WORKSPACE python deploy.py From 0697699fb188e52dd85e9c3ef534affcec01d3f1 Mon Sep 17 00:00:00 2001 From: masonmenges Date: Fri, 19 Apr 2024 12:18:54 -0600 Subject: [PATCH 14/14] setup dev vs demo workpools in CI --- .github/workflows/aws_datalake.yaml | 2 ++ flows/aws/datalake/deploy.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aws_datalake.yaml b/.github/workflows/aws_datalake.yaml index a746e4a..a9fbc5f 100644 --- a/.github/workflows/aws_datalake.yaml +++ b/.github/workflows/aws_datalake.yaml @@ -47,6 +47,8 @@ jobs: GITHUB_SHA: ${{ steps.get-commit-hash.outputs.COMMIT_HASH }} PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} ECR_REPO: ${{ secrets.ECR_REPO }} + WORKSPACE: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }} + WORK_POOL_NAME: ${{ github.ref == 'refs/heads/main' && 'Demo-ECS' || 'Dev-ECS' }} run: | cd flows/aws/datalake pip install -r requirements-ci.txt diff --git a/flows/aws/datalake/deploy.py b/flows/aws/datalake/deploy.py index 5b03784..39d6642 100644 --- a/flows/aws/datalake/deploy.py +++ b/flows/aws/datalake/deploy.py @@ -30,6 +30,7 @@ ecr_repo = os.getenv("ECR_REPO") image_tag = os.getenv("GITHUB_SHA") +work_pool_name = os.getenv("WORK_POOL_NAME") deploy( @@ -40,5 +41,5 @@ tag=image_tag, dockerfile="Dockerfile", ), - work_pool_name="Demo-ECS", + work_pool_name=work_pool_name, )