Setup CI/CD process around the se-demos-dev workspace

.github/workflows/aws_datalake.yaml

@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - main
+      - dev
     paths:
       - "flows/aws/datalake/**"
   workflow_dispatch:
@@ -16,6 +17,9 @@ jobs:
       name: Deploy AWS datalake flows
       runs-on: ubuntu-latest
 
+      env:
+        WORKSPACE: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }}
+
       steps:
         - name: Checkout
           uses: actions/checkout@v4
@@ -39,12 +43,15 @@ jobs:
 
         - name: Prefect Deploy
           env:
+            BRANCH: ${{ github.ref_name }}
             GITHUB_SHA: ${{ steps.get-commit-hash.outputs.COMMIT_HASH }}
             PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
             ECR_REPO: ${{ secrets.ECR_REPO }}
+            WORKSPACE: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }}
+            WORK_POOL_NAME: ${{ github.ref == 'refs/heads/main' && 'Demo-ECS' || 'Dev-ECS' }}
           run: |
             cd flows/aws/datalake
             pip install -r requirements-ci.txt
-            prefect cloud workspace set -w sales-engineering/se-datalake
+            prefect cloud workspace set -w sales-engineering/$WORKSPACE
             python deploy.py
 

flows/aws/datalake/deploy.py

@@ -30,6 +30,7 @@
 
 ecr_repo = os.getenv("ECR_REPO")
 image_tag = os.getenv("GITHUB_SHA")
+work_pool_name = os.getenv("WORK_POOL_NAME")
 
 
 deploy(
@@ -40,5 +41,5 @@
         tag=image_tag,
         dockerfile="Dockerfile",
     ),
-    work_pool_name="Demo-ECS",
+    work_pool_name=work_pool_name,
 )