load-metrics #797
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: load-metrics | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "14 7 * * 1" # Every Monday at 7:14 AM UTC | |
| env: | |
| METRICS_PROD_ENV: "prod" | |
| jobs: | |
| load-metrics: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| # Authentication via credentials json | |
| - id: "auth" | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" | |
| create_credentials_file: true | |
| - name: Install Conda environment using mamba | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| environment-file: environment.yml | |
| cache-environment: true | |
| condarc: | | |
| channels: | |
| - conda-forge | |
| channel_priority: strict | |
| - name: Get GitHub Action runner's IP Address | |
| id: ip | |
| run: ipv4=$(curl --silent --url https://api.ipify.org); echo "ipv4=$ipv4" >> $GITHUB_OUTPUT | |
| - name: Echo IP for github runner | |
| run: | | |
| echo ${{ steps.ip.outputs.ipv4 }} | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| version: ">= 363.0.0" | |
| - name: Whitelist Github Action and Superset IPs | |
| run: | | |
| gcloud sql instances patch ${{ secrets.GCSQL_INSTANCE_NAME }} --authorized-networks=${{ steps.ip.outputs.ipv4 }} | |
| - name: Run ETL on the latest full week of data | |
| id: load-data | |
| env: | |
| IPINFO_TOKEN: ${{ secrets.IPINFO_TOKEN }} | |
| POSTGRES_IP: ${{ secrets.POSTGRES_IP }} | |
| POSTGRES_USER: ${{ secrets.POSTGRES_USER }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| POSTGRES_DB: ${{ secrets.POSTGRES_DB }} | |
| POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }} | |
| shell: bash -l {0} | |
| run: | | |
| python run_data_update.py | |
| - name: Remove Github Action runner's IP | |
| run: | | |
| gcloud sql instances patch ${{ secrets.GCSQL_INSTANCE_NAME }} --clear-authorized-networks | |
| - name: Post to pudl-deployments channel | |
| if: always() | |
| id: slack | |
| uses: slackapi/slack-github-action@v1 | |
| with: | |
| channel-id: "C03FHB9N0PQ" | |
| slack-message: "Weekly usage metrics processing ran with status: ${{ job.status }}." | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} |