load-metrics #425
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: load-metrics | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "14 0 * * *" # Every day at 12:14 AM UTC | |
| env: | |
| PRESET_IP1: 44.193.153.196 | |
| PRESET_IP2: 52.70.123.52 | |
| PRESET_IP3: 54.83.88.93 | |
| jobs: | |
| load-metrics: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| # Authentication via credentials json | |
| - id: "auth" | |
| uses: "google-github-actions/auth@v1" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" | |
| create_credentials_file: true | |
| - name: Set up conda environment for testing | |
| uses: conda-incubator/[email protected] | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| use-mamba: true | |
| mamba-version: "*" | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| python-version: ${{ matrix.python-version }} | |
| activate-environment: pudl-usage-metrics | |
| environment-file: environment.yml | |
| - shell: bash -l {0} | |
| run: | | |
| mamba info | |
| mamba list | |
| conda config --show-sources | |
| conda config --show | |
| printenv | sort | |
| - name: Get GitHub Action runner's IP Address | |
| id: ip | |
| uses: haythem/[email protected] | |
| - name: Whitelist Github Action and Superset IPs | |
| run: | | |
| gcloud sql instances patch ${{ secrets.GCSQL_INSTANCE_NAME }} --authorized-networks=${{ steps.ip.outputs.ipv4 }},${{ env.PRESET_IP1 }},${{ env.PRESET_IP2 }},${{ env.PRESET_IP3 }} | |
| - name: Run ETL | |
| env: | |
| IPINFO_TOKEN: ${{ secrets.IPINFO_TOKEN }} | |
| POSTGRES_IP: ${{ secrets.POSTGRES_IP }} | |
| POSTGRES_USER: ${{ secrets.POSTGRES_USER }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| POSTGRES_DB: ${{ secrets.POSTGRES_DB }} | |
| POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }} | |
| run: | | |
| mamba run -n pudl-usage-metrics python run_data_update.py | |
| - name: Remove Github Action runner's IP | |
| run: | | |
| gcloud sql instances patch ${{ secrets.GCSQL_INSTANCE_NAME }} --authorized-networks=${{ env.PRESET_IP1 }},${{ env.PRESET_IP2 }},${{ env.PRESET_IP3 }} | |
| ci-notify: | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| needs: load-metrics | |
| steps: | |
| - name: Inform the Codemonkeys | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| fields: workflow,job,commit,repo,ref,author,took | |
| custom_payload: | | |
| { | |
| username: 'action-slack', | |
| icon_emoji: ':octocat:', | |
| attachments: [{ | |
| color: '${{ needs.ci-test.result }}' === 'success' ? 'good' : '${{ needs.ci-test.result }}' === 'failure' ? 'danger' : 'warning', | |
| text: `${process.env.AS_REPO}@${process.env.AS_REF}\n ${process.env.AS_WORKFLOW} (${process.env.AS_COMMIT})\n by ${process.env.AS_AUTHOR}\n Status: ${{ needs.ci-test.result }}`, | |
| }] | |
| } | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} # required | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required | |
| MATRIX_CONTEXT: ${{ toJson(matrix) }} # required |