diff --git a/.github/workflows/ornl_crusher_mirror.yaml b/.github/workflows/ornl_crusher_mirror.yaml new file mode 100644 index 00000000..cf9ec4e7 --- /dev/null +++ b/.github/workflows/ornl_crusher_mirror.yaml @@ -0,0 +1,38 @@ +name: ORNL Crusher Mirror + +# triggers a github action everytime there is a push or mr +on: + pull_request: + push: + branches: + - develop + - main + +jobs: + # To test on HPC resources we must first mirror the repo and then trigger a pipeline + push: + # Latest Ubuntu as of Feb 2023 is 20.04 + runs-on: ubuntu-22.04 + steps: + # Action checks-out your repository under $GITHUB_WORKSPACE, so your workflow can access it. + - uses: actions/checkout@v1 + + # Necessary to get branch name independent of PR/push + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + + # Action for mirroring your commits to a different remote repository + - uses: yesolutions/mirror-action@master + # Give actions access to some secrets + with: + GIT_PUSH_ARGS: '--tags --force --push-option=ci.skip' + REMOTE: 'https://code.olcf.ornl.gov/ci/csc359/dev/exago' + GIT_USERNAME: ${{ secrets.GIT_USER }} + GIT_PASSWORD: ${{ secrets.ORNL_CRUSHER_GIT_PASSWORD }} + + # Can only run on Crusher with web or schedule pipeline, so no way to trigger + # # Trigger CI pipeline since it was skipped in the above push + # - name: Trigger Pipeline + # run: curl -X POST -F token=${{ secrets.ORNL_CRUSHER_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline diff --git a/.gitlab/ornl/ascent.gitlab-ci.yml b/.gitlab/ornl/ascent.gitlab-ci.yml index 86a99952..ee5c8e05 100644 --- a/.gitlab/ornl/ascent.gitlab-ci.yml +++ b/.gitlab/ornl/ascent.gitlab-ci.yml @@ -214,6 +214,7 @@ pending: stage: .pre success: + needs: ['Ascent Test'] extends: .report-status stage: .post diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 65b18570..20801367 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -1,18 +1,89 @@ -# Crusher Variables -.crusher_rules: +workflow: rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PINELINE_SOURCE == "schedule" - - when: never + +# https://stackoverflow.com/questions/67009007 +# We have to use YAML anchors for rules here +# Rules are also evaluated sequentially, so you need to be careful about +# the order in which these rules are included in child jobs - list "never" rules before other rules +.rules-template: + rules: + # -- + # Rule to fall back on... + - &default + when: always + # -- + # Only run when the commit **DOES NOT** contains "[newell-rebuild]" + - &rule_no_newell_rebuild + if: '$CI_COMMIT_TITLE =~ /\[newell-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[deception-rebuild]" + - &rule_no_deception_rebuild + if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[incline-rebuild]" + - &rule_no_incline_rebuild + if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[ascent-rebuild]" + - &rule_no_ascent_rebuild + if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[newell-test]" + - &rule_no_newell_test + if: '$CI_COMMIT_TITLE =~ /\[newell-test\]/' + when: never + # Only run when the commit **DOES** contains "[newell-test]" + - &rule_yes_newell_test + if: '$CI_COMMIT_TITLE =~ /\[newell-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[deception-test]" + - &rule_no_deception_test + if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/' + when: never + # Only run when the commit **DOES** contains "[deception-test]" + - &rule_yes_deception_test + if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[incline-test]" + - &rule_no_incline_test + if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/' + when: never + # Only run when the commit **DOES** contains "[incline-test]" + - &rule_yes_incline_test + if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[ascent-test]" + - &rule_no_ascent_test + if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/' + when: never + # Only run when the commit **DOES** contains "[ascent-test]" + - &rule_yes_ascent_test + if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/' + when: always .crusher_variables: # Only for slurm tagged jobs... variables: - SCHEDULER_PARAMETERS: "-N 1 -A CSC359 --time=60" - WORKDIR: /gpfs/alpine/csc359/proj-shared/ci/${CI_PIPELINE_ID} + SCHEDULER_PARAMETERS: "-N 1 -A CSC359_crusher --time=60" + WORKDIR: /lustre/orion/csc359/proj-shared/ci/${CI_PIPELINE_ID} + +.crusher_rules: + rules: + - *rule_no_deception_test + - *rule_no_deception_rebuild + - *rule_no_newell_test + - *rule_no_newell_rebuild + - *rule_no_incline_test + - *rule_no_incline_rebuild + - *rule_no_ascent_test + - *rule_no_ascent_rebuild + - *default # Crusher Jobs Crusher Build: + needs: [] stage: build tags: [crusher, shell] script: @@ -38,7 +109,7 @@ Crusher Test: script: - cd "$WORKDIR" - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install - - export CTESTARGS="--output-on-failure -E Python" + - export CTESTARGS="--output-on-failure" - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip - res=$? - exit $res @@ -49,22 +120,56 @@ Crusher Test: - .crusher_rules - .crusher_variables -Crusher Python Test: - stage: test - dependencies: - - "Crusher Build" +# -- Reporting Ascent Status to PNNL GitHub +.report-job: + when: always variables: - # Don't clone for test jobs + MY_CLUSTER: Crusher GIT_STRATEGY: none - allow_failure: true - tags: [crusher, slurm] + STATUS_PROJECT: pnnl/ExaGO + STATUS_NAME: amd64/clang@14.0/rocm@5.2.0/MI100@gfx908 + tags: [crusher, shell] + extends: .crusher_variables + environment: + name: reporting-gitlab + resource_group: status + retry: 1 + +.report-status: + extends: .report-job script: - - cd "$WORKDIR" - - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install - - export CTESTARGS="--output-on-failure -R Python" - - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip - - res=$? - - exit $res - extends: - - .crusher_rules - - .crusher_variables + - | + curl -L \ + -X POST \ + -H @${GITHUB_CURL_HEADER}\ + https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} \ + -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_NAME}\",\"context\":\"${MY_CLUSTER}\"}" + environment: + name: reporting-gitlab + +.report-pending: + extends: .report-job + script: + - | + set -x + curl -L \ + -X POST \ + -H @${GITHUB_CURL_HEADER}\ + https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} \ + -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${MY_CLUSTER}\"}" + +pending: + extends: .report-pending + stage: .pre + +success: + needs: ['Crusher Test'] + extends: .report-status + stage: .post + +failure: + stage: .post + extends: .report-status + rules: + - when: on_failure +# ---