From ff0d5fa093084384bcfffc08cf63ce6bd6dc7469 Mon Sep 17 00:00:00 2001
From: Alec Scott <scott112@llnl.gov>
Date: Fri, 10 Nov 2023 10:53:34 -0800
Subject: [PATCH] Add simple saxpy CI test (#53)

* Add saxpy ci run

* Write Ramble and Spack to persistent env

* Explicitly call out ramble and spack environment modifications

* Fix PATH format in env

* Fix argument positioning in ramble execuiton

* Add gfortran to ubuntu image as system dependency

* sudo make me a sandwich

* use long version of parameters

* Add debug step to check what compilers are on host

* Remove gcc-12 and gcc-13 from the runner

* cd to workspace dir for each stage

* Enforce running on a single node

* Add limit to number of ranks in addition to nodes

* Limit to a single process per node

* add oversubscribe

* Add binary cache to speed up CI

* fix oci-username parameter

* activate environment as part of buildcache push

* Provide specs to binary cache push cmd

* Uncomment conditional for uploading to CI buildcache

---------

Co-authored-by: Stephanie Brink <brink2@llnl.gov>
---
 .github/workflows/ci.yml   | 12 +++++++++
 .github/workflows/run.yml  | 52 ++++++++++++++++++++++++++++++++++++++
 configs/x86/variables.yaml |  2 +-
 3 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/run.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a9b867ad0..31d5f51fc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ jobs:
     outputs:
       docs: ${{ steps.filter.outputs.docs }}
       style: ${{ steps.filter.outputs.style }}
+      run: ${{ steps.filter.outputs.run }}
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
@@ -41,6 +42,12 @@ jobs:
               - 'docs/conf.py'
               - 'experiments/**'
               - 'repo/**'
+            run:
+              - '.github/**'
+              - 'bin/**'
+              - 'configs/**'
+              - 'experiments/**'
+              - 'repo/**'
 
   docs:
     if: ${{ needs.changes.outputs.docs == 'true' }}
@@ -51,3 +58,8 @@ jobs:
     if: ${{ needs.changes.outputs.style == 'true' }}
     needs: changes
     uses: ./.github/workflows/style.yml
+
+  run:
+    if: ${{ needs.changes.outputs.run == 'true' }}
+    needs: changes
+    uses: ./.github/workflows/run.yml
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
new file mode 100644
index 000000000..57082d5a6
--- /dev/null
+++ b/.github/workflows/run.yml
@@ -0,0 +1,52 @@
+name: Run Benchpark and Simple Benchmark Suite
+on:
+  # This Workflow can be triggered manually
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  saxpy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove gcc > 12 # to prevent Spack from picking a gcc without gfortan
+        run: |
+          sudo apt-get remove -y gcc-13
+
+      - name: Checkout Benchpark
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+
+      - name: Build Saxpy Workspace
+        run: |
+          ./bin/benchpark setup saxpy/openmp x86 workspace/
+
+      - name: Setup Ramble & Spack
+        run: |
+          . workspace/saxpy/openmp/x86/spack/share/spack/setup-env.sh
+          . workspace/saxpy/openmp/x86/ramble/share/ramble/setup-env.sh
+
+          spack mirror add ci-buildcache oci://ghcr.io/llnl/benchpark-binary-cache
+
+          env | grep SPACK >> "$GITHUB_ENV"
+          env | grep SPACK >> "$GITHUB_ENV"
+          echo "PATH=$PATH" >> "$GITHUB_ENV"
+
+      - name: Setup Saxpy Workspace
+        working-directory: ./workspace/saxpy/openmp/x86/workspace/
+        run: |
+          ramble --workspace-dir . --disable-progress-bar --disable-logger workspace setup
+
+      - name: Run Saxpy Experiments
+        working-directory: ./workspace/saxpy/openmp/x86/workspace/
+        run: |
+          ramble -c variables:n_nodes:1 -c variables:n_ranks:1 --workspace-dir . --disable-progress-bar --disable-logger on
+
+      - name: Analyze Saxpy Results
+        working-directory: ./workspace/saxpy/openmp/x86/workspace/
+        run: |
+          ramble --workspace-dir . --disable-progress-bar --disable-logger workspace analyze
+
+      - name: Upload Binaries to CI Cache
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" ci-buildcache
+          spack buildcache push -j $(($(nproc) + 1)) --base-image ubuntu:22.04 --unsigned --update-index ci-buildcache $(spack find --format '/{hash}')
diff --git a/configs/x86/variables.yaml b/configs/x86/variables.yaml
index d3aafc893..097ac05c4 100644
--- a/configs/x86/variables.yaml
+++ b/configs/x86/variables.yaml
@@ -1,6 +1,6 @@
 variables:
   batch_time: ''
-  mpi_command: 'mpirun -n {n_nodes} -c {n_ranks}'
+  mpi_command: 'mpirun -n {n_nodes} -c {n_ranks} --oversubscribe'
   batch_submit: '{execute_experiment}'
   batch_nodes: ''
   batch_ranks: ''