Skip to content
E2ESHARK Test Suite

E2ESHARK Test Suite #370

Sign in to view logs

E2ESHARK Test Suite

E2ESHARK Test Suite #370

Workflow file for this run

.github/workflows/test_e2eshark.yml at efb341f
# Copyright 2024 Advanced Micro Devices
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
name: E2ESHARK Test Suite
on:
workflow_dispatch:
# schedule:
# # Runs at 12:00 PM UTC, which is 5:00 AM PST
# - cron: '0 12 * * *'
jobs:
e2eshark:
timeout-minutes: 480
name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}"
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include:
- name: mi300_gpu1_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: nlp-shard1
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu2_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: nlp-shard2
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu3_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: nlp-shard3
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu4_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: shark-test-suite
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu5_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-hf-cnn-fp32-shard1
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu6_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-hf-cnn-fp32-shard2
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu7_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-hf-cnn-fp32-shard3
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu8_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-int8-p0p1-shard1
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu9_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-int8-p0p1-shard2
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu10_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-int8-p0p1-shard3
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu11_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: vai-vision-int8
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu12_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
device: hip
target-chip: gfx942
test-file: migraphx
cache-dir: /home/sai/shark-test-suite-models-cache
- name: mi300_gpu13_test
runs-on: nodai-amdgpu-mi300-x86-64
backend: hip
device: hip
target-chip: gfx942
test-file: onnxrt-iree-ep
cache-dir: /home/sai/shark-test-suite-models-cache
- name: cpu_shard1_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: nlp-shard1
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard2_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: nlp-shard2
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard3_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: nlp-shard3
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard4_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: shark-test-suite
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard5_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-hf-cnn-fp32-shard1
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard6_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-hf-cnn-fp32-shard2
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard7_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-hf-cnn-fp32-shard3
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard8_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-int8-p0p1-shard1
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard9_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-int8-p0p1-shard2
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard10_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-int8-p0p1-shard3
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard11_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: vai-vision-int8
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard12_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: migraphx
cache-dir: /groups/aig_sharks/test-suite-ci-cache
- name: cpu_shard13_test
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
device: local-task
target-chip: x86_64-linux-gnu
test-file: onnxrt-iree-ep
cache-dir: /groups/aig_sharks/test-suite-ci-cache
env:
E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv
EP_VENV_DIR: ${{ github.workspace }}/ep_venv
ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv
HF_TOKEN: ${{ secrets.HF_TOKEN }}
AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }}
CACHE_DIR: ${{ matrix.cache-dir }}
steps:
- name: Checkout Test Suite
uses: actions/checkout@v2
with:
repository: nod-ai/SHARK-TestSuite
ref: alt-merge-reports
path: test-suite
- name: "Setup alt e2eshark python venv"
run: |
rm -rf ${ALT_E2E_VENV_DIR}
python3.11 -m venv ${ALT_E2E_VENV_DIR}
source ${ALT_E2E_VENV_DIR}/bin/activate
pip install --upgrade pip
pip install -r ./alt_e2eshark/base_requirements.txt
pip install -r ./alt_e2eshark/iree_requirements.txt
pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt
pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade
working-directory: ./test-suite
- name: "Setup IREE EP python venv"
run: |
python3.11 -m venv ${EP_VENV_DIR}
source ${EP_VENV_DIR}/bin/activate
pip install --upgrade pip
pip install -r ./alt_e2eshark/base_requirements.txt
pip install -r ./alt_e2eshark/iree_requirements.txt
pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt
pip uninstall -y onnxruntime
wget https://sharkpublic.blob.core.windows.net/sharkpublic/onnxruntime/pip_whl/onnxruntime-1.20.0-cp311-cp311-linux_x86_64.whl
pip install ./onnxruntime-1.20.0-cp311-cp311-linux_x86_64.whl
working-directory: ./test-suite
- name: Run Onnx Bench Mode
if: contains(matrix.test-file, 'migraphx')
run: |
source ${ALT_E2E_VENV_DIR}/bin/activate
pip freeze
cd alt_e2eshark
free -mh
python3.11 ./run.py \
-r ./test-onnx \
--report \
--torchtolinalg \
--testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \
-b ${{ matrix.backend }} \
-d ${{ matrix.device }} \
--report-file reports/${{ matrix.test-file }}.md \
--mode=cl-onnx-iree \
--cleanup=3 \
--benchmark \
-v
working-directory: ./test-suite
- name: Run Onnx Default Mode
if: ${{ ! contains(matrix.test-file, 'migraphx') && ! contains(matrix.test-file, 'onnxrt-iree-ep') }}
run: |
source ${ALT_E2E_VENV_DIR}/bin/activate
pip freeze
cd alt_e2eshark
free -mh
python3.11 ./run.py \
-r ./test-onnx \
--report \
--torchtolinalg \
--testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \
-b ${{ matrix.backend }} \
-d ${{ matrix.device }} \
--report-file reports/${{ matrix.test-file }}.md \
--mode=cl-onnx-iree \
--cleanup=3 \
-v
working-directory: ./test-suite
- name: Run OnnxRT IREE EP
if: contains(matrix.test-file, 'onnxrt-iree-ep')
run: |
source ${EP_VENV_DIR}/bin/activate
export COMPILER_PATH=$(python -c "import iree.compiler as _; print(_.__path__[0])")
export LD_LIBRARY_PATH="$COMPILER_PATH/_mlir_libs/"
echo "COMPILER_PATH=$COMPILER_PATH" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
pip freeze
cd alt_e2eshark
free -mh
ulimit -n 1000000
python ./run.py -m ort-ep \
-r ./test-onnx \
--report \
--testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \
-b ${{ matrix.backend }} \
-d ${{ matrix.device }} \
--report-file reports/${{ matrix.test-file }}.md \
--cleanup=3 \
-v
working-directory: ./test-suite
- uses: actions/upload-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md
path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md
- uses: actions/upload-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json
path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json
push_artifacts:
needs: [e2eshark]
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
max-parallel: 1
matrix:
include:
- name: merge_rocm_reports
runs-on: nodai-amdgpu-mi300-x86-64
backend: rocm
regression-blob: rocm
- name: merge_cpu_reports
runs-on: nodai-amdgpu-mi250-x86-64
backend: llvm-cpu
regression-blob: cpu
env:
AZ_PUBLIC_KEY: ${{ secrets.SHARKPUBLIC_AZ_PUBLIC_KEY }}
steps:
- name: Checkout Test Suite
uses: actions/checkout@v2
with:
repository: nod-ai/SHARK-TestSuite
ref: alt-merge-reports
path: test-suite
- name: Checkout repo
uses: actions/checkout@v2
with:
repository: nod-ai/e2eshark-reports
ref: main
token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }}
path: e2eshark-reports
- name: "Setup alt test suite venv"
run: |
python3.11 -m venv report_venv_alt
source report_venv_alt/bin/activate
pip install --upgrade pip
pip install -r ./test-suite/alt_e2eshark/base_requirements.txt
pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt
pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt
pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_md
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_md
- uses: actions/download-artifact@master
with:
name: ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_json
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_json
- name: Merge Reports
run: |
source report_venv_alt/bin/activate
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \
--output ./e2eshark-reports/vai-hf-cnn-fp32.json \
--report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \
--output ./e2eshark-reports/vai-int8-p0p1.json \
--report --report-file ./e2eshark-reports/vai-int8-p0p1.md
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \
--output ./e2eshark-reports/nlp.json \
--report --report-file ./e2eshark-reports/nlp.md
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
--sources ./e2eshark-reports/vai-int8-p0p1.json \
./e2eshark-reports/vai-hf-cnn-fp32.json \
./e2eshark-reports/nlp.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \
./e2eshark-reports/ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_json/onnxrt-iree-ep.json \
--output ./e2eshark-reports/combined_reports.json \
--report --report-file ./e2eshark-reports/combined_reports.md
- name: Push status artifacts
run: |
git config user.name "GitHub Actions Bot"
git config user.email "<>"
git pull
date=$(date '+%Y-%m-%d')
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/onnxrt-iree-ep
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports
cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md
cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md
cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md
cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md
cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md
cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md
cp ci_reports_${{ matrix.backend }}_onnxrt-iree-ep_onnx_md/onnxrt-iree-ep.md ${date}/ci_reports_onnx/${{ matrix.backend }}/onnxrt-iree-ep/summary.md
cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md
git add $date
git commit -m "add CI status reports for e2eshark for ${{ matrix.backend }}"
git push origin main
working-directory: ./e2eshark-reports
- name: Regression Reports
run: |
source report_venv_alt/bin/activate
cd test-suite
mkdir latest
mkdir baseline
wget https://sharkpublic.blob.core.windows.net/sharkpublic/latest-test-suite/combined_reports_${{ matrix.regression-blob }}.json -O latest/combined_reports_${{ matrix.backend }}.json
wget https://sharkpublic.blob.core.windows.net/sharkpublic/baseline-test-suite/combined_reports_${{ matrix.regression-blob }}.json -O baseline/combined_reports_${{ matrix.backend }}.json
cd ..
python ./test-suite/alt_e2eshark/utils/check_regressions.py \
--new ./e2eshark-reports/combined_reports.json \
--old ./test-suite/latest/combined_reports_${{ matrix.backend }}.json \
--report-file ./e2eshark-reports/yesterday_comparison.md \
--perf_tol_regression=0.05 \
--perf_tol_progression=0.05
python ./test-suite/alt_e2eshark/utils/check_regressions.py \
--new ./e2eshark-reports/combined_reports.json \
--old ./test-suite/baseline/combined_reports_${{ matrix.backend }}.json \
--report-file ./e2eshark-reports/baseline_comparison.md \
--perf_tol_regression=0.1 \
--perf_tol_progression=0.1
az storage blob upload --account-name sharkpublic --container-name sharkpublic \
--name latest-test-suite/combined_reports_${{ matrix.backend }}.json \
--file ./e2eshark-reports/combined_reports.json \
--account-key ${AZ_PUBLIC_KEY} --overwrite
- name: Push regression artifacts
run: |
git config user.name "GitHub Actions Bot"
git config user.email "<>"
git pull
date=$(date '+%Y-%m-%d')
cp yesterday_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/yesterday_comparison.md
cp baseline_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/baseline_comparison.md
git add $date
git commit -m "add CI regression reports for e2eshark for ${{ matrix.backend }}"
git push origin main
working-directory: ./e2eshark-reports