From 5e0771ec1c96fdee9fe4b5514f2b4f55e013eebd Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 13 Nov 2024 14:44:11 -0600 Subject: [PATCH 1/4] Add single line Signed-off-by: Andre Kurait --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b7b035118..5e06bb353 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -230,3 +230,4 @@ jobs: echo "One or more job cancelled, failed, or skipped" && exit 1 - run: | echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY + From 9e3d6e3123be25ce341511ad6e58059ef3156ec3 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 13 Nov 2024 15:50:46 -0600 Subject: [PATCH 2/4] Move to tokenless codecov Signed-off-by: Andre Kurait --- .github/workflows/CI.yml | 211 ++++++++++++++------------ .github/workflows/codecov-publish.yml | 81 ++++++---- 2 files changed, 167 insertions(+), 125 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5e06bb353..010def3df 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -77,99 +77,99 @@ jobs: name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }} path: ${{ env.WORKING_DIR }}/coverage.xml -# gradle-tests: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-java@v4 -# with: -# java-version: ${{ env.java-version }} -# distribution: 'corretto' -# - uses: gradle/actions/setup-gradle@v4 -# with: -# gradle-version: ${{ env.gradle-version }} -# gradle-home-cache-cleanup: true -# - name: Run Gradle Build -# run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace -# env: -# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' -# -# - name: Run Tests with Coverage -# run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace -# env: -# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' -# -# - uses: actions/upload-artifact@v4 -# if: always() -# with: -# name: test-reports-gradle-tests -# path: | -# **/build/reports/tests/ -# **/reports/jacoco/mergedReport/ -# - name: Upload Coverage Reports -# uses: actions/upload-artifact@v4 -# with: -# if-no-files-found: error -# name: coverage-reports-gradle-tests -# path: /**/jacocoMergedReport.xml -# -# -# python-e2e-tests: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-java@v4 -# with: -# java-version: ${{ env.java-version }} -# distribution: 'corretto' -# - uses: gradle/actions/setup-gradle@v4 -# with: -# gradle-version: ${{ env.gradle-version }} -# - uses: actions/setup-python@v5 -# with: -# python-version: ${{ env.python-version }} -# - name: Generate Cache Key from Dockerfiles -# id: generate_cache_key -# run: | -# files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) -# file_contents=$(cat $files) -# key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') -# echo "key=${key}" >> "$GITHUB_OUTPUT" -# - name: Cache Docker Images -# uses: ScribeMD/docker-cache@0.5.0 -# with: -# key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} -# - name: Start Docker Solution -# run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace -# env: -# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' -# - name: Run E2E test script -# working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test -# run: | -# docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s -# - name: Collect Docker, OpenSearch Benchmark, and Shared Logs -# if: always() -# run: | -# mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output -# for container in $(docker ps -aq); do -# container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///') -# docker logs $container > logs/docker/${container_name}_logs.txt 2>&1 -# done -# docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs -# docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output -# - name: Upload Logs -# if: always() -# uses: actions/upload-artifact@v4 -# with: -# name: e2e-test-logs -# path: | -# logs/docker -# logs/opensearch_benchmark_logs -# logs/shared_logs_output -# - name: Clean up migrations docker images before caching -# run: | -# docker stop $(docker ps -q) && docker system prune --volumes -f -# docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {} + gradle-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ env.java-version }} + distribution: 'corretto' + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: ${{ env.gradle-version }} + gradle-home-cache-cleanup: true + - name: Run Gradle Build + run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace + env: + OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' + + - name: Run Tests with Coverage + run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace + env: + OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: test-reports-gradle-tests + path: | + **/build/reports/tests/ + **/reports/jacoco/mergedReport/ + - name: Upload Coverage Reports + uses: actions/upload-artifact@v4 + with: + if-no-files-found: error + name: coverage-reports-gradle-tests + path: /**/jacocoMergedReport.xml + + + python-e2e-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ env.java-version }} + distribution: 'corretto' + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: ${{ env.gradle-version }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} + - name: Generate Cache Key from Dockerfiles + id: generate_cache_key + run: | + files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) + file_contents=$(cat $files) + key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') + echo "key=${key}" >> "$GITHUB_OUTPUT" + - name: Cache Docker Images + uses: ScribeMD/docker-cache@0.5.0 + with: + key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} + - name: Start Docker Solution + run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace + env: + OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' + - name: Run E2E test script + working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test + run: | + docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s + - name: Collect Docker, OpenSearch Benchmark, and Shared Logs + if: always() + run: | + mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output + for container in $(docker ps -aq); do + container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///') + docker logs $container > logs/docker/${container_name}_logs.txt 2>&1 + done + docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs + docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output + - name: Upload Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-logs + path: | + logs/docker + logs/opensearch_benchmark_logs + logs/shared_logs_output + - name: Clean up migrations docker images before caching + run: | + docker stop $(docker ps -q) && docker system prune --volumes -f + docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {} node-tests: runs-on: ubuntu-latest @@ -213,13 +213,34 @@ jobs: env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - all-ci-checks-pass: + upload-codecov: needs: - console-python-tests # - gradle-tests + if: success() + runs-on: ubuntu-latest + steps: + - name: Download Coverage Artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-reports-* + path: ./coverage-reports + - name: Upload Coverage + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + files: ./coverage-reports/**/*.* + flags: unittests + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + + all-ci-checks-pass: + needs: + - console-python-tests + - gradle-tests - link-checker - node-tests -# - python-e2e-tests + - python-e2e-tests - python-lint - style-check if: always() diff --git a/.github/workflows/codecov-publish.yml b/.github/workflows/codecov-publish.yml index f439464fb..c4ea54401 100644 --- a/.github/workflows/codecov-publish.yml +++ b/.github/workflows/codecov-publish.yml @@ -1,30 +1,51 @@ -name: Codecov Publish - -on: - workflow_run: - workflows: [CI] - types: - - completed - -jobs: - publish-codecov: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Download Coverage Artifacts - uses: actions/download-artifact@v4 - with: - pattern: coverage-reports-* - path: ./coverage-reports - - - name: Upload Coverage - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: true - files: ./coverage-reports/**/*.* - flags: unittests - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true +#name: Codecov Publish +# +## Due to the behavior of workflow_run, changes to this file will +## only be reflected once they are pushed up to the `default` (main) branch +# +#on: +# workflow_run: +# workflows: [CI] +# types: +# - completed +# branches: +# - gha_codecov_separate_workflow +# - main +# +#jobs: +# publish-codecov: +# if: ${{ github.event.workflow_run.conclusion == 'success' }} +# runs-on: ubuntu-latest +# steps: +# - name: get PR +# env: +# GH_TOKEN: ${{ github.token }} +# run: | +# - echo "pull_request=$(gh api -H 'Accept: +# application/vnd.github+json' -H 'X-Github-Api-Version: 2022-11-28' \ +# - /repos/${{ github.repository}}/actions/runs/${{ github.event.workflow_run.id }} | jq .pull_requests[0].number)" >> $GITHUB_ENV +# - echo "Pull Request: $pull_request" +# +# - name: Checkout Repository +# uses: actions/checkout@v4 +# +# - name: Download Coverage Artifacts +# uses: actions/download-artifact@v4 +# with: +# pattern: coverage-reports-* +# path: ./coverage-reports +# +# - name: Upload Coverage +# uses: codecov/codecov-action@v4 +# with: +# fail_ci_if_error: true +# files: ./coverage-reports/**/*.* +# flags: unittests +# token: ${{ secrets.CODECOV_TOKEN }} +# verbose: true +# # Manual overrides for these parameters are needed because automatic detection +# # in codecov-action does not work for non-`pull_request` workflows. +# # In `main` branch push, these default to empty strings since we want to run +# # the analysis on HEAD. +# override_commit: ${{ steps.parse_previous_artifacts.outputs.override_commit || '' }} +# override_pr: ${{ steps.parse_previous_artifacts.outputs.override_pr || '' }} From c7ff7c48552b8f9dc5470377e9d0e4fb83da5fca Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 13 Nov 2024 15:58:29 -0600 Subject: [PATCH 3/4] Remove token Signed-off-by: Andre Kurait --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 010def3df..089822d86 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -231,7 +231,7 @@ jobs: fail_ci_if_error: true files: ./coverage-reports/**/*.* flags: unittests - token: ${{ secrets.CODECOV_TOKEN }} +# token: ${{ secrets.CODECOV_TOKEN }} verbose: true all-ci-checks-pass: From 35126427d6d04a822a8f7d6d9cb2b36bf84cb9f6 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 13 Nov 2024 16:27:27 -0600 Subject: [PATCH 4/4] Simple test Signed-off-by: Andre Kurait --- .github/workflows/CI.yml | 406 +++++++++++++------------- .github/workflows/codecov-publish.yml | 99 +++---- 2 files changed, 251 insertions(+), 254 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 089822d86..333f38af8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,36 +11,36 @@ env: node-version: '18.x' jobs: - style-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ env.java-version }} - distribution: "corretto" - - uses: gradle/actions/setup-gradle@v4 - with: - gradle-version: ${{ env.gradle-version }} - - name: Run Gradle Build - run: ./gradlew spotlessCheck - env: - OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' - - python-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.python-version }} - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install flake8 - - name: Analysing the code with flake8 - run: | - flake8 $(git ls-files '*.py') +# style-check: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ env.java-version }} +# distribution: "corretto" +# - uses: gradle/actions/setup-gradle@v4 +# with: +# gradle-version: ${{ env.gradle-version }} +# - name: Run Gradle Build +# run: ./gradlew spotlessCheck +# env: +# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' +# +# python-lint: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-python@v5 +# with: +# python-version: ${{ env.python-version }} +# - name: Install dependencies +# run: | +# python3 -m pip install --upgrade pip +# python3 -m pip install flake8 +# - name: Analysing the code with flake8 +# run: | +# flake8 $(git ls-files '*.py') console-python-tests: runs-on: ubuntu-latest @@ -77,178 +77,178 @@ jobs: name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }} path: ${{ env.WORKING_DIR }}/coverage.xml - gradle-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ env.java-version }} - distribution: 'corretto' - - uses: gradle/actions/setup-gradle@v4 - with: - gradle-version: ${{ env.gradle-version }} - gradle-home-cache-cleanup: true - - name: Run Gradle Build - run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace - env: - OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' - - - name: Run Tests with Coverage - run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace - env: - OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-reports-gradle-tests - path: | - **/build/reports/tests/ - **/reports/jacoco/mergedReport/ - - name: Upload Coverage Reports - uses: actions/upload-artifact@v4 - with: - if-no-files-found: error - name: coverage-reports-gradle-tests - path: /**/jacocoMergedReport.xml - - - python-e2e-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ env.java-version }} - distribution: 'corretto' - - uses: gradle/actions/setup-gradle@v4 - with: - gradle-version: ${{ env.gradle-version }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.python-version }} - - name: Generate Cache Key from Dockerfiles - id: generate_cache_key - run: | - files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) - file_contents=$(cat $files) - key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') - echo "key=${key}" >> "$GITHUB_OUTPUT" - - name: Cache Docker Images - uses: ScribeMD/docker-cache@0.5.0 - with: - key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} - - name: Start Docker Solution - run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace - env: - OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' - - name: Run E2E test script - working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test - run: | - docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s - - name: Collect Docker, OpenSearch Benchmark, and Shared Logs - if: always() - run: | - mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output - for container in $(docker ps -aq); do - container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///') - docker logs $container > logs/docker/${container_name}_logs.txt 2>&1 - done - docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs - docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output - - name: Upload Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: e2e-test-logs - path: | - logs/docker - logs/opensearch_benchmark_logs - logs/shared_logs_output - - name: Clean up migrations docker images before caching - run: | - docker stop $(docker ps -q) && docker system prune --volumes -f - docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {} - - node-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - npm-project: - - ./deployment/cdk/opensearch-service-migration - - ./deployment/migration-assistant-solution - defaults: - run: - working-directory: ${{ matrix.npm-project }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.node-version }} - - name: Install NPM dependencies - run: npm ci - - name: Run CDK Jest Tests (using mocked images) - run: npm run test - - link-checker: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: lychee Link Checker - id: lychee - uses: lycheeverse/lychee-action@v1 - with: - args: --verbose --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" "**/*.json" - --offline - --exclude "file:///github/workspace/*" - --exclude "http://localhost*" - --exclude "https://localhost*" - --exclude "http://capture-proxy*" - --exclude "https://capture-proxy*" - --exclude-mail - fail: true - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - - upload-codecov: - needs: - - console-python-tests +# gradle-tests: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ env.java-version }} +# distribution: 'corretto' +# - uses: gradle/actions/setup-gradle@v4 +# with: +# gradle-version: ${{ env.gradle-version }} +# gradle-home-cache-cleanup: true +# - name: Run Gradle Build +# run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace +# env: +# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' +# +# - name: Run Tests with Coverage +# run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace +# env: +# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' +# +# - uses: actions/upload-artifact@v4 +# if: always() +# with: +# name: test-reports-gradle-tests +# path: | +# **/build/reports/tests/ +# **/reports/jacoco/mergedReport/ +# - name: Upload Coverage Reports +# uses: actions/upload-artifact@v4 +# with: +# if-no-files-found: error +# name: coverage-reports-gradle-tests +# path: /**/jacocoMergedReport.xml + + +# python-e2e-tests: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ env.java-version }} +# distribution: 'corretto' +# - uses: gradle/actions/setup-gradle@v4 +# with: +# gradle-version: ${{ env.gradle-version }} +# - uses: actions/setup-python@v5 +# with: +# python-version: ${{ env.python-version }} +# - name: Generate Cache Key from Dockerfiles +# id: generate_cache_key +# run: | +# files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) +# file_contents=$(cat $files) +# key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') +# echo "key=${key}" >> "$GITHUB_OUTPUT" +# - name: Cache Docker Images +# uses: ScribeMD/docker-cache@0.5.0 +# with: +# key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} +# - name: Start Docker Solution +# run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace +# env: +# OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: '' +# - name: Run E2E test script +# working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test +# run: | +# docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s +# - name: Collect Docker, OpenSearch Benchmark, and Shared Logs +# if: always() +# run: | +# mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output +# for container in $(docker ps -aq); do +# container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///') +# docker logs $container > logs/docker/${container_name}_logs.txt 2>&1 +# done +# docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs +# docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output +# - name: Upload Logs +# if: always() +# uses: actions/upload-artifact@v4 +# with: +# name: e2e-test-logs +# path: | +# logs/docker +# logs/opensearch_benchmark_logs +# logs/shared_logs_output +# - name: Clean up migrations docker images before caching +# run: | +# docker stop $(docker ps -q) && docker system prune --volumes -f +# docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {} + +# node-tests: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# npm-project: +# - ./deployment/cdk/opensearch-service-migration +# - ./deployment/migration-assistant-solution +# defaults: +# run: +# working-directory: ${{ matrix.npm-project }} +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: ${{ env.node-version }} +# - name: Install NPM dependencies +# run: npm ci +# - name: Run CDK Jest Tests (using mocked images) +# run: npm run test +# +# link-checker: +# runs-on: ubuntu-latest +# +# steps: +# - uses: actions/checkout@v4 +# - name: lychee Link Checker +# id: lychee +# uses: lycheeverse/lychee-action@v1 +# with: +# args: --verbose --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" "**/*.json" +# --offline +# --exclude "file:///github/workspace/*" +# --exclude "http://localhost*" +# --exclude "https://localhost*" +# --exclude "http://capture-proxy*" +# --exclude "https://capture-proxy*" +# --exclude-mail +# fail: true +# env: +# GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + +# upload-codecov: +# needs: +# - console-python-tests +## - gradle-tests +# if: success() +# runs-on: ubuntu-latest +# steps: +# - name: Download Coverage Artifacts +# uses: actions/download-artifact@v4 +# with: +# pattern: coverage-reports-* +# path: ./coverage-reports +# - name: Upload Coverage +# uses: codecov/codecov-action@v4 +# with: +# fail_ci_if_error: true +# files: ./coverage-reports/**/*.* +# flags: unittests +## token: ${{ secrets.CODECOV_TOKEN }} +# verbose: true + +# all-ci-checks-pass: +# needs: +# - console-python-tests # - gradle-tests - if: success() - runs-on: ubuntu-latest - steps: - - name: Download Coverage Artifacts - uses: actions/download-artifact@v4 - with: - pattern: coverage-reports-* - path: ./coverage-reports - - name: Upload Coverage - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: true - files: ./coverage-reports/**/*.* - flags: unittests -# token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - - all-ci-checks-pass: - needs: - - console-python-tests - - gradle-tests - - link-checker - - node-tests - - python-e2e-tests - - python-lint - - style-check - if: always() - runs-on: ubuntu-latest - steps: - - if: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') }} - run: | - echo "One or more job cancelled, failed, or skipped" && exit 1 - - run: | - echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY +# - link-checker +# - node-tests +# - python-e2e-tests +# - python-lint +# - style-check +# if: always() +# runs-on: ubuntu-latest +# steps: +# - if: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') }} +# run: | +# echo "One or more job cancelled, failed, or skipped" && exit 1 +# - run: | +# echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/codecov-publish.yml b/.github/workflows/codecov-publish.yml index c4ea54401..1805fbdfd 100644 --- a/.github/workflows/codecov-publish.yml +++ b/.github/workflows/codecov-publish.yml @@ -1,51 +1,48 @@ -#name: Codecov Publish -# -## Due to the behavior of workflow_run, changes to this file will -## only be reflected once they are pushed up to the `default` (main) branch -# -#on: -# workflow_run: -# workflows: [CI] -# types: -# - completed -# branches: -# - gha_codecov_separate_workflow -# - main -# -#jobs: -# publish-codecov: -# if: ${{ github.event.workflow_run.conclusion == 'success' }} -# runs-on: ubuntu-latest -# steps: -# - name: get PR -# env: -# GH_TOKEN: ${{ github.token }} -# run: | -# - echo "pull_request=$(gh api -H 'Accept: -# application/vnd.github+json' -H 'X-Github-Api-Version: 2022-11-28' \ -# - /repos/${{ github.repository}}/actions/runs/${{ github.event.workflow_run.id }} | jq .pull_requests[0].number)" >> $GITHUB_ENV -# - echo "Pull Request: $pull_request" -# -# - name: Checkout Repository -# uses: actions/checkout@v4 -# -# - name: Download Coverage Artifacts -# uses: actions/download-artifact@v4 -# with: -# pattern: coverage-reports-* -# path: ./coverage-reports -# -# - name: Upload Coverage -# uses: codecov/codecov-action@v4 -# with: -# fail_ci_if_error: true -# files: ./coverage-reports/**/*.* -# flags: unittests -# token: ${{ secrets.CODECOV_TOKEN }} -# verbose: true -# # Manual overrides for these parameters are needed because automatic detection -# # in codecov-action does not work for non-`pull_request` workflows. -# # In `main` branch push, these default to empty strings since we want to run -# # the analysis on HEAD. -# override_commit: ${{ steps.parse_previous_artifacts.outputs.override_commit || '' }} -# override_pr: ${{ steps.parse_previous_artifacts.outputs.override_pr || '' }} +name: Codecov Publish + +# Due to the behavior of workflow_run, changes to this file will +# only be reflected once they are pushed up to the `default` (main) branch + +on: + workflow_run: + workflows: [CI] + types: + - completed + +jobs: + publish-codecov: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: get PR + env: + GH_TOKEN: ${{ github.token }} + run: | + - echo "pull_request=$(gh api -H 'Accept: + application/vnd.github+json' -H 'X-Github-Api-Version: 2022-11-28' \ + - /repos/${{ github.repository}}/actions/runs/${{ github.event.workflow_run.id }} | jq .pull_requests[0].number)" >> $GITHUB_ENV + - echo "Pull Request: $pull_request" + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Download Coverage Artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-reports-* + path: ./coverage-reports + + - name: Upload Coverage + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + files: ./coverage-reports/**/*.* + flags: unittests + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + # Manual overrides for these parameters are needed because automatic detection + # in codecov-action does not work for non-`pull_request` workflows. + # In `main` branch push, these default to empty strings since we want to run + # the analysis on HEAD. + override_commit: ${{ steps.parse_previous_artifacts.outputs.override_commit || '' }} + override_pr: ${{ steps.parse_previous_artifacts.outputs.override_pr || '' }}