From f9079c17006c30287b43f34790b4f5d2be28dd1b Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 26 Jan 2024 09:48:58 -0800 Subject: [PATCH 1/9] Run actionlint on all github actions Fixes #266 --- .github/workflows/actionlint.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/actionlint.yml diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 00000000..c0412b14 --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,26 @@ +on: + pull_request: + paths: + - ".github/workflows/*.yml" + push: + paths: + - ".github/workflows/*.yml" + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +jobs: + action-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: devops-actions/actionlint@v0.1.2 + id: action-lint + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: actionlint-results + path: ${{ steps.action-lint.outputs.results-file }} From a6b124efabbf1e685b344f84b06d741d0c85444c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 26 Jan 2024 09:53:05 -0800 Subject: [PATCH 2/9] Exclude release.yml --- .github/workflows/actionlint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index c0412b14..45f2d4d2 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -1,10 +1,14 @@ +name: actionlint + on: pull_request: paths: - ".github/workflows/*.yml" + - "!.github/workflows/release.yml" # generated push: paths: - ".github/workflows/*.yml" + - "!.github/workflows/release.yml" workflow_dispatch: permissions: From a400c68fdadc679b3442a49c6d8a94290f32edf5 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 26 Jan 2024 09:53:32 -0800 Subject: [PATCH 3/9] Only run on pushes to main --- .github/workflows/actionlint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 45f2d4d2..f5ef9da4 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -6,6 +6,8 @@ on: - ".github/workflows/*.yml" - "!.github/workflows/release.yml" # generated push: + branches: + - main paths: - ".github/workflows/*.yml" - "!.github/workflows/release.yml" From 53e69c91730d369cb2ff5bf512d93c1485e05ee1 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 26 Jan 2024 10:39:59 -0800 Subject: [PATCH 4/9] Try to ignore release.yml Run from docker directly --- .github/workflows/actionlint.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index f5ef9da4..26da302c 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -4,13 +4,11 @@ on: pull_request: paths: - ".github/workflows/*.yml" - - "!.github/workflows/release.yml" # generated push: branches: - main paths: - ".github/workflows/*.yml" - - "!.github/workflows/release.yml" workflow_dispatch: permissions: @@ -21,12 +19,9 @@ jobs: action-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: devops-actions/actionlint@v0.1.2 - id: action-lint - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: actionlint-results - path: ${{ steps.action-lint.outputs.results-file }} + - uses: actions/checkout@v4 + - name: Check workflow files + shell: bash + run: | + bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + ./actionlint -color $(find .github/workflows -name '*.yml' -type f ! -name release.yml -print) From 3558837668b2e9ea3f357b975bf653b5f01ca240 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 27 Jan 2024 23:15:50 -0800 Subject: [PATCH 5/9] Try to avoid shellcheck errors --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eaafc66c..e07b080f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -107,7 +107,7 @@ jobs: name: cargo-mutants-linux - name: Install cargo-mutants binary run: | - install cargo-mutants $HOME/.cargo/bin/ + install cargo-mutants ~/.cargo/bin/ - name: Mutants in-diff # Normally this would have --in-place, but for the sake of exercising more cases, it does not. run: > @@ -143,7 +143,7 @@ jobs: name: cargo-mutants-linux - name: Install cargo-mutants binary run: | - install cargo-mutants $HOME/.cargo/bin/ + install cargo-mutants ~/.cargo/bin/ - name: Mutants # Skip baselines because this action depends on the tests run: > From 24da40ce4a14e76c79f5fcbc179ea1697c29fec6 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 27 Jan 2024 23:21:29 -0800 Subject: [PATCH 6/9] Silence one shellcheck warning --- .github/workflows/actionlint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 26da302c..eb466ef3 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -24,4 +24,5 @@ jobs: shell: bash run: | bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + # shellcheck disable=SC2046 ./actionlint -color $(find .github/workflows -name '*.yml' -type f ! -name release.yml -print) From 400203d7aaeb47d0e6e7c48426aaefba8cce685a Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 2 Feb 2024 07:27:41 -0800 Subject: [PATCH 7/9] Split out and test basic CI workflow --- .github/workflows/actionlint.yml | 4 ++- book/src/ci.md | 41 +------------------------------ examples/workflows/basic.yml | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 examples/workflows/basic.yml diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index eb466ef3..bb543289 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -4,11 +4,13 @@ on: pull_request: paths: - ".github/workflows/*.yml" + - "examples/workflows/*.yml" push: branches: - main paths: - ".github/workflows/*.yml" + - "examples/workflows/*.yml" workflow_dispatch: permissions: @@ -25,4 +27,4 @@ jobs: run: | bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) # shellcheck disable=SC2046 - ./actionlint -color $(find .github/workflows -name '*.yml' -type f ! -name release.yml -print) + ./actionlint -color $(find .github/workflows examples/workflows -name '*.yml' -type f ! -name release.yml -print) diff --git a/book/src/ci.md b/book/src/ci.md index 1a589996..91525093 100644 --- a/book/src/ci.md +++ b/book/src/ci.md @@ -21,44 +21,5 @@ The recommended way to install cargo-mutants is using [install-action](https://g Here is an example of a GitHub Actions workflow that runs mutation tests and uploads the results as an artifact. This will fail if it finds any uncaught mutants. ```yml -name: cargo-mutants - -env: - CARGO_TERM_COLOR: always - -on: - push: - branches: - - main - pull_request: - # Only test PR if it changes something that's likely to affect the results, because - # mutant tests can take a long time. Adjust these paths to suit your project. - paths: - - ".cargo/**" - - ".github/workflows/mutants.yml" - - "Cargo.*" - - "src/**" - - "testdata/**" - - "tests/**" - -jobs: - cargo-mutants: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - uses: taiki-e/install-action@v2 - name: Install cargo-mutants using install-action - with: - tool: cargo-mutants - - name: Run mutant tests - run: cargo mutants -vV --in-place - - name: Archive results - uses: actions/upload-artifact@v4 - if: always() - with: - name: mutants-out - path: mutants.out +{{#include ../../examples/workflows/basic.yml}} ``` diff --git a/examples/workflows/basic.yml b/examples/workflows/basic.yml new file mode 100644 index 00000000..45059e26 --- /dev/null +++ b/examples/workflows/basic.yml @@ -0,0 +1,42 @@ +# Example of how to configure a GitHub Actions workflow to run `cargo mutants` +# on every push to main and every pull request that changes the code. + +# You could run this standalone or merge it into a workflow that runs other tests. + +name: cargo-mutants + +env: + CARGO_TERM_COLOR: always + +on: + push: + branches: + - main + pull_request: + # Only test PR if it changes something that's likely to affect the results, because + # mutant tests can take a long time. Adjust these paths to suit your project. + paths: + - ".cargo/mutants.toml" + - ".github/workflows/tests.yml" + - "Cargo.*" + - "src/**" + - "testdata/**" + - "tests/**" + +jobs: + cargo-mutants: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - uses: taiki-e/install-action@v2 + with: + tool: cargo-mutants + - run: cargo mutants -vV --in-place + - uses: actions/upload-artifact@v4 + if: always() + with: + name: mutants-out + path: mutants.out From 79ca762ef13a9b569ebd4741fccff98f21ff695c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 2 Feb 2024 07:33:05 -0800 Subject: [PATCH 8/9] Also split out in-diff example --- book/src/pr-diff.md | 41 +----------------------------- examples/workflows/in-diff.yml | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 40 deletions(-) create mode 100644 examples/workflows/in-diff.yml diff --git a/book/src/pr-diff.md b/book/src/pr-diff.md index 89121059..e516893e 100644 --- a/book/src/pr-diff.md +++ b/book/src/pr-diff.md @@ -5,44 +5,5 @@ You can use `--in-diff` to test only the code that has changed in a pull request For example, you can use the following workflow to test only the code that has changed in a pull request: ```yaml -name: Tests - -permissions: - contents: read - -env: - CARGO_TERM_COLOR: always - -on: - push: - branches: - - main - pull_request: - -jobs: - incremental-mutants: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Relative diff - run: | - git branch -av - git diff origin/${{ github.base_ref }}.. | tee git.diff - - uses: Swatinem/rust-cache@v2 - - uses: taiki-e/install-action@v2 - name: Install cargo-mutants using install-action - with: - tool: cargo-mutants - - name: Mutants - run: | - cargo mutants --no-shuffle -j 2 -vV --in-diff git.diff - - name: Archive mutants.out - uses: actions/upload-artifact@v4 - if: always() - with: - name: mutants-incremental.out - path: mutants.out +{{#include ../../examples/workflows/in-diff.yml}} ``` diff --git a/examples/workflows/in-diff.yml b/examples/workflows/in-diff.yml new file mode 100644 index 00000000..d017edfd --- /dev/null +++ b/examples/workflows/in-diff.yml @@ -0,0 +1,46 @@ +# An example of how to run cargo-mutants on only the sections of code that have changed in a pull request, +# using the `--in-diff` feature of cargo-mutants. +# +# This can give much faster feedback on pull requests, but can miss some problems that +# would be found by running mutants on the whole codebase. + +name: Tests + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +on: + push: + branches: + - main + pull_request: + +jobs: + incremental-mutants: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Relative diff + run: | + git branch -av + git diff origin/${{ github.base_ref }}.. | tee git.diff + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@v2 + name: Install cargo-mutants using install-action + with: + tool: cargo-mutants + - name: Mutants + run: | + cargo mutants --no-shuffle -vV --in-diff git.diff + - name: Archive mutants.out + uses: actions/upload-artifact@v4 + if: always() + with: + name: mutants-incremental.out + path: mutants.out From af04d6d23d8be97b03183fea2d811c0f18802430 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 2 Feb 2024 07:37:25 -0800 Subject: [PATCH 9/9] Link to CI instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2fb9702f..63ecbcb7 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,10 @@ To generate mutants in only one file: cargo mutants -f src/something.rs ``` +## Integration with CI + +The [manual includes instructions and examples for automatically testing mutants in CI](https://mutants.rs/ci.html), including incremental testing of pull requests and full testing of the development branch. + ## Help advance cargo-mutants If you use cargo-mutants or just like the idea you can help it get better: