Add --test-workspace and --test-packages (#425)

Allows testing packages that get some test coverage from another package in the workspace

Fixes #394

NEWS.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- New: `--test-workspace` and `--test-package` arguments and config options support projects whose tests live in a different package.
+
 - New: Mutate `proc_macro` targets and functions.
 
 - New: Write diffs to dedicated files under `mutants.out/diff/`. The filename is included in the mutant json output.

book/src/workspaces.md

@@ -1,21 +1,49 @@
 # Workspace and package support
 
-cargo-mutants supports testing Cargo workspaces that contain multiple packages. The entire workspace tree is copied.
+cargo-mutants supports testing Cargo workspaces that contain multiple packages.
 
-By default, cargo-mutants has [the same behavior as Cargo](https://doc.rust-lang.org/cargo/reference/workspaces.html):
+The entire workspace tree is copied to the temporary directory (unless `--in-place` is used).
 
-* If `--workspace` is given, all packages in the workspace are tested.
-* If `--package` is given, the named packages are tested.
-* If the starting directory (or `-d` directory) is in a package, that package is tested.
-* Otherwise, the starting directory must be in a virtual workspace. If it specifies default members, they are tested. Otherwise, all packages are tested.
+In workspaces with multiple packages, there are two considerations:
 
-For each mutant, only the containing package's tests are run, on the theory that
-each package's tests are responsible for testing the package's code.
+1. Which packages to generate mutants in, and
+2. Which tests to run on those mutants.
 
-The baseline tests exercise all and only the packages for which mutants will
-be generated.
+## Selecting packages to mutate
+
+By default, cargo-mutants selects packages to mutate using [similar heuristics to other Cargo commands](https://doc.rust-lang.org/cargo/reference/workspaces.html).
+
+These rules work from the "starting directory", which is the directory selected by `--dir` or the current working directory.
+
+* If `--workspace` is given, all packages in the workspace are mutated.
+* If `--package` is given, the named packages are mutated.
+* If the starting directory is in a package, that package is mutated. Concretely, this means: if the starting directory or its parents contain a `Cargo.toml` containing a `[package]` section.
+* If the starting directory's parents contain a `Cargo.toml` with a `[workspace]` section but no `[package]` section, then the directory is said to be in a "virtual workspace". If the `[workspace]` section has a `default-members` key then these packages are mutated. Otherwise, all packages are mutated.
+
+Selection of packages can be combined with [`--file`](skip_files.md) and other filters.
 
 You can also use the `--file` options to restrict cargo-mutants to testing only files
 from some subdirectory, e.g. with `-f "utils/**/*.rs"`. (Remember to quote globs
 on the command line, so that the shell doesn't expand them.) You can use `--list` or
 `--list-files` to preview the effect of filters.
+
+## Selecting tests to run
+
+For each baseline and mutant scenario, cargo-mutants selects some tests to see if the mutant is caught.
+These selections turn into `--package` or `--workspace` arguments to `cargo test`.
+
+There are different behaviors for the baseline tests (before mutation), which run once for all packages, and then for the tests applied to each mutant.
+
+These behaviors can be controlled by the `--test-workspace` and `--test-package` command line options and the corresponding configuration options.
+
+By default, the baseline runs the tests from all and only the packages for which mutants will be generated. That is, if the whole workspace is being tested, then it runs `cargo test --workspace`, and otherwise runs tests for each selected package.
+
+By default, each mutant runs only the tests from the package that's being mutated.
+
+If the `--test-workspace=true` argument or `test_workspace` configuration key is set, then all tests from the workspace are run for the baseline and against each mutant.
+
+If the `--test-package` argument or `test_package` configuration key is set then the specified packages are tested for the baseline and all mutants.
+
+As for other options, the command line arguments have priority over the configuration file.
+
+Like `--package`, the argument to `--test-package` can be a comma-separated list, or the option can be repeated.

src/build_dir.rs

@@ -2,20 +2,29 @@
 
 //! A directory containing mutated source to run cargo builds and tests.
 
-use std::fmt::{self, Debug};
+#![warn(clippy::pedantic)]
+
 use std::fs::write;
 
+use anyhow::{ensure, Context};
+use camino::{Utf8Path, Utf8PathBuf};
 use tempfile::TempDir;
 use tracing::info;
 
-use crate::copy_tree::copy_tree;
-use crate::manifest::fix_cargo_config;
-use crate::*;
+use crate::{
+    console::Console,
+    copy_tree::copy_tree,
+    manifest::{fix_cargo_config, fix_manifest},
+    options::Options,
+    workspace::Workspace,
+    Result,
+};
 
 /// A directory containing source, that can be mutated, built, and tested.
 ///
 /// Depending on how its constructed, this might be a copy in a tempdir
 /// or the original source directory.
+#[derive(Debug)]
 pub struct BuildDir {
     /// The path of the root of the build directory.
     path: Utf8PathBuf,
@@ -25,42 +34,45 @@ pub struct BuildDir {
     temp_dir: Option<TempDir>,
 }
 
-impl Debug for BuildDir {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("BuildDir")
-            .field("path", &self.path)
-            .finish()
-    }
-}
-
 impl BuildDir {
-    /// Make a new build dir, copying from a source directory, subject to exclusions.
-    pub fn copy_from(
-        source: &Utf8Path,
-        gitignore: bool,
-        leak_temp_dir: bool,
+    /// Make the build dir for the baseline.
+    ///
+    /// Depending on the options, this might be either a copy of the source directory
+    /// or in-place.
+    pub fn for_baseline(
+        workspace: &Workspace,
+        options: &Options,
         console: &Console,
     ) -> Result<BuildDir> {
+        if options.in_place {
+            BuildDir::in_place(workspace.root())
+        } else {
+            BuildDir::copy_from(workspace.root(), options, console)
+        }
+    }
+
+    /// Make a new build dir, copying from a source directory, subject to exclusions.
+    pub fn copy_from(source: &Utf8Path, options: &Options, console: &Console) -> Result<BuildDir> {
         let name_base = format!("cargo-mutants-{}-", source.file_name().unwrap_or("unnamed"));
         let source_abs = source
             .canonicalize_utf8()
             .context("canonicalize source path")?;
-        let temp_dir = copy_tree(source, &name_base, gitignore, console)?;
+        let temp_dir = copy_tree(source, &name_base, options.gitignore, console)?;
         let path: Utf8PathBuf = temp_dir
             .path()
             .to_owned()
             .try_into()
             .context("tempdir path to UTF-8")?;
         fix_manifest(&path.join("Cargo.toml"), &source_abs)?;
         fix_cargo_config(&path, &source_abs)?;
-        let temp_dir = if leak_temp_dir {
+        let temp_dir = if options.leak_dirs {
             let _ = temp_dir.into_path();
             info!(?path, "Build directory will be leaked for inspection");
             None
         } else {
             Some(temp_dir)
         };
-        let build_dir = BuildDir { temp_dir, path };
+        let build_dir = BuildDir { path, temp_dir };
         Ok(build_dir)
     }
 
@@ -70,8 +82,7 @@ impl BuildDir {
             temp_dir: None,
             path: source_path
                 .canonicalize_utf8()
-                .context("canonicalize source path")?
-                .to_owned(),
+                .context("canonicalize source path")?,
         })
     }
 
@@ -90,16 +101,21 @@ impl BuildDir {
 
 #[cfg(test)]
 mod test {
-    use test_util::copy_of_testdata;
+    use crate::test_util::copy_of_testdata;
 
     use super::*;
 
     #[test]
     fn build_dir_copy_from() {
         let tmp = copy_of_testdata("factorial");
         let workspace = Workspace::open(tmp.path()).unwrap();
-        let build_dir =
-            BuildDir::copy_from(workspace.root(), true, false, &Console::new()).unwrap();
+        let options = Options {
+            in_place: false,
+            gitignore: true,
+            leak_dirs: false,
+            ..Default::default()
+        };
+        let build_dir = BuildDir::copy_from(workspace.root(), &options, &Console::new()).unwrap();
         let debug_form = format!("{build_dir:?}");
         println!("debug form is {debug_form:?}");
         assert!(debug_form.starts_with("BuildDir { path: "));
@@ -108,6 +124,43 @@ mod test {
         assert!(build_dir.path().join("src").is_dir());
     }
 
+    #[test]
+    fn for_baseline_in_place() -> Result<()> {
+        let tmp = copy_of_testdata("factorial");
+        let workspace = Workspace::open(tmp.path())?;
+        let options = Options {
+            in_place: true,
+            ..Default::default()
+        };
+        let build_dir = BuildDir::for_baseline(&workspace, &options, &Console::new())?;
+        assert_eq!(
+            build_dir.path().canonicalize_utf8()?,
+            workspace.root().canonicalize_utf8()?
+        );
+        assert!(build_dir.temp_dir.is_none());
+        Ok(())
+    }
+
+    #[test]
+    fn for_baseline_copied() -> Result<()> {
+        let tmp = copy_of_testdata("factorial");
+        let workspace = Workspace::open(tmp.path())?;
+        let options = Options {
+            in_place: false,
+            ..Default::default()
+        };
+        let build_dir = BuildDir::for_baseline(&workspace, &options, &Console::new())?;
+        assert!(build_dir.path().is_dir());
+        assert!(build_dir.path().join("Cargo.toml").is_file());
+        assert!(build_dir.path().join("src").is_dir());
+        assert!(build_dir.temp_dir.is_some());
+        assert_ne!(
+            build_dir.path().canonicalize_utf8()?,
+            workspace.root().canonicalize_utf8()?
+        );
+        Ok(())
+    }
+
     #[test]
     fn build_dir_in_place() -> Result<()> {
         let tmp = copy_of_testdata("factorial");