From 8ad6ae3bf14ea303dd2b6c36ceda0106e25037a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 May 2024 13:38:19 +0200 Subject: [PATCH 1/2] Include max cases in ExUnit reports Some CI environments may report the wrong number of cores to the Erlang VM. So we choose to print the number of max cases along side the seed (both which affect randomness of tests) at the top of each suite. --- lib/ex_unit/lib/ex_unit/cli_formatter.ex | 22 +++++++++------------- lib/ex_unit/test/ex_unit_test.exs | 12 +++++------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/cli_formatter.ex b/lib/ex_unit/lib/ex_unit/cli_formatter.ex index 613144880b..a48b3648d2 100644 --- a/lib/ex_unit/lib/ex_unit/cli_formatter.ex +++ b/lib/ex_unit/lib/ex_unit/cli_formatter.ex @@ -8,10 +8,12 @@ defmodule ExUnit.CLIFormatter do ## Callbacks def init(opts) do - print_filters(Keyword.take(opts, [:include, :exclude])) + IO.puts("Running ExUnit with seed: #{opts[:seed]}, max_cases: #{opts[:max_cases]}") + print_filters(opts, :include) + print_filters(opts, :exclude) + IO.puts("") config = %{ - seed: opts[:seed], trace: opts[:trace], colors: colors(opts), width: get_terminal_width(), @@ -356,22 +358,16 @@ defmodule ExUnit.CLIFormatter do true -> IO.puts(success(message, config)) end - - IO.puts("\nRandomized with seed #{config.seed}") end defp if_true(value, false, _fun), do: value defp if_true(value, true, fun), do: fun.(value) - defp print_filters(include: [], exclude: []) do - :ok - end - - defp print_filters(include: include, exclude: exclude) do - if exclude != [], do: IO.puts(format_filters(exclude, :exclude)) - if include != [], do: IO.puts(format_filters(include, :include)) - IO.puts("") - :ok + defp print_filters(opts, key) do + case opts[key] do + [] -> :ok + filters -> IO.puts(format_filters(filters, key)) + end end defp print_failure(formatted, config) do diff --git a/lib/ex_unit/test/ex_unit_test.exs b/lib/ex_unit/test/ex_unit_test.exs index 3936b6754a..765376f2ac 100644 --- a/lib/ex_unit/test/ex_unit_test.exs +++ b/lib/ex_unit/test/ex_unit_test.exs @@ -138,8 +138,6 @@ defmodule ExUnitTest do Showing results so far... 0 failures - - Randomized with seed 0 """ end @@ -926,7 +924,7 @@ defmodule ExUnitTest do end describe ":repeat_until_failure" do - test "default value 0" do + test "defaults to 0" do configure_and_reload_on_exit([]) ExUnit.start(autorun: false) config = ExUnit.configuration() @@ -940,7 +938,7 @@ defmodule ExUnitTest do assert config[:repeat_until_failure] == 5 end - test ":repeat_until_failure repeats tests up to the configured number of times" do + test "repeats tests up to the configured number of times" do defmodule TestRepeatUntilFailureReached do use ExUnit.Case @@ -962,12 +960,12 @@ defmodule ExUnitTest do assert ExUnit.run() == %{total: 5, failures: 0, skipped: 1, excluded: 1} end) - runs = String.split(output, "Excluding", trim: true) + runs = String.split(output, "Running ExUnit", trim: true) # 6 runs in total, 5 repeats assert length(runs) == 6 end - test ":repeat_until_failure stops on failure" do + test "stops on failure" do {:ok, pid} = Agent.start_link(fn -> 0 end) Process.register(pid, :ex_unit_repeat_until_failure_count) @@ -1002,7 +1000,7 @@ defmodule ExUnitTest do assert ExUnit.run() == %{total: 4, excluded: 2, failures: 1, skipped: 1} end) - runs = String.split(output, "Excluding", trim: true) + runs = String.split(output, "Running ExUnit", trim: true) # four runs in total, the first two repeats work fine, the third repeat (4th run) # fails, therefore we stop assert length(runs) == 4 From 308195b23c8db399972e80a48fba4807065c7da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Wed, 1 May 2024 14:00:39 +0200 Subject: [PATCH 2/2] Fix mix tests and docs --- .../pages/mix-and-otp/introduction-to-mix.md | 3 +-- lib/ex_unit/lib/ex_unit/cli_formatter.ex | 2 +- lib/mix/lib/mix/tasks/test.ex | 3 +-- lib/mix/test/mix/tasks/test_test.exs | 27 ++++++++----------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/elixir/pages/mix-and-otp/introduction-to-mix.md b/lib/elixir/pages/mix-and-otp/introduction-to-mix.md index af1e5fa171..ed131fe916 100644 --- a/lib/elixir/pages/mix-and-otp/introduction-to-mix.md +++ b/lib/elixir/pages/mix-and-otp/introduction-to-mix.md @@ -213,12 +213,11 @@ This file will be required by Mix every time before we run our tests. We can run $ mix test Compiled lib/kv.ex Generated kv app +Running ExUnit with seed: 540224, max_cases: 16 .. Finished in 0.04 seconds 1 doctest, 1 test, 0 failures - -Randomized with seed 540224 ``` Notice that by running `mix test`, Mix has compiled the source files and generated the application manifest once again. This happens because Mix supports multiple environments, which we will discuss later in this chapter. diff --git a/lib/ex_unit/lib/ex_unit/cli_formatter.ex b/lib/ex_unit/lib/ex_unit/cli_formatter.ex index a48b3648d2..67a1dda53d 100644 --- a/lib/ex_unit/lib/ex_unit/cli_formatter.ex +++ b/lib/ex_unit/lib/ex_unit/cli_formatter.ex @@ -9,8 +9,8 @@ defmodule ExUnit.CLIFormatter do def init(opts) do IO.puts("Running ExUnit with seed: #{opts[:seed]}, max_cases: #{opts[:max_cases]}") - print_filters(opts, :include) print_filters(opts, :exclude) + print_filters(opts, :include) IO.puts("") config = %{ diff --git a/lib/mix/lib/mix/tasks/test.ex b/lib/mix/lib/mix/tasks/test.ex index 8109142c41..7b705adb17 100644 --- a/lib/mix/lib/mix/tasks/test.ex +++ b/lib/mix/lib/mix/tasks/test.ex @@ -36,6 +36,7 @@ defmodule Mix.Tasks.Test do a summary at the end, as seen below: $ mix test + Running ExUnit with seed: 646219, max_cases: 16 ... 1) test greets the world (FooTest) @@ -52,8 +53,6 @@ defmodule Mix.Tasks.Test do Finished in 0.05 seconds (0.00s async, 0.05s sync) 1 doctest, 11 tests, 1 failure - Randomized with seed 646219 - For each test, the test suite will print a dot. Failed tests are printed immediately in the format described in the next section. diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs index f0809cd7b9..0097e52236 100644 --- a/lib/mix/test/mix/tasks/test_test.exs +++ b/lib/mix/test/mix/tasks/test_test.exs @@ -265,7 +265,7 @@ defmodule Mix.Tasks.TestTest do in_fixture("test_stale", fn -> port = mix_port(~w[test --stale --listen-on-stdin]) - assert receive_until_match(port, "seed", "") =~ "2 tests" + assert receive_until_match(port, "0 failures", "") =~ "2 tests" Port.command(port, "\n") @@ -294,7 +294,7 @@ defmodule Mix.Tasks.TestTest do Port.command(port, "\n") - assert receive_until_match(port, "seed", "") =~ "2 tests" + assert receive_until_match(port, "0 failures", "") =~ "2 tests" File.write!("test/b_test_stale.exs", """ defmodule BTest do @@ -323,7 +323,7 @@ defmodule Mix.Tasks.TestTest do Port.command(port, "\n") - assert receive_until_match(port, "seed", "") =~ "2 tests" + assert receive_until_match(port, "0 failures", "") =~ "2 tests" end) end end @@ -477,38 +477,33 @@ defmodule Mix.Tasks.TestTest do output = mix(["test", "apps/bar/test/bar_tests.exs"]) - assert output =~ """ - ==> bar - .... - """ + assert output =~ "==> bar" + assert output =~ "...." refute output =~ "==> foo" refute output =~ "Paths given to \"mix test\" did not match any directory/file" output = mix(["test", "./apps/bar/test/bar_tests.exs"]) - assert output =~ """ - ==> bar - .... - """ + assert output =~ "==> bar" + assert output =~ "...." refute output =~ "==> foo" refute output =~ "Paths given to \"mix test\" did not match any directory/file" output = mix(["test", Path.expand("apps/bar/test/bar_tests.exs")]) - assert output =~ """ - ==> bar - .... - """ + assert output =~ "==> bar" + assert output =~ "...." refute output =~ "==> foo" refute output =~ "Paths given to \"mix test\" did not match any directory/file" output = mix(["test", "apps/bar/test/bar_tests.exs:10"]) + assert output =~ "==> bar" + assert output =~ """ - ==> bar Excluding tags: [:test] Including tags: [location: {"test/bar_tests.exs", 10}]