From 07bd15d6d5aea84b6cb2f13a8ee3bb8b6301d097 Mon Sep 17 00:00:00 2001 From: Koen Bekkenutte <2912652+kbekkenutte@users.noreply.github.com> Date: Mon, 26 Apr 2021 19:18:10 +0800 Subject: [PATCH] Refactored testCaseRunner --- samples/BasicSample/BasicScenario.cs | 1 + ...redMessageBus.cs => BufferedMessageBus.cs} | 14 +- .../Internal/ScenarioFactTestCaseRunner.cs | 256 ++++++++---------- tests/ScenarioTests.Tests/IntegrationTests.cs | 49 +++- 4 files changed, 162 insertions(+), 158 deletions(-) rename src/ScenarioTests/Internal/{FilteredMessageBus.cs => BufferedMessageBus.cs} (57%) diff --git a/samples/BasicSample/BasicScenario.cs b/samples/BasicSample/BasicScenario.cs index f1286fb..80d8e51 100644 --- a/samples/BasicSample/BasicScenario.cs +++ b/samples/BasicSample/BasicScenario.cs @@ -1,3 +1,4 @@ +using NuGet.Frameworks; using ScenarioTests; using System; using System.Threading.Tasks; diff --git a/src/ScenarioTests/Internal/FilteredMessageBus.cs b/src/ScenarioTests/Internal/BufferedMessageBus.cs similarity index 57% rename from src/ScenarioTests/Internal/FilteredMessageBus.cs rename to src/ScenarioTests/Internal/BufferedMessageBus.cs index ffbc52d..3c5a7d9 100644 --- a/src/ScenarioTests/Internal/FilteredMessageBus.cs +++ b/src/ScenarioTests/Internal/BufferedMessageBus.cs @@ -8,27 +8,25 @@ namespace ScenarioTests.Internal { - sealed internal class FilteredMessageBus : IMessageBus + sealed internal class BufferedMessageBus : IMessageBus { readonly IMessageBus _messageBus; - readonly Func _filter; + readonly List _queue = new(); - public FilteredMessageBus(IMessageBus messageBus, Func filter) + public BufferedMessageBus(IMessageBus messageBus) { _messageBus = messageBus; - _filter = filter; } public bool QueueMessage(IMessageSinkMessage message) { - if (_filter(message)) - { - return _messageBus.QueueMessage(message); - } + _queue.Add(message); return true; // prevent xunit from cancelling } + public IEnumerable QueuedMessages => _queue; + public void Dispose() { } } } diff --git a/src/ScenarioTests/Internal/ScenarioFactTestCaseRunner.cs b/src/ScenarioTests/Internal/ScenarioFactTestCaseRunner.cs index 167c8a1..d9e320e 100644 --- a/src/ScenarioTests/Internal/ScenarioFactTestCaseRunner.cs +++ b/src/ScenarioTests/Internal/ScenarioFactTestCaseRunner.cs @@ -12,54 +12,6 @@ namespace ScenarioTests.Internal { sealed internal class ScenarioFactTestCaseRunner : XunitTestCaseRunner { - readonly HashSet _testedArguments = new(); - readonly Queue _queuedMessages = new(); - readonly Queue _backupQueuedMessages = new(); - - ScenarioContext _scenarioContext; - bool _skipAdditionalTests; - bool _pendingRestart; - - void FlushQueuedMessages() - { - // Only if we were able to run at least 1 fact/theory case - if (_queuedMessages.OfType().Any()) - { - var outputBuilder = new StringBuilder(); - - foreach (var outputMessage in _queuedMessages.OfType()) - { - outputBuilder.Append(outputMessage.Output); - } - - var output = outputBuilder.ToString(); - - while (_queuedMessages.Count > 0) - { - var message = _queuedMessages.Dequeue(); - - var transformedMessage = message switch - { - TestPassed testPassed => new TestPassed(testPassed.Test, testPassed.ExecutionTime, output), - TestFailed testFailed => new TestFailed(testFailed.Test, testFailed.ExecutionTime, output, testFailed.ExceptionTypes, testFailed.Messages, testFailed.StackTraces, testFailed.ExceptionParentIndices), - TestFinished testFinished => new TestFinished(testFinished.Test, testFinished.ExecutionTime, output), - _ => message - }; - - MessageBus.QueueMessage(transformedMessage); - } - } - else - { - // We likely ran into an exception before our fact or theory could run, report here - while (_backupQueuedMessages.Count > 0) - { - var message = _backupQueuedMessages.Dequeue(); - MessageBus.QueueMessage(message); - } - } - } - public ScenarioFactTestCaseRunner(IXunitTestCase testCase, string displayName, string skipReason, @@ -81,128 +33,138 @@ public ScenarioFactTestCaseRunner(IXunitTestCase testCase, protected override async Task RunTestAsync() { var scenarioFactTestCase = (ScenarioFactTestCase)TestCase; - _scenarioContext = new ScenarioContext(scenarioFactTestCase.FactName, RecordTestCase); - - TestMethodArguments = new object[] { _scenarioContext }; - - var filteredMessageBus = new FilteredMessageBus(MessageBus, message => - { - _backupQueuedMessages.Enqueue(message); - - if (message is not ITestStarting and not ITestPassed and not ITestFailed and not ITestFinished ) - { - _queuedMessages.Enqueue(message); - } - - return false; - }); + var test = CreateTest(TestCase, DisplayName); + var aggregatedResult = new RunSummary(); - RunSummary aggregatedResult = new(); + // Theories are called with required arguments. Keep track of what arguments we already tested so that we can skip those accordingly + var testedArguments = new HashSet(); - _testedArguments.Clear(); + // Each time we find a new theory argument, we will want to restart our Test so that we can collect subsequent test cases + bool pendingRestart; do { - _queuedMessages.Clear(); - _backupQueuedMessages.Clear(); - _skipAdditionalTests = false; - _pendingRestart = false; - - var test = CreateTest(TestCase, DisplayName); - RunSummary result; - - // safeguarding against abuse - if (_testedArguments.Count >= scenarioFactTestCase.TheoryTestCaseLimit) + // Safeguarding against abuse + if (testedArguments.Count >= scenarioFactTestCase.TheoryTestCaseLimit) { - _queuedMessages.Enqueue(new TestSkipped(test, "Theory tests are capped to prevent infinite loops. You can configure a different limit by setting TheoryTestCaseLimit on the Scenario attribute")); - result = new RunSummary + pendingRestart = false; + MessageBus.QueueMessage(new TestSkipped(test, "Theory tests are capped to prevent infinite loops. You can configure a different limit by setting TheoryTestCaseLimit on the Scenario attribute")); + aggregatedResult.Aggregate(new RunSummary { Skipped = 1, Total = 1 - }; + }); } else { - result = await CreateTestRunner(test, filteredMessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource).RunAsync(); - aggregatedResult.Aggregate(result); - } - - FlushQueuedMessages(); - } - while (_pendingRestart); - - Console.WriteLine(_pendingRestart); - - return aggregatedResult; - } - - async Task RecordTestCase(object? argument, Func invocation) - { - if (_skipAdditionalTests) - { - _pendingRestart = true; // when we discovered more tests after a test completed, allow us to restart - return; - } - - if (argument is not null) - { - if (_testedArguments.Contains(argument)) - { - return; - } + var bufferedMessageBus = new BufferedMessageBus(MessageBus); + var stopwatch = Stopwatch.StartNew(); + var skipAdditionalTests = false; + pendingRestart = false; // By default we dont expect a new restart - _testedArguments.Add(argument); - } - - var testDisplayName = argument is not null ? $"{DisplayName}({argument})" : DisplayName; - var test = CreateTest(TestCase, testDisplayName); - var stopwatch = new Stopwatch(); - - _queuedMessages.Enqueue(new TestStarting(test)); + object? capturedArgument = null; + ScenarioContext scenarioContext = null; - if (_scenarioContext.Skipped) - { - _queuedMessages.Enqueue(new TestSkipped(test, _scenarioContext.SkippedReason)); - return; // We dont want to run this test case - } + scenarioContext = new ScenarioContext(scenarioFactTestCase.FactName, async (object? argument, Func invocation) => + { + if (skipAdditionalTests) + { + pendingRestart = true; // when we discovered more tests after a test completed, allow us to restart + return; + } + + if (argument is not null) + { + if (testedArguments.Contains(argument)) + { + return; + } + + testedArguments.Add(argument); + capturedArgument = argument; + } + + // At this stage we found our first valid test case, any subsequent test case should issue a restart instead + skipAdditionalTests = true; + + if (scenarioContext.Skipped) + { + bufferedMessageBus.QueueMessage(new TestSkipped(test, scenarioContext.SkippedReason)); + } + else + { + try + { + await invocation(); + } + catch (Exception ex) + { + bufferedMessageBus.QueueMessage(new TestFailed(test, 0, string.Empty, ex)); + throw; + } + finally + { + if (scenarioContext.Skipped) + { + bufferedMessageBus.QueueMessage(new TestSkipped(test, scenarioContext.SkippedReason)); + } + } + } + }); + + TestMethodArguments = new object[] { scenarioContext }; + + RunSummary result; + + result = await CreateTestRunner(test, bufferedMessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource).RunAsync(); + aggregatedResult.Aggregate(result); - stopwatch.Start(); + stopwatch.Stop(); + var testInvocationTest = capturedArgument switch + { + null => CreateTest(TestCase, DisplayName), + not null => CreateTest(TestCase, $"{DisplayName} ({capturedArgument})") + }; - try - { - await invocation(); + var bufferedMessages = bufferedMessageBus.QueuedMessages; + if (bufferedMessages.OfType().Any()) + { + bufferedMessages = bufferedMessages.Where(x => x is not TestPassed and not TestFailed); + } - stopwatch.Stop(); + if (bufferedMessages.OfType().Any()) + { + bufferedMessages = bufferedMessages.Where(x => x is not TestPassed); + } - if (_scenarioContext.Skipped) - { - _queuedMessages.Enqueue(new TestSkipped(test, _scenarioContext.SkippedReason)); - } - else - { - _queuedMessages.Enqueue(new TestPassed(test, (decimal)stopwatch.Elapsed.TotalSeconds, null)); - } - } - catch (Exception ex) - { - stopwatch.Stop(); + var output = string.Join("", bufferedMessages + .OfType() + .Select(x => x.Output)); - if (_scenarioContext.Skipped) - { - _queuedMessages.Enqueue(new TestSkipped(test, _scenarioContext.SkippedReason)); - } - else - { var duration = (decimal)stopwatch.Elapsed.TotalSeconds; - _queuedMessages.Enqueue(new TestFailed(test, duration, null, ex)); + + foreach (var queuedMessage in bufferedMessages) + { + var transformedMessage = queuedMessage switch + { + TestStarting testStarting => new TestStarting(testInvocationTest), + TestSkipped testSkipped => new TestSkipped(testInvocationTest, testSkipped.Reason), + TestPassed testPassed => new TestPassed(testInvocationTest, duration, output), + TestFailed testFailed => new TestFailed(testInvocationTest, duration, output, testFailed.ExceptionTypes, testFailed.Messages, testFailed.StackTraces, testFailed.ExceptionParentIndices), + TestFinished testFinished => new TestFinished(testInvocationTest, duration, output), + _ => queuedMessage + }; + + if (!MessageBus.QueueMessage(transformedMessage)) + { + return aggregatedResult; + } + } } } - finally - { - _skipAdditionalTests = true; - } + while (pendingRestart); - _queuedMessages.Enqueue(new TestFinished(test, (decimal)stopwatch.Elapsed.TotalSeconds, null)); + return aggregatedResult; } } } diff --git a/tests/ScenarioTests.Tests/IntegrationTests.cs b/tests/ScenarioTests.Tests/IntegrationTests.cs index 3178454..393015c 100644 --- a/tests/ScenarioTests.Tests/IntegrationTests.cs +++ b/tests/ScenarioTests.Tests/IntegrationTests.cs @@ -9,7 +9,7 @@ namespace ScenarioTests.Tests { public class IntegrationTests { - [Internal.ScenarioFact(DisplayName = nameof(SimpleFact), FactName = "X")] + [Internal.ScenarioFact(FactName = "X")] public void SimpleFact(ScenarioContext scenarioContext) { scenarioContext.Fact("X", () => @@ -19,7 +19,7 @@ public void SimpleFact(ScenarioContext scenarioContext) } - [Internal.ScenarioFact(DisplayName = nameof(SimpleTheory), FactName = "X")] + [Internal.ScenarioFact(FactName = "X")] public void SimpleTheory(ScenarioContext scenarioContext) { var invocations = 0; @@ -35,7 +35,7 @@ public void SimpleTheory(ScenarioContext scenarioContext) Assert.Equal(1, invocations); } - [Internal.ScenarioFact(DisplayName = nameof(SimpleTheory2), FactName = "X")] + [Internal.ScenarioFact(FactName = "X")] public void SimpleTheory2(ScenarioContext scenarioContext) { var invocations = 0; @@ -50,5 +50,48 @@ public void SimpleTheory2(ScenarioContext scenarioContext) Assert.Equal(1, invocations); } + + [Internal.ScenarioFact(FactName = "X")] + public async Task DelayedPreconditions(ScenarioContext scenarioContext) + { + // We need to manually confirm that indeed the 200ms was added to the total test duration + await Task.Delay(200); + + scenarioContext.Fact("X", () => + { + }); + } + + [Internal.ScenarioFact(FactName = "X")] + public async Task DelayedPostconditions(ScenarioContext scenarioContext) + { + scenarioContext.Fact("X", () => + { + }); + + // We need to manually confirm that indeed the 200ms was added to the total test duration + await Task.Delay(200); + } + + //[Internal.ScenarioFact(FactName = "X")] + //public void PostException(ScenarioContext scenarioContext) + //{ + // scenarioContext.Fact("X", () => + // { + // }); + + // // We need to manually confirm that indeed the 200ms was added to the total test duration + // throw new Exception(); + //} + + [Internal.ScenarioFact(FactName = "X")] + public void SkippedTest(ScenarioContext scenarioContext) + { + scenarioContext.Skip("Foo"); + + scenarioContext.Fact("X", () => + { + }); + } } }