Skip to content

Commit

Permalink
Implement UntilPassing
Browse files Browse the repository at this point in the history
Add an annotation that will retry a test until it succeeds.

Affects: #1
  • Loading branch information
io7m committed Sep 15, 2023
1 parent 6edfeb2 commit c22a456
Show file tree
Hide file tree
Showing 6 changed files with 283 additions and 5 deletions.
9 changes: 8 additions & 1 deletion README.in
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ working correctly 98% of the time is acceptable.

* Percentage passing; specify that a percentage of the iterations of a test must succeed.
* Minimum passing: specify that an integral number of the iterations of a test must succeed.
* Until passing: specify that the first invocations of a test might fail, but will eventually succeed.
* Written in pure Java 17.
* [OSGi](https://www.osgi.org/) ready
* [JPMS](https://en.wikipedia.org/wiki/Java_Platform_Module_System) ready
Expand All @@ -22,7 +23,7 @@ working correctly 98% of the time is acceptable.

### Usage

Annotate tests with `@MinimumPassing` or `@PercentPassing`:
Annotate tests with `@MinimumPassing`, `@PercentPassing`, or `@UntilPassing`:


```
Expand All @@ -37,4 +38,10 @@ Annotate tests with `@MinimumPassing` or `@PercentPassing`:
{
// Do something that fails 1% of the time.
}

@UntilPassing(maximumExecutionCount = 1000)
public void testMinimumOK()
{
// Do something that might fail the first few times, but will eventually succeed.
}
```
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ working correctly 98% of the time is acceptable.

* Percentage passing; specify that a percentage of the iterations of a test must succeed.
* Minimum passing: specify that an integral number of the iterations of a test must succeed.
* Until passing: specify that the first invocations of a test might fail, but will eventually succeed.
* Written in pure Java 17.
* [OSGi](https://www.osgi.org/) ready
* [JPMS](https://en.wikipedia.org/wiki/Java_Platform_Module_System) ready
Expand All @@ -37,7 +38,7 @@ working correctly 98% of the time is acceptable.

### Usage

Annotate tests with `@MinimumPassing` or `@PercentPassing`:
Annotate tests with `@MinimumPassing`, `@PercentPassing`, or `@UntilPassing`:


```
Expand All @@ -52,5 +53,11 @@ Annotate tests with `@MinimumPassing` or `@PercentPassing`:
{
// Do something that fails 1% of the time.
}
@UntilPassing(maximumExecutionCount = 1000)
public void testMinimumOK()
{
// Do something that might fail the first few times, but will eventually succeed.
}
```

Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.io7m.percentpass.extension.internal.MinimumPassContext;
import com.io7m.percentpass.extension.internal.PercentPassContext;
import com.io7m.percentpass.extension.internal.PercentPassDisplayNameFormatter;
import com.io7m.percentpass.extension.internal.UntilPassContext;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.InvocationInterceptor;
Expand Down Expand Up @@ -62,16 +63,20 @@ public boolean supportsTestTemplate(
final ExtensionContext context)
{
return isAnnotated(context.getTestMethod(), PercentPassing.class)
|| isAnnotated(context.getTestMethod(), MinimumPassing.class);
|| isAnnotated(context.getTestMethod(), MinimumPassing.class)
|| isAnnotated(context.getTestMethod(), UntilPassing.class);
}

@Override
public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(
final ExtensionContext context)
{
return Stream.concat(
invocationsForPercentPassing(context),
invocationsForMinimumPassing(context)
Stream.concat(
invocationsForPercentPassing(context),
invocationsForMinimumPassing(context)
),
invocationsForUntilPassing(context)
);
}

Expand Down Expand Up @@ -178,6 +183,46 @@ private static Stream<TestTemplateInvocationContext> invocationsForMinimumPassin
.mapToObj(invocation -> container);
}

private static Stream<TestTemplateInvocationContext> invocationsForUntilPassing(
final ExtensionContext context)
{
final UntilPassing passing =
context.getRequiredTestMethod()
.getAnnotation(UntilPassing.class);

if (passing == null) {
return Stream.of();
}

Preconditions.checkPreconditionV(
Integer.valueOf(passing.maximumExecutionCount()),
passing.maximumExecutionCount() > 1,
"Execution count must be greater than 1"
);

final var store =
context.getStore(ExtensionContext.Namespace.create(PercentPassExtension.class));

final var name =
createUntilName(context);

final var existing =
store.get(name, PercentPassContext.class);
if (existing != null) {
throw new IllegalStateException(
String.format("Context %s already registered", name)
);
}

final var formatter =
new PercentPassDisplayNameFormatter(context.getDisplayName());
final var container =
new UntilPassContext(passing, formatter);

store.put(name, container);
return IntStream.rangeClosed(1, passing.maximumExecutionCount())
.mapToObj(invocation -> container);
}

@Override
public void interceptTestTemplateMethod(
Expand All @@ -194,6 +239,8 @@ public void interceptTestTemplateMethod(
createMinimumName(extensionContext);
final var percentName =
createPercentName(extensionContext);
final var untilName =
createUntilName(extensionContext);

final var percentContext =
store.get(percentName, PercentPassContext.class);
Expand All @@ -209,15 +256,32 @@ public void interceptTestTemplateMethod(
minimumContext.addInvocation();
}

final var untilContext =
store.get(untilName, UntilPassContext.class);

try {
if (untilContext != null) {
if (untilContext.hasSucceeded()) {
invocation.skip();
return;
}
}

invocation.proceed();

if (untilContext != null) {
untilContext.setSuccess();
}
} catch (final Throwable e) {
if (percentContext != null) {
percentContext.addFailure();
}
if (minimumContext != null) {
minimumContext.addFailure();
}
if (untilContext != null) {
untilContext.addFailure();
}
throw e;
}
}
Expand All @@ -233,6 +297,7 @@ public void handleTestExecutionException(

checkPercentExecution(context, throwable, store);
checkMinimumExecution(context, throwable, store);
checkUntilExecution(context, throwable, store);
}

private static void checkMinimumExecution(
Expand Down Expand Up @@ -325,6 +390,34 @@ private static void checkPercentExecution(
}
}

private static void checkUntilExecution(
final ExtensionContext context,
final Throwable throwable,
final ExtensionContext.Store store)
{
final var untilName =
createUntilName(context);
final var untilContext =
store.get(untilName, UntilPassContext.class);

if (untilContext == null) {
return;
}

if (untilContext.hasSucceeded()) {
return;
}

if (!untilContext.hasRemaining()) {
Assertions.fail(
new StringBuilder(128)
.append("The test did not succeed within the given maximum number of attempts.")
.append(System.lineSeparator())
.toString()
);
}
}

private static String createPercentName(
final ExtensionContext context)
{
Expand All @@ -344,4 +437,14 @@ private static String createMinimumName(
extensionContext.getRequiredTestMethod().getName()
);
}

private static String createUntilName(
final ExtensionContext extensionContext)
{
return String.format(
"Until %s:%s",
extensionContext.getRequiredTestClass().getCanonicalName(),
extensionContext.getRequiredTestMethod().getName()
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright © 2020 Mark Raynsford <[email protected]> https://www.io7m.com
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
* IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

package com.io7m.percentpass.extension;

import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* An annotation that produces a templated test where the test will be run
* until it passes, up to a configurable maximum number of attempts.
*/

@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD})
@ExtendWith(PercentPassExtension.class)
@TestTemplate
public @interface UntilPassing
{
/**
* @return The maximum number of executions of each test
*/

int maximumExecutionCount() default 100;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright © 2020 Mark Raynsford <[email protected]> https://www.io7m.com
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
* IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

package com.io7m.percentpass.extension.internal;

import com.io7m.percentpass.extension.UntilPassing;
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;

import java.util.Objects;

/**
* The percent pass context.
*/

public final class UntilPassContext implements TestTemplateInvocationContext
{
private final UntilPassing configuration;
private final PercentPassDisplayNameFormatter formatter;
private int remaining;
private boolean succeeded;

/**
* The percent pass context.
*
* @param inConfiguration The configuration
* @param inFormatter The formatter
*/

public UntilPassContext(
final UntilPassing inConfiguration,
final PercentPassDisplayNameFormatter inFormatter)
{
this.configuration =
Objects.requireNonNull(inConfiguration, "configuration");
this.formatter =
Objects.requireNonNull(inFormatter, "formatter");
this.remaining =
this.configuration.maximumExecutionCount();
}

/**
* @return The percent passing configuration
*/

public UntilPassing configuration()
{
return this.configuration;
}

@Override
public String getDisplayName(final int invocationIndex)
{
return this.formatter.format(
invocationIndex,
this.configuration().maximumExecutionCount());
}

/**
* Add a new invocation.
*/

public void addFailure()
{
this.remaining = Math.max(0, this.remaining - 1);
}

/**
* Set the context as having succeeded.
*/

public void setSuccess()
{
this.succeeded = true;
}

/**
* @return {@code true} if the context has succeeded
*/

public boolean hasSucceeded()
{
return this.succeeded;
}

/**
* @return {@code true} if the context has attempts left
*/

public boolean hasRemaining()
{
return this.remaining > 0;
}
}
Loading

0 comments on commit c22a456

Please sign in to comment.