From 199037f9c80afd885f1f536d91b40a8397cd6bf2 Mon Sep 17 00:00:00 2001
From: Carlo Zancanaro <carlo@zancanaro.id.au>
Date: Sun, 31 May 2015 16:48:45 +1000
Subject: Improve lots of things
In particular:
+ make output more sane and less all over the place
+ just ignore original exception, the shrunk one is the only one which really
matters
+ fit into the jUnit framework more (so now @Before, @After and stuff work)
---
.../java/au/id/zancanaro/PropertyTestError.java | 45 +++
.../java/au/id/zancanaro/PropertyTestRunner.java | 323 +++++++++++++--------
src/main/java/au/id/zancanaro/ShrinkResult.java | 11 +
.../java/au/id/zancanaro/annotations/Property.java | 2 +-
4 files changed, 252 insertions(+), 129 deletions(-)
create mode 100644 src/main/java/au/id/zancanaro/PropertyTestError.java
create mode 100644 src/main/java/au/id/zancanaro/ShrinkResult.java
(limited to 'src/main/java')
diff --git a/src/main/java/au/id/zancanaro/PropertyTestError.java b/src/main/java/au/id/zancanaro/PropertyTestError.java
new file mode 100644
index 0000000..afd614b
--- /dev/null
+++ b/src/main/java/au/id/zancanaro/PropertyTestError.java
@@ -0,0 +1,45 @@
+package au.id.zancanaro;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+
+public class PropertyTestError extends AssertionError {
+ public PropertyTestError(String methodName, long seed, ShrinkResult shrunk) {
+ super(String.format("%s(%s)\n\tSeed: %s\n%s",
+ methodName, join(", ", shrunk.args),
+ seed,
+ shrunk.thrown.getMessage()));
+ initCause(shrunk.thrown);
+ }
+
+
+ public static String join(String delimiter, Object... params) {
+ return join(delimiter, Arrays.asList(params));
+ }
+
+ public static String join(String delimiter, Collection<Object> values) {
+ StringBuilder sb = new StringBuilder();
+ Iterator<Object> iter = values.iterator();
+ while (iter.hasNext()) {
+ Object next = iter.next();
+ sb.append(stringValueOf(next));
+ if (iter.hasNext()) {
+ sb.append(delimiter);
+ }
+ }
+ return sb.toString();
+ }
+
+ private static String stringValueOf(Object next) {
+ if (next instanceof String) {
+ return '"' + ((String) next).replace("\"", "\\\"") + '"';
+ } else {
+ try {
+ return String.valueOf(next);
+ } catch (Throwable e) {
+ return "[toString failed]";
+ }
+ }
+ }
+}
diff --git a/src/main/java/au/id/zancanaro/PropertyTestRunner.java b/src/main/java/au/id/zancanaro/PropertyTestRunner.java
index 8df9b2a..b3563ff 100644
--- a/src/main/java/au/id/zancanaro/PropertyTestRunner.java
+++ b/src/main/java/au/id/zancanaro/PropertyTestRunner.java
@@ -3,127 +3,204 @@ package au.id.zancanaro;
import au.id.zancanaro.annotations.Property;
import au.id.zancanaro.annotations.Seed;
import org.junit.AssumptionViolatedException;
-import org.junit.Ignore;
-import org.junit.runner.Description;
-import org.junit.runner.notification.Failure;
-import org.junit.runner.notification.RunNotifier;
-import org.junit.runners.ParentRunner;
+import org.junit.runners.BlockJUnit4ClassRunner;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+import org.junit.runners.model.TestClass;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
+import java.lang.reflect.*;
import java.util.*;
-public class PropertyTestRunner extends ParentRunner<FrameworkMethod> {
- private final Class<?> classUnderTest;
+public class PropertyTestRunner extends BlockJUnit4ClassRunner {
+ private final Map<Type, Generator<Object>> generators = new HashMap<>();
- public PropertyTestRunner(Class<?> classUnderTest) throws InitializationError{
- super(classUnderTest);
- this.classUnderTest = classUnderTest;
+ public PropertyTestRunner(Class<?> klass) throws InitializationError {
+ super(klass);
}
@Override
- protected boolean isIgnored(FrameworkMethod child) {
- return child.getAnnotation(Ignore.class) != null;
+ protected void collectInitializationErrors(List<Throwable> errors) {
+ super.collectInitializationErrors(errors);
+ validateGeneratorFields(errors);
}
- @Override
- protected List<FrameworkMethod> getChildren() {
- List<FrameworkMethod> result = new ArrayList<>();
- for (Method method : classUnderTest.getDeclaredMethods()) {
- if (method.isAnnotationPresent(Property.class)
- && !method.isAnnotationPresent(Ignore.class)) {
- result.add(new FrameworkMethod(method));
+ private void validateGeneratorFields(List<Throwable> errors) {
+ Field[] fields = getTestClass().getJavaClass().getDeclaredFields();
+
+ for (Field field : fields) {
+ Type type = field.getGenericType();
+ if (!(type instanceof ParameterizedType)) {
+ continue;
+ }
+ ParameterizedType ptype = (ParameterizedType) type;
+ if (!(ptype.getRawType() instanceof Class)) {
+ continue;
+ }
+ Class<?> c = (Class) ptype.getRawType();
+ if (c != Generator.class) {
+ continue;
+ }
+ if (!Modifier.isStatic(field.getModifiers())) {
+ errors.add(new Error("Generator field " + field.getName() + " must be static"));
+ }
+ if (!Modifier.isPublic(field.getModifiers())) {
+ errors.add(new Error("Generator field " + field.getName() + " must be public"));
}
}
- return result;
}
- @Override
- protected Description describeChild(FrameworkMethod child) {
- return Description.createTestDescription(classUnderTest, child.getName());
+ private static final Map<Type, Type> rawTypes;
+
+ static {
+ Map<Type, Type> types = new HashMap<>();
+ types.put(Double.class, Double.TYPE);
+ types.put(Float.class, Float.TYPE);
+ types.put(Long.class, Long.TYPE);
+ types.put(Integer.class, Integer.TYPE);
+ types.put(Short.class, Short.TYPE);
+ types.put(Byte.class, Byte.TYPE);
+ types.put(Character.class, Character.TYPE);
+ types.put(Boolean.class, Boolean.TYPE);
+ rawTypes = Collections.unmodifiableMap(types);
}
- @Override
- protected void runChild(FrameworkMethod child, RunNotifier notifier) {
- try {
- Property details = child.getAnnotation(Property.class);
- Description description = Description.createTestDescription(classUnderTest, child.getName());
- boolean failed = false;
- int assumptionsFailed = 0;
-
- long seed = getSeed(child.getMethod());
- Random random = new Random(seed);
- int numRuns = details.runs();
- for (int i = 0; i < numRuns && !failed; ++i) {
- int size = details.size();
- notifier.fireTestStarted(description);
- Object obj;
- try {
- obj = classUnderTest.getConstructor().newInstance();
- } catch (Throwable ex) { // HACKY
- System.out.println(ex);
- return;
+ private Map<Type, Generator<Object>> computeGenerators() {
+ if (generators.isEmpty()) {
+ Field[] fields = getTestClass().getJavaClass().getDeclaredFields();
+
+ for (Field field : fields) {
+ Type type = field.getGenericType();
+ if (!(type instanceof ParameterizedType)) {
+ continue;
+ }
+ ParameterizedType ptype = (ParameterizedType) type;
+ if (!(ptype.getRawType() instanceof Class)) {
+ continue;
+ }
+ Class<?> c = (Class) ptype.getRawType();
+ if (c != Generator.class) {
+ continue;
}
- RoseTree<Object[]> generated = generateArgs(random, size,
- child.getMethod().getGenericParameterTypes(),
- child.getMethod().getParameterAnnotations());
try {
- child.getMethod().invoke(obj, generated.getValue());
- } catch (InvocationTargetException ex) {
- if (ex.getTargetException() instanceof AssumptionViolatedException) {
- assumptionsFailed++;
- i--;
- } else {
- System.out.println("Test failed with seed: " + seed);
- System.out.println("Failing arguments: " + Arrays.asList(generated.getValue()));
- Object[] shrinkResult = shrink(child.getMethod(), obj, generated);
- if (shrinkResult == null) {
- System.out.println("Arguments could not be shrunk any further");
- } else {
- System.out.println("Arguments shrunk to: " + Arrays.asList(shrinkResult));
- }
- notifier.fireTestFailure(new Failure(description, ex.getTargetException()));
- failed = true;
+ Type target = ptype.getActualTypeArguments()[0];
+ @SuppressWarnings("unchecked")
+ Generator<Object> generator = (Generator<Object>) field.get(null);
+ generators.put(target, generator);
+ if (rawTypes.containsKey(target)) {
+ generators.put(rawTypes.get(target), generator);
}
} catch (IllegalAccessException ex) {
- notifier.fireTestFailure(new Failure(description, ex));
- failed = true;
+
}
}
+ }
+ return generators;
+ }
- if (assumptionsFailed > 0) {
- System.out.println("Failed " + assumptionsFailed + " assumptions");
+ @Override
+ protected void validateConstructor(List<Throwable> errors) {
+ validateOnlyOneConstructor(errors);
+ }
+
+ @Override
+ protected void validateTestMethods(List<Throwable> errors) {
+ for (FrameworkMethod each : computeTestMethods()) {
+ if (each.getAnnotation(Property.class) != null) {
+ each.validatePublicVoid(false, errors);
+ each.validateNoTypeParametersOnArgs(errors);
+ } else {
+ each.validatePublicVoidNoArg(false, errors);
}
- notifier.fireTestFinished(description);
- } catch (Throwable ex) {
- ex.printStackTrace();
}
}
- private long getSeed(Method method) {
- Seed seed = method.getAnnotation(Seed.class);
- if (seed == null) {
- return System.currentTimeMillis();
- } else {
- return seed.value();
- }
+ @Override
+ protected List<FrameworkMethod> computeTestMethods() {
+ List<FrameworkMethod> testMethods = new ArrayList<>(super.computeTestMethods());
+ List<FrameworkMethod> theoryMethods = getTestClass().getAnnotatedMethods(Property.class);
+ testMethods.removeAll(theoryMethods);
+ testMethods.addAll(theoryMethods);
+ return testMethods;
}
- private Object[] shrink(Method method, Object obj, RoseTree<Object[]> failed) {
- Object[] smallest = failed.getValue();
- Iterator<RoseTree<Object[]>> trees = failed.getChildren();
- while (trees.hasNext()) {
- RoseTree<Object[]> tree = trees.next();
- try {
- method.invoke(obj, tree.getValue());
- } catch (InvocationTargetException ex) {
- if (!(ex.getTargetException() instanceof AssumptionViolatedException)) {
- smallest = tree.getValue();
+ @Override
+ public Statement methodBlock(final FrameworkMethod method) {
+ return new GenerativeTester(method, getTestClass(), computeGenerators());
+ }
+
+ public static class GenerativeTester extends Statement {
+ private final FrameworkMethod testMethod;
+ private final TestClass testClass;
+ private final Map<Type, Generator<Object>> generators;
+
+ public GenerativeTester(FrameworkMethod testMethod, TestClass testClass, Map<Type, Generator<Object>> generators) {
+ this.testMethod = testMethod;
+ this.testClass = testClass;
+ this.generators = generators;
+ }
+
+ private long getSeed(Method method) {
+ Seed seed = method.getAnnotation(Seed.class);
+ if (seed == null) {
+ return System.currentTimeMillis();
+ } else {
+ return seed.value();
+ }
+ }
+
+ @Override
+ public void evaluate() throws Throwable {
+ Method method = testMethod.getMethod();
+ if (method.getParameterCount() == 0) {
+ runTest(new Object[0]);
+ } else {
+ @SuppressWarnings("unchecked")
+ Generator<Object>[] generators = (Generator<Object>[]) new Generator[method.getParameterCount()];
+ int index = 0;
+ for (Type type : method.getGenericParameterTypes()) {
+ generators[index++] = this.generators.get(type);
+ }
+ Generator<Object[]> generator = Generator.tuple(generators);
+
+ long seed = getSeed(method);
+ Random random = new Random(seed);
+
+ Property property = testMethod.getAnnotation(Property.class);
+ int assumptionsViolated = 0;
+ int maxSize = property.maxSize();
+ int numTests = property.runs();
+ for (int i = 0; i < numTests; ++i) {
+ int size = Math.min(i + 1, maxSize);
+ RoseTree<Object[]> tree = generator.generate(random, size);
+ try {
+ runTest(tree.getValue());
+ assumptionsViolated = 0;
+ } catch (AssumptionViolatedException ex) {
+ i--;
+ if (assumptionsViolated++ == 50) {
+ throw new Error("Violated 50 assumptions in a row: failing test");
+ }
+ ;
+ } catch (Throwable ex) {
+ throw new PropertyTestError(method.getName(), seed, shrink(tree, ex));
+ }
+ }
+ }
+ }
+
+ private ShrinkResult shrink(RoseTree<Object[]> failed, Throwable originalEx) {
+ ShrinkResult smallest = new ShrinkResult(failed.getValue(), originalEx);
+ Iterator<RoseTree<Object[]>> trees = failed.getChildren();
+ while (trees.hasNext()) {
+ RoseTree<Object[]> tree = trees.next();
+ try {
+ runTest(tree.getValue());
+ } catch (AssumptionViolatedException ex) {
+ // ignore, because it's not useful
+ } catch (Throwable ex) {
+ smallest = new ShrinkResult(tree.getValue(), ex);
Iterator<RoseTree<Object[]>> children = tree.getChildren();
if (children.hasNext()) {
trees = children;
@@ -131,49 +208,39 @@ public class PropertyTestRunner extends ParentRunner<FrameworkMethod> {
break;
}
}
- } catch (IllegalAccessException ex) {
- System.out.println(ex);
}
+ return smallest;
}
- return smallest;
- }
- private <T> String printShrinkTree(RoseTree<T[]> generated) {
- StringBuilder builder = new StringBuilder();
- builder.append('(');
- builder.append(Arrays.toString(generated.getValue()));
- generated.getChildren().forEachRemaining((child) -> {
- builder.append(' ');
- builder.append(printShrinkTree(child));
- });
- builder.append(')');
- return builder.toString();
- }
+ public void runTest(final Object[] args) throws Throwable {
+ new BlockJUnit4ClassRunner(testClass.getJavaClass()) {
+ @Override
+ protected void collectInitializationErrors(
+ List<Throwable> errors) {
+ // do nothing
+ }
+ @Override
+ public Statement methodBlock(FrameworkMethod method) {
+ return super.methodBlock(method);
+ }
- private RoseTree<Object[]> generateArgs(Random random, int size, Type[] types, Annotation[][] annotations) {
- Generator<?>[] generators = new Generator[types.length];
- for (int i = 0; i < types.length; ++i) {
-// generators[i] = getGeneratorFromAnnotations(annotations[i]);
-// if (generators[i] == null) {
- generators[i] = getGeneratorFromType(types[i]);
-// }
- }
- @SuppressWarnings("unchecked")
- Generator<Object>[] argsGenerators = (Generator<Object>[]) generators;
- return Generator.tuple(argsGenerators).generate(random, size);
- }
+ @Override
+ protected Statement methodInvoker(FrameworkMethod method, Object test) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ method.invokeExplosively(test, args);
+ }
+ };
+ }
- private Generator<?> getGeneratorFromType(Type type) {
- if (type instanceof Class) {
- Class<?> clazz = (Class<?>) type;
- if (clazz.isPrimitive() && clazz == Integer.TYPE) {
- return Generators.integer();
- } else {
- throw new RuntimeException("Unknown type for generator (atm only int is supported)");
- }
- } else {
- throw new RuntimeException("Unknown type for generator (atm only int is supported)");
+ @Override
+ public Object createTest() throws Exception {
+ return getTestClass().getOnlyConstructor().newInstance();
+ }
+ }.methodBlock(testMethod).evaluate();
}
}
-}
+
+}
\ No newline at end of file
diff --git a/src/main/java/au/id/zancanaro/ShrinkResult.java b/src/main/java/au/id/zancanaro/ShrinkResult.java
new file mode 100644
index 0000000..fb2f2d4
--- /dev/null
+++ b/src/main/java/au/id/zancanaro/ShrinkResult.java
@@ -0,0 +1,11 @@
+package au.id.zancanaro;
+
+class ShrinkResult {
+ public final Object[] args;
+ public final Throwable thrown;
+
+ public ShrinkResult(Object[] args, Throwable thrown) {
+ this.args = args;
+ this.thrown = thrown;
+ }
+}
diff --git a/src/main/java/au/id/zancanaro/annotations/Property.java b/src/main/java/au/id/zancanaro/annotations/Property.java
index cfe8e45..f750596 100644
--- a/src/main/java/au/id/zancanaro/annotations/Property.java
+++ b/src/main/java/au/id/zancanaro/annotations/Property.java
@@ -8,6 +8,6 @@ import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Property {
- int size() default 100;
+ int maxSize() default 100;
int runs() default 100;
}
--
cgit v1.2.3