From 199037f9c80afd885f1f536d91b40a8397cd6bf2 Mon Sep 17 00:00:00 2001 From: Carlo Zancanaro Date: Sun, 31 May 2015 16:48:45 +1000 Subject: Improve lots of things In particular: + make output more sane and less all over the place + just ignore original exception, the shrunk one is the only one which really matters + fit into the jUnit framework more (so now @Before, @After and stuff work) --- .../java/au/id/zancanaro/PropertyTestError.java | 45 +++ .../java/au/id/zancanaro/PropertyTestRunner.java | 323 +++++++++++++-------- src/main/java/au/id/zancanaro/ShrinkResult.java | 11 + .../java/au/id/zancanaro/annotations/Property.java | 2 +- src/test/java/au/id/zancanaro/PropertyTests.java | 17 +- src/test/java/au/id/zancanaro/TheoriesTest.java | 22 ++ 6 files changed, 286 insertions(+), 134 deletions(-) create mode 100644 src/main/java/au/id/zancanaro/PropertyTestError.java create mode 100644 src/main/java/au/id/zancanaro/ShrinkResult.java create mode 100644 src/test/java/au/id/zancanaro/TheoriesTest.java (limited to 'src') diff --git a/src/main/java/au/id/zancanaro/PropertyTestError.java b/src/main/java/au/id/zancanaro/PropertyTestError.java new file mode 100644 index 0000000..afd614b --- /dev/null +++ b/src/main/java/au/id/zancanaro/PropertyTestError.java @@ -0,0 +1,45 @@ +package au.id.zancanaro; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; + +public class PropertyTestError extends AssertionError { + public PropertyTestError(String methodName, long seed, ShrinkResult shrunk) { + super(String.format("%s(%s)\n\tSeed: %s\n%s", + methodName, join(", ", shrunk.args), + seed, + shrunk.thrown.getMessage())); + initCause(shrunk.thrown); + } + + + public static String join(String delimiter, Object... params) { + return join(delimiter, Arrays.asList(params)); + } + + public static String join(String delimiter, Collection values) { + StringBuilder sb = new StringBuilder(); + Iterator iter = values.iterator(); + while (iter.hasNext()) { + Object next = iter.next(); + sb.append(stringValueOf(next)); + if (iter.hasNext()) { + sb.append(delimiter); + } + } + return sb.toString(); + } + + private static String stringValueOf(Object next) { + if (next instanceof String) { + return '"' + ((String) next).replace("\"", "\\\"") + '"'; + } else { + try { + return String.valueOf(next); + } catch (Throwable e) { + return "[toString failed]"; + } + } + } +} diff --git a/src/main/java/au/id/zancanaro/PropertyTestRunner.java b/src/main/java/au/id/zancanaro/PropertyTestRunner.java index 8df9b2a..b3563ff 100644 --- a/src/main/java/au/id/zancanaro/PropertyTestRunner.java +++ b/src/main/java/au/id/zancanaro/PropertyTestRunner.java @@ -3,127 +3,204 @@ package au.id.zancanaro; import au.id.zancanaro.annotations.Property; import au.id.zancanaro.annotations.Seed; import org.junit.AssumptionViolatedException; -import org.junit.Ignore; -import org.junit.runner.Description; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.ParentRunner; +import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; +import org.junit.runners.model.TestClass; -import java.lang.annotation.Annotation; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; +import java.lang.reflect.*; import java.util.*; -public class PropertyTestRunner extends ParentRunner { - private final Class classUnderTest; +public class PropertyTestRunner extends BlockJUnit4ClassRunner { + private final Map> generators = new HashMap<>(); - public PropertyTestRunner(Class classUnderTest) throws InitializationError{ - super(classUnderTest); - this.classUnderTest = classUnderTest; + public PropertyTestRunner(Class klass) throws InitializationError { + super(klass); } @Override - protected boolean isIgnored(FrameworkMethod child) { - return child.getAnnotation(Ignore.class) != null; + protected void collectInitializationErrors(List errors) { + super.collectInitializationErrors(errors); + validateGeneratorFields(errors); } - @Override - protected List getChildren() { - List result = new ArrayList<>(); - for (Method method : classUnderTest.getDeclaredMethods()) { - if (method.isAnnotationPresent(Property.class) - && !method.isAnnotationPresent(Ignore.class)) { - result.add(new FrameworkMethod(method)); + private void validateGeneratorFields(List errors) { + Field[] fields = getTestClass().getJavaClass().getDeclaredFields(); + + for (Field field : fields) { + Type type = field.getGenericType(); + if (!(type instanceof ParameterizedType)) { + continue; + } + ParameterizedType ptype = (ParameterizedType) type; + if (!(ptype.getRawType() instanceof Class)) { + continue; + } + Class c = (Class) ptype.getRawType(); + if (c != Generator.class) { + continue; + } + if (!Modifier.isStatic(field.getModifiers())) { + errors.add(new Error("Generator field " + field.getName() + " must be static")); + } + if (!Modifier.isPublic(field.getModifiers())) { + errors.add(new Error("Generator field " + field.getName() + " must be public")); } } - return result; } - @Override - protected Description describeChild(FrameworkMethod child) { - return Description.createTestDescription(classUnderTest, child.getName()); + private static final Map rawTypes; + + static { + Map types = new HashMap<>(); + types.put(Double.class, Double.TYPE); + types.put(Float.class, Float.TYPE); + types.put(Long.class, Long.TYPE); + types.put(Integer.class, Integer.TYPE); + types.put(Short.class, Short.TYPE); + types.put(Byte.class, Byte.TYPE); + types.put(Character.class, Character.TYPE); + types.put(Boolean.class, Boolean.TYPE); + rawTypes = Collections.unmodifiableMap(types); } - @Override - protected void runChild(FrameworkMethod child, RunNotifier notifier) { - try { - Property details = child.getAnnotation(Property.class); - Description description = Description.createTestDescription(classUnderTest, child.getName()); - boolean failed = false; - int assumptionsFailed = 0; - - long seed = getSeed(child.getMethod()); - Random random = new Random(seed); - int numRuns = details.runs(); - for (int i = 0; i < numRuns && !failed; ++i) { - int size = details.size(); - notifier.fireTestStarted(description); - Object obj; - try { - obj = classUnderTest.getConstructor().newInstance(); - } catch (Throwable ex) { // HACKY - System.out.println(ex); - return; + private Map> computeGenerators() { + if (generators.isEmpty()) { + Field[] fields = getTestClass().getJavaClass().getDeclaredFields(); + + for (Field field : fields) { + Type type = field.getGenericType(); + if (!(type instanceof ParameterizedType)) { + continue; + } + ParameterizedType ptype = (ParameterizedType) type; + if (!(ptype.getRawType() instanceof Class)) { + continue; + } + Class c = (Class) ptype.getRawType(); + if (c != Generator.class) { + continue; } - RoseTree generated = generateArgs(random, size, - child.getMethod().getGenericParameterTypes(), - child.getMethod().getParameterAnnotations()); try { - child.getMethod().invoke(obj, generated.getValue()); - } catch (InvocationTargetException ex) { - if (ex.getTargetException() instanceof AssumptionViolatedException) { - assumptionsFailed++; - i--; - } else { - System.out.println("Test failed with seed: " + seed); - System.out.println("Failing arguments: " + Arrays.asList(generated.getValue())); - Object[] shrinkResult = shrink(child.getMethod(), obj, generated); - if (shrinkResult == null) { - System.out.println("Arguments could not be shrunk any further"); - } else { - System.out.println("Arguments shrunk to: " + Arrays.asList(shrinkResult)); - } - notifier.fireTestFailure(new Failure(description, ex.getTargetException())); - failed = true; + Type target = ptype.getActualTypeArguments()[0]; + @SuppressWarnings("unchecked") + Generator generator = (Generator) field.get(null); + generators.put(target, generator); + if (rawTypes.containsKey(target)) { + generators.put(rawTypes.get(target), generator); } } catch (IllegalAccessException ex) { - notifier.fireTestFailure(new Failure(description, ex)); - failed = true; + } } + } + return generators; + } - if (assumptionsFailed > 0) { - System.out.println("Failed " + assumptionsFailed + " assumptions"); + @Override + protected void validateConstructor(List errors) { + validateOnlyOneConstructor(errors); + } + + @Override + protected void validateTestMethods(List errors) { + for (FrameworkMethod each : computeTestMethods()) { + if (each.getAnnotation(Property.class) != null) { + each.validatePublicVoid(false, errors); + each.validateNoTypeParametersOnArgs(errors); + } else { + each.validatePublicVoidNoArg(false, errors); } - notifier.fireTestFinished(description); - } catch (Throwable ex) { - ex.printStackTrace(); } } - private long getSeed(Method method) { - Seed seed = method.getAnnotation(Seed.class); - if (seed == null) { - return System.currentTimeMillis(); - } else { - return seed.value(); - } + @Override + protected List computeTestMethods() { + List testMethods = new ArrayList<>(super.computeTestMethods()); + List theoryMethods = getTestClass().getAnnotatedMethods(Property.class); + testMethods.removeAll(theoryMethods); + testMethods.addAll(theoryMethods); + return testMethods; } - private Object[] shrink(Method method, Object obj, RoseTree failed) { - Object[] smallest = failed.getValue(); - Iterator> trees = failed.getChildren(); - while (trees.hasNext()) { - RoseTree tree = trees.next(); - try { - method.invoke(obj, tree.getValue()); - } catch (InvocationTargetException ex) { - if (!(ex.getTargetException() instanceof AssumptionViolatedException)) { - smallest = tree.getValue(); + @Override + public Statement methodBlock(final FrameworkMethod method) { + return new GenerativeTester(method, getTestClass(), computeGenerators()); + } + + public static class GenerativeTester extends Statement { + private final FrameworkMethod testMethod; + private final TestClass testClass; + private final Map> generators; + + public GenerativeTester(FrameworkMethod testMethod, TestClass testClass, Map> generators) { + this.testMethod = testMethod; + this.testClass = testClass; + this.generators = generators; + } + + private long getSeed(Method method) { + Seed seed = method.getAnnotation(Seed.class); + if (seed == null) { + return System.currentTimeMillis(); + } else { + return seed.value(); + } + } + + @Override + public void evaluate() throws Throwable { + Method method = testMethod.getMethod(); + if (method.getParameterCount() == 0) { + runTest(new Object[0]); + } else { + @SuppressWarnings("unchecked") + Generator[] generators = (Generator[]) new Generator[method.getParameterCount()]; + int index = 0; + for (Type type : method.getGenericParameterTypes()) { + generators[index++] = this.generators.get(type); + } + Generator generator = Generator.tuple(generators); + + long seed = getSeed(method); + Random random = new Random(seed); + + Property property = testMethod.getAnnotation(Property.class); + int assumptionsViolated = 0; + int maxSize = property.maxSize(); + int numTests = property.runs(); + for (int i = 0; i < numTests; ++i) { + int size = Math.min(i + 1, maxSize); + RoseTree tree = generator.generate(random, size); + try { + runTest(tree.getValue()); + assumptionsViolated = 0; + } catch (AssumptionViolatedException ex) { + i--; + if (assumptionsViolated++ == 50) { + throw new Error("Violated 50 assumptions in a row: failing test"); + } + ; + } catch (Throwable ex) { + throw new PropertyTestError(method.getName(), seed, shrink(tree, ex)); + } + } + } + } + + private ShrinkResult shrink(RoseTree failed, Throwable originalEx) { + ShrinkResult smallest = new ShrinkResult(failed.getValue(), originalEx); + Iterator> trees = failed.getChildren(); + while (trees.hasNext()) { + RoseTree tree = trees.next(); + try { + runTest(tree.getValue()); + } catch (AssumptionViolatedException ex) { + // ignore, because it's not useful + } catch (Throwable ex) { + smallest = new ShrinkResult(tree.getValue(), ex); Iterator> children = tree.getChildren(); if (children.hasNext()) { trees = children; @@ -131,49 +208,39 @@ public class PropertyTestRunner extends ParentRunner { break; } } - } catch (IllegalAccessException ex) { - System.out.println(ex); } + return smallest; } - return smallest; - } - private String printShrinkTree(RoseTree generated) { - StringBuilder builder = new StringBuilder(); - builder.append('('); - builder.append(Arrays.toString(generated.getValue())); - generated.getChildren().forEachRemaining((child) -> { - builder.append(' '); - builder.append(printShrinkTree(child)); - }); - builder.append(')'); - return builder.toString(); - } + public void runTest(final Object[] args) throws Throwable { + new BlockJUnit4ClassRunner(testClass.getJavaClass()) { + @Override + protected void collectInitializationErrors( + List errors) { + // do nothing + } + @Override + public Statement methodBlock(FrameworkMethod method) { + return super.methodBlock(method); + } - private RoseTree generateArgs(Random random, int size, Type[] types, Annotation[][] annotations) { - Generator[] generators = new Generator[types.length]; - for (int i = 0; i < types.length; ++i) { -// generators[i] = getGeneratorFromAnnotations(annotations[i]); -// if (generators[i] == null) { - generators[i] = getGeneratorFromType(types[i]); -// } - } - @SuppressWarnings("unchecked") - Generator[] argsGenerators = (Generator[]) generators; - return Generator.tuple(argsGenerators).generate(random, size); - } + @Override + protected Statement methodInvoker(FrameworkMethod method, Object test) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + method.invokeExplosively(test, args); + } + }; + } - private Generator getGeneratorFromType(Type type) { - if (type instanceof Class) { - Class clazz = (Class) type; - if (clazz.isPrimitive() && clazz == Integer.TYPE) { - return Generators.integer(); - } else { - throw new RuntimeException("Unknown type for generator (atm only int is supported)"); - } - } else { - throw new RuntimeException("Unknown type for generator (atm only int is supported)"); + @Override + public Object createTest() throws Exception { + return getTestClass().getOnlyConstructor().newInstance(); + } + }.methodBlock(testMethod).evaluate(); } } -} + +} \ No newline at end of file diff --git a/src/main/java/au/id/zancanaro/ShrinkResult.java b/src/main/java/au/id/zancanaro/ShrinkResult.java new file mode 100644 index 0000000..fb2f2d4 --- /dev/null +++ b/src/main/java/au/id/zancanaro/ShrinkResult.java @@ -0,0 +1,11 @@ +package au.id.zancanaro; + +class ShrinkResult { + public final Object[] args; + public final Throwable thrown; + + public ShrinkResult(Object[] args, Throwable thrown) { + this.args = args; + this.thrown = thrown; + } +} diff --git a/src/main/java/au/id/zancanaro/annotations/Property.java b/src/main/java/au/id/zancanaro/annotations/Property.java index cfe8e45..f750596 100644 --- a/src/main/java/au/id/zancanaro/annotations/Property.java +++ b/src/main/java/au/id/zancanaro/annotations/Property.java @@ -8,6 +8,6 @@ import java.lang.annotation.Target; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) public @interface Property { - int size() default 100; + int maxSize() default 100; int runs() default 100; } diff --git a/src/test/java/au/id/zancanaro/PropertyTests.java b/src/test/java/au/id/zancanaro/PropertyTests.java index 3620e67..374a974 100644 --- a/src/test/java/au/id/zancanaro/PropertyTests.java +++ b/src/test/java/au/id/zancanaro/PropertyTests.java @@ -1,13 +1,20 @@ package au.id.zancanaro; import au.id.zancanaro.annotations.Property; -import org.junit.Assert; -import org.junit.Assume; +import org.junit.*; import org.junit.experimental.theories.DataPoint; import org.junit.runner.RunWith; @RunWith(PropertyTestRunner.class) public class PropertyTests { + public static Generator gen = Generators.integer(); + + @Test + public void testblah() throws Exception { + Assert.assertEquals(10, 11); + + } + @Property public void aIsNotOdd(int a, int b) { Assume.assumeFalse(a % 2 == 1); @@ -21,8 +28,8 @@ public class PropertyTests { } @Property - public void aIsNotGreaterThanB(int a, int b) { -// Assume.assumeFalse(a > b); - Assert.assertFalse(a > b); + public void aPlusBLessThanOneHundred(int a, int b) { + Assert.assertTrue("a plus b is five", a + b == 5); + Assert.assertEquals(a + b, 5); } } diff --git a/src/test/java/au/id/zancanaro/TheoriesTest.java b/src/test/java/au/id/zancanaro/TheoriesTest.java new file mode 100644 index 0000000..7d5071c --- /dev/null +++ b/src/test/java/au/id/zancanaro/TheoriesTest.java @@ -0,0 +1,22 @@ +package au.id.zancanaro; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.theories.DataPoint; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; + +@RunWith(Theories.class) +public class TheoriesTest { + @DataPoint + public static String car = "carlo"; + + @DataPoint + public static String car2 = "carlo2"; + + @Theory + public void testName(String str) throws Exception { + Assert.assertEquals(str, car2); + } +} -- cgit v1.2.3