Unified benchmarks

vavr-io · May 23, 2016 · 21c901b · 21c901b
1 parent 930163f
commit 21c901b
Show file tree

Hide file tree

Showing 5 changed files with 127 additions and 110 deletions.
diff --git a/javaslang-benchmark/src/test/java/javaslang/benchmark/BenchmarkPerformanceReporter.java b/javaslang-benchmark/src/test/java/javaslang/benchmark/BenchmarkPerformanceReporter.java
@@ -224,11 +224,11 @@ private String calculateRatioStr(TestExecution baseResult, Option<TestExecution>
             if (!alternativeResult.isDefined()) {
                 return "";
             }
-            double alternativeScore = alternativeResult.get().getScore();
+            final double alternativeScore = alternativeResult.get().getScore();
             if (alternativeScore == 0.0) {
                 return "";
             }
-            double ratio = baseResult.getScore() / alternativeScore;
+            final double ratio = baseResult.getScore() / alternativeScore;
             return ratio == 1.0 ? "" : PERFORMANCE_FORMAT.format(ratio) + "x";
         }
     }
@@ -390,14 +390,14 @@ public static TestExecution of(BenchmarkResult benchmarkResult, double outlierLo
         }
 
         public TestExecution(BenchmarkResult benchmark) {
-            Result primaryResult = benchmark.getPrimaryResult();
+            final Result<?> primaryResult = benchmark.getPrimaryResult();
             fullName = benchmark.getParams().getBenchmark();
             target = extractPart(fullName, 2);
             operation = extractPart(fullName, 1);
             implementation = extractPart(fullName, 0);
             paramKey = getParameterKey(benchmark);
 
-            ListStatistics statistics = createStatisticsWithoutOutliers(benchmark, outlierLowPct, outlierHighPct);
+            final ListStatistics statistics = createStatisticsWithoutOutliers(benchmark, outlierLowPct, outlierHighPct);
             sampleCount = statistics.getN();
             score = statistics.getMean();
             scoreError = statistics.getMeanErrorAt(0.999);
@@ -409,15 +409,15 @@ private ListStatistics createStatisticsWithoutOutliers(BenchmarkResult benchmark
                     .map(r -> r.getPrimaryResult().getScore())
                     .sorted()
                     .collect(Vector.collector());
-            int size = results.size();
-            int outliersLow = (int) (size * outlierLowPct);
-            int outliersHigh = (int) (size * outlierHighPct);
+            final int size = results.size();
+            final int outliersLow = (int) (size * outlierLowPct);
+            final int outliersHigh = (int) (size * outlierHighPct);
             results = results.drop(outliersLow).dropRight(outliersHigh);
             return new ListStatistics(results.toJavaList().stream().mapToDouble(r -> r).toArray());
         }
 
         private String getParameterKey(BenchmarkResult benchmarkResult) {
-            BenchmarkParams params = benchmarkResult.getParams();
+            final BenchmarkParams params = benchmarkResult.getParams();
             return params.getParamsKeys().stream().map(params::getParam).collect(Collectors.joining(","));
         }
 

diff --git a/javaslang-benchmark/src/test/java/javaslang/benchmark/JmhRunner.java b/javaslang-benchmark/src/test/java/javaslang/benchmark/JmhRunner.java
@@ -3,70 +3,95 @@
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
-import org.openjdk.jmh.runner.options.TimeValue;
+import org.openjdk.jmh.runner.options.*;
 
-import java.util.Collection;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 public class JmhRunner {
-    private static final int WARMUP_ITERATIONS = 10;
-    private static final int MEASUREMENT_ITERATIONS = 40;
+    private static final int WARMUP_ITERATIONS = 20;
+    private static final int MEASUREMENT_ITERATIONS = 30;
 
-    private static final int QUICK_WARMUP_ITERATIONS = 5;
+    private static final int QUICK_WARMUP_ITERATIONS = 10;
     private static final int QUICK_MEASUREMENT_ITERATIONS = 10;
 
     public static void run(Class<?> benchmarkClass) {
-        runAndReport(benchmarkClass, WARMUP_ITERATIONS, MEASUREMENT_ITERATIONS, Assertions.Disable);
+        runAndReport(benchmarkClass, WARMUP_ITERATIONS, MEASUREMENT_ITERATIONS, 500, PrintGc.Enable, Assertions.Disable);
     }
 
     public static void devRun(Class<?> benchmarkClass) {
-        runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, Assertions.Disable);
+        runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, 200, PrintGc.Disable, Assertions.Disable);
     }
 
     public static void devRunWithAssertions(Class<?> benchmarkClass) {
-        runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, Assertions.Enable);
+        runAndReport(benchmarkClass, QUICK_WARMUP_ITERATIONS, QUICK_MEASUREMENT_ITERATIONS, 200, PrintGc.Disable, Assertions.Enable);
     }
 
-    private static void runAndReport(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, Assertions assertions) {
-        Collection<RunResult> results = run(benchmarkClass, warmupIterations, measurementIterations, assertions);
+    private static void runAndReport(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, int millis, PrintGc printGc, Assertions assertions) {
+        final Collection<RunResult> results = run(benchmarkClass, warmupIterations, measurementIterations, millis, printGc, assertions);
         BenchmarkPerformanceReporter.of(results).print();
     }
 
-    private static Collection<RunResult> run(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, Assertions assertions) {
+    private static Collection<RunResult> run(Class<?> benchmarkClass, int warmupIterations, int measurementIterations, int millis, PrintGc printGc, Assertions assertions) {
         final Options opts = new OptionsBuilder()
                 .include(benchmarkClass.getSimpleName())
                 .shouldDoGC(true)
                 .shouldFailOnError(true)
                 .mode(Mode.Throughput)
                 .timeUnit(TimeUnit.SECONDS)
-                .warmupTime(TimeValue.milliseconds(500))
+                .warmupTime(TimeValue.milliseconds(millis))
                 .warmupIterations(warmupIterations)
-                .measurementTime(TimeValue.milliseconds(500))
+                .measurementTime(TimeValue.milliseconds(millis))
                 .measurementIterations(measurementIterations)
                 .forks(1)
                 // We are using 4Gb and setting NewGen to 100% to avoid GC during testing.
                 // Any GC during testing will destroy the iteration, which should get ignored as an outlier
-                .jvmArgsAppend("-XX:+UseG1GC", "-Xss100m", "-Xms4g", "-Xmx4g", "-XX:+PrintGC", "-XX:MaxGCPauseMillis=1000", "-XX:+UnlockExperimentalVMOptions", "-XX:G1NewSizePercent=100", "-XX:G1MaxNewSizePercent=100", assertions.vmArg)
+                .jvmArgsAppend("-XX:+UseG1GC", "-Xss100m", "-Xms4g", "-Xmx4g", "-XX:MaxGCPauseMillis=1000", "-XX:+UnlockExperimentalVMOptions", "-XX:G1NewSizePercent=100", "-XX:G1MaxNewSizePercent=100", printGc.vmArg, assertions.vmArg)
                 .build();
 
         try {
             return new Runner(opts).run();
-        }
-        catch (Exception e) {
+        } catch (Exception e) {
             throw new RuntimeException(e.getMessage(), e);
         }
     }
 
-    private static enum Assertions {
+    private enum Assertions {
         Enable("-enableassertions"),
-        Disable("-disableassertions")
-        ;
+        Disable("-disableassertions");
 
         final String vmArg;
+
         Assertions(String vmArg) {
             this.vmArg = vmArg;
         }
     }
+
+    private enum PrintGc {
+        Enable("-XX:+PrintGC"),
+        Disable("-XX:-PrintGC");
+
+        final String vmArg;
+
+        PrintGc(String vmArg) {
+            this.vmArg = vmArg;
+        }
+    }
+
+    public static <T> void assertEquals(T a, T b) {
+        if (!Objects.equals(a, b)) {
+            throw new IllegalStateException(a + " != " + b);
+        }
+    }
+
+    public static Integer[] getRandomValues(int size, int seed) {
+        final Random random = new Random(seed);
+
+        final Integer[] results = new Integer[size];
+        for (int i = 0; i < size; i++) {
+            final int value = random.nextInt(size) - (size / 2);
+            results[i] = value;
+        }
+        return results;
+    }
 }
diff --git a/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ArrayBenchmark.java b/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ArrayBenchmark.java
@@ -1,45 +1,25 @@
 package javaslang.benchmark.collection;
 
-import javaslang.benchmark.JmhRunner;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-
-import java.util.Objects;
-import java.util.Random;
+import org.openjdk.jmh.annotations.*;
+
+import static javaslang.benchmark.JmhRunner.*;
 
 public class ArrayBenchmark {
 
     public static void main(String... args) { /* main is more reliable than a test */
-        JmhRunner.run(ArrayBenchmark.class);
+        run(ArrayBenchmark.class);
     }
 
     @State(Scope.Benchmark)
     public static class Base {
-        @Param({ "10", "100", "1000", "10000"})
+        @Param({ "10", "100", "1000", "10000" })
         public int CONTAINER_SIZE;
 
         public Integer[] ELEMENTS;
 
         @Setup
         public void setup() {
-            final Random random = new Random(0);
-
-            ELEMENTS = new Integer[CONTAINER_SIZE];
-            for (int i = 0; i < CONTAINER_SIZE; i++) {
-                final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
-                ELEMENTS[i] = value;
-            }
-        }
-
-        protected static <T> void assertEquals(T a, T b) {
-            if (!Objects.equals(a, b)) {
-                throw new IllegalStateException(a + " != " + b);
-            }
+            ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
         }
     }
 

diff --git a/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ListBenchmark.java b/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/ListBenchmark.java
@@ -1,47 +1,28 @@
 package javaslang.benchmark.collection;
 
-import javaslang.benchmark.JmhRunner;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Objects;
-import java.util.Random;
+import org.openjdk.jmh.annotations.*;
+import scala.compat.java8.JFunction;
 
-public class ListBenchmark {
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static javaslang.benchmark.JmhRunner.*;
 
+public class ListBenchmark {
     public static void main(String... args) { /* main is more reliable than a test */
-        JmhRunner.run(ListBenchmark.class);
+        run(ListBenchmark.class);
     }
 
     @State(Scope.Benchmark)
     public static class Base {
-        @Param({ "10", "100", "1000", "10000"})
+        @Param({ "10", "100", "1000" })
         public int CONTAINER_SIZE;
 
         public Integer[] ELEMENTS;
 
         @Setup
         public void setup() {
-            final Random random = new Random(0);
-
-            ELEMENTS = new Integer[CONTAINER_SIZE];
-            for (int i = 0; i < CONTAINER_SIZE; i++) {
-                final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
-                ELEMENTS[i] = value;
-            }
-        }
-
-        protected static <T> void assertEquals(T a, T b) {
-            if (!Objects.equals(a, b)) {
-                throw new IllegalStateException(a + " != " + b);
-            }
+            ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
         }
     }
 
@@ -240,4 +221,52 @@ public void slang_persistent(Initialized state) {
         }
     }
 
+    public static class GroupBy extends Base {
+        @State(Scope.Thread)
+        public static class Initialized {
+            final java.util.ArrayList<Integer> javaMutable = new java.util.ArrayList<>();
+            scala.collection.immutable.List<Integer> scalaPersistent = scala.collection.immutable.List$.MODULE$.empty();
+            fj.data.List<Integer> fjavaPersistent = fj.data.List.list();
+            javaslang.collection.List<Integer> slangPersistent = javaslang.collection.List.empty();
+
+            @Setup
+            public void initializeMutable(Base state) {
+                assertEquals(javaMutable.size(), 0);
+                Collections.addAll(javaMutable, state.ELEMENTS);
+                assertEquals(javaMutable.size(), state.CONTAINER_SIZE);
+
+                assertEquals(fjavaPersistent.length(), 0);
+                assertEquals(scalaPersistent.size(), 0);
+                assertEquals(slangPersistent.size(), 0);
+                for (Integer element : state.ELEMENTS) {
+                    fjavaPersistent = fjavaPersistent.cons(element);
+                    scalaPersistent = scalaPersistent.$colon$colon(element);
+                    slangPersistent = slangPersistent.prepend(element);
+                }
+                assertEquals(fjavaPersistent.length(), state.CONTAINER_SIZE);
+                assertEquals(scalaPersistent.size(), state.CONTAINER_SIZE);
+                assertEquals(slangPersistent.size(), state.CONTAINER_SIZE);
+            }
+        }
+
+        @Benchmark
+        public Object java_mutable(Initialized state) {
+            return state.javaMutable.stream().collect(Collectors.groupingBy(Integer::bitCount));
+        }
+
+        @Benchmark
+        public Object scala_persistent(Initialized state) {
+            return state.scalaPersistent.groupBy(JFunction.func(Integer::bitCount));
+        }
+
+        @Benchmark
+        public Object fjava_persistent(Initialized state) {
+            return state.fjavaPersistent.groupBy(Integer::bitCount);
+        }
+
+        @Benchmark
+        public Object slang_persistent(Initialized state) {
+            return state.slangPersistent.groupBy(Integer::bitCount);
+        }
+    }
 }
diff --git a/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/PriorityQueueBenchmark.java b/javaslang-benchmark/src/test/java/javaslang/benchmark/collection/PriorityQueueBenchmark.java
@@ -1,57 +1,40 @@
 package javaslang.benchmark.collection;
 
 import javaslang.Tuple2;
-import javaslang.benchmark.JmhRunner;
 import javaslang.collection.Traversable;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import scala.math.Ordering;
 import scala.math.Ordering$;
-import scalaz.Heap;
-import scalaz.Order;
-import scalaz.Order$;
+import scalaz.*;
 
 import java.util.Collections;
-import java.util.Objects;
-import java.util.Random;
+
+import static javaslang.benchmark.JmhRunner.*;
 
 public class PriorityQueueBenchmark {
 
     public static void main(String... args) { /* main is more reliable than a test */
-        JmhRunner.run(PriorityQueueBenchmark.class);
+        run(PriorityQueueBenchmark.class);
     }
 
     @State(Scope.Benchmark)
     public static class Base {
         protected static final Ordering<Integer> SCALA_ORDERING = Ordering$.MODULE$.comparatorToOrdering(Integer::compareTo);
         protected static final Order<Integer> SCALAZ_ORDER = Order$.MODULE$.fromScalaOrdering(SCALA_ORDERING);
 
-        @Param({ "10", "100", "1000", "10000"})
+        @Param({ "10", "100", "1000", "10000" })
         public int CONTAINER_SIZE;
 
         public Integer[] ELEMENTS;
         int expectedAggregate = 0;
 
         @Setup
         public void setup() {
-            final Random random = new Random(0);
-
-            ELEMENTS = new Integer[CONTAINER_SIZE];
-            for (int i = 0; i < CONTAINER_SIZE; i++) {
-                final int value = random.nextInt(CONTAINER_SIZE) - (CONTAINER_SIZE / 2);
-                ELEMENTS[i] = value;
-                expectedAggregate ^= value;
-            }
-        }
+            ELEMENTS = getRandomValues(CONTAINER_SIZE, 0);
 
-        protected static <T> void assertEquals(T a, T b) {
-            if (!Objects.equals(a, b)) {
-                throw new IllegalStateException(a + " != " + b);
+            for (int element : ELEMENTS) {
+                expectedAggregate ^= element;
             }
         }
     }