A more robust JMH benchmarking set-up

build.gradle

@@ -9,8 +9,20 @@ apply from: file('gradle/release.gradle')
 buildscript {
     repositories {
         mavenLocal()
-        mavenCentral() // maven { url 'http://jcenter.bintray.com' }
+        mavenCentral()
+        maven {
+            //FIXME: waiting for https://github.com/johnrengelman/shadow/pull/38 to merge
+            name 'Shadow'
+            url 'http://dl.bintray.com/content/gvsmirnov/gradle-plugins'
+        }
+        jcenter()
+    }
+
+    dependencies {
+        // Required for benchmarks
+        classpath 'com.github.jengelman.gradle.plugins:shadow:0.8.1'
     }
+
     apply from: file('gradle/buildscript.gradle'), to: buildscript 
 }
 
@@ -19,7 +31,7 @@ allprojects {
     apply plugin: 'idea'
     repositories { 
         mavenLocal()
-        mavenCentral() // maven { url: 'http://jcenter.bintray.com' }
+        mavenCentral()
     }
 }
 
@@ -31,57 +43,67 @@ subprojects {
     configurations { 
         examplesCompile.extendsFrom compile
         examplesRuntime.extendsFrom runtime
-	    perfCompile.extendsFrom compile
-	    perfRuntime.extendsFrom runtime
     }
 
 
     tasks.withType(Javadoc).each {
         it.classpath = sourceSets.main.compileClasspath
     }
 
-    sourceSets { 
-		//include /src/examples folder
+    sourceSets {
         examples
-		//include /src/perf folder
-        perf {
-            java {
-                srcDir 'src/perf/java'
-                compileClasspath += main.output
-                runtimeClasspath += main.output
-            }
-        }
-    }
-
-    dependencies {
-        perfCompile 'org.openjdk.jmh:jmh-core:0.2'
+        perf
     }
 
     tasks.build { 
-		//include 'examples' in build task
+        //include 'examples' in build task
         dependsOn(examplesClasses)
-		//include 'perf' in build task
-        // dependsOn(perfClasses)  //-> Not working so commented out
+    }
+
+    dependencies {
+        perfCompile 'org.openjdk.jmh:jmh-core:0.5.3'
+        perfCompile 'org.openjdk.jmh:jmh-generator-annprocess:0.5.3'
+
+        perfCompile project
     }
 
     eclipse {  
-        classpath {  
-            // include 'provided' dependencies on the classpath
-            plusConfigurations += configurations.provided
+        classpath {
             plusConfigurations += configurations.perfCompile
 
             downloadSources = true
             downloadJavadoc = true
         }
     }
-    
+
     idea {
         module {
-            // include 'provided' dependencies on the classpath
-            scopes.PROVIDED.plus += configurations.provided
-            // TODO not sure what to add it to
-            //scopes.PROVIDED.plus += configurations.perfCompile
+            scopes.PROVIDED.plus  += configurations.perfCompile
+            scopes.PROVIDED.minus += configurations.compile
+        }
+    }
+
+    task perfJar(type: Jar, dependsOn: perfClasses) {
+        from sourceSets.perf.output + sourceSets.main.output
+    }
+
+    task benchmarks(dependsOn: perfJar) {
+
+        apply plugin: "shadow"
+
+        shadow {
+            classifier = "benchmarks"
+            includeDependenciesFor = ["runtime", "perfRuntime"]
+
+            transformer(com.github.jengelman.gradle.plugins.shadow.transformers.ManifestResourceTransformer) {
+                mainClass = "org.openjdk.jmh.Main"
+            }
         }
+
+        doLast {
+            shadowJar.execute()
+        }
+
     }
 }
 

rxjava-core/build.gradle

@@ -1,6 +1,5 @@
 apply plugin: 'maven'
 apply plugin: 'osgi'
-apply plugin:'application'
 
 sourceCompatibility = JavaVersion.VERSION_1_6
 targetCompatibility = JavaVersion.VERSION_1_6
@@ -13,7 +12,7 @@ dependencies {
 javadoc {
     // we do not want the org.rx.operations package include
     exclude '**/operations/**'
-	
+
     options {
         doclet = "org.benjchristensen.doclet.DocletExclude"
         docletpath = [rootProject.file('./gradle/doclet-exclude.jar')]
@@ -31,11 +30,4 @@ jar {
         instruction 'Import-Package', '!org.junit,!junit.framework,!org.mockito.*,*'
         instruction 'Eclipse-ExtensibleAPI', 'true'
     }
-}
-
-task time(type:JavaExec) {
-    classpath = sourceSets.perf.runtimeClasspath
-    group 'Application'
-    description 'Execute the calipser benchmark timing of Rx'
-    main 'rx.operators.ObservableBenchmark'
 }

rxjava-core/src/perf/java/rx/operators/ObservableBenchmark.java

@@ -1,14 +1,12 @@
 package rx.operators;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
 
-import org.openjdk.jmh.annotations.GenerateMicroBenchmark;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.RunnerException;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.annotations.*;
 
+import org.openjdk.jmh.logic.BlackHole;
 import rx.Observable;
 import rx.Observable.OnSubscribe;
 import rx.Observable.Operator;
@@ -19,107 +17,84 @@
 public class ObservableBenchmark {
 
     @GenerateMicroBenchmark
-    public void timeBaseline() {
-        observableOfInts.subscribe(newObserver());
-        awaitAllObservers();
-    }
-
-    @GenerateMicroBenchmark
-    public int timeMapIterate() {
-        int x = 0;
-        for (int j = 0; j < intValues.length; j++) {
-            // use hash code to make sure the JIT doesn't optimize too much and remove all of
-            // our code.
-            x |= ident.call(intValues[j]).hashCode();
+    public void measureBaseline(BlackHole bh, Input input) {
+        for (Integer value : input.values) {
+            bh.consume(IDENTITY_FUNCTION.call(value));
         }
-        return x;
     }
 
     @GenerateMicroBenchmark
-    public void timeMap() {
-        timeOperator(new OperatorMap<Integer, Object>(ident));
+    public void measureMap(Input input) throws InterruptedException {
+        input.observable.lift(MAP_OPERATOR).subscribe(input.observer);
+
+        input.awaitCompletion();
     }
 
-    /**************************************************************************
-     * Below is internal stuff to avoid object allocation and time overhead of anything that isn't
-     * being tested.
-     * 
-     * @throws RunnerException
-     **************************************************************************/
+    private static final Func1<Integer, Integer> IDENTITY_FUNCTION = new Func1<Integer, Integer>() {
+        @Override
+        public Integer call(Integer value) {
+            return value;
+        }
+    };
 
-    public static void main(String[] args) throws RunnerException {
-        Options opt = new OptionsBuilder()
-                .include(ObservableBenchmark.class.getName()+".*")
-                .forks(1)
-                .build();
+    private static final Operator<Integer, Integer> MAP_OPERATOR = new OperatorMap<Integer, Integer>(IDENTITY_FUNCTION);
 
-        new Runner(opt).run();
-    }
+    @State(Scope.Thread)
+    public static class Input {
 
-    private void timeOperator(Operator<Object, Integer> op) {
-        observableOfInts.lift(op).subscribe(newObserver());
-        awaitAllObservers();
-    }
+        @Param({"1", "1024", "1048576"})
+        public int size;
 
-    private final static AtomicInteger outstanding = new AtomicInteger(0);
-    private final static CountDownLatch latch = new CountDownLatch(1);
+        public Collection<Integer> values;
+        public Observable<Integer> observable;
+        public Observer<Integer> observer;
 
-    private static <T> Observer<T> newObserver() {
-        outstanding.incrementAndGet();
-        return new Observer<T>() {
-            @Override
-            public void onCompleted() {
-                int left = outstanding.decrementAndGet();
-                if (left == 0) {
-                    latch.countDown();
-                }
+        private CountDownLatch latch;
+
+        @Setup
+        public void setup() {
+            values = new ArrayList<Integer>();
+            for(int i = 0; i < size; i ++) {
+                values.add(i);
             }
 
-            @Override
-            public void onError(Throwable e) {
-                int left = outstanding.decrementAndGet();
-                if (left == 0) {
+            observable = Observable.create(new OnSubscribe<Integer>() {
+                @Override
+                public void call(Subscriber<? super Integer> o) {
+                    for (Integer value : values) {
+                        if (o.isUnsubscribed())
+                            return;
+                        o.onNext(value);
+                    }
+                    o.onCompleted();
+                }
+            });
+
+            final BlackHole bh = new BlackHole();
+            latch = new CountDownLatch(1);
+
+            observer = new Observer<Integer>() {
+                @Override
+                public void onCompleted() {
                     latch.countDown();
                 }
-            }
 
-            @Override
-            public void onNext(T t) {
-                // do nothing
-            }
-        };
-    }
+                @Override
+                public void onError(Throwable e) {
+                    throw new RuntimeException(e);
+                }
+
+                @Override
+                public void onNext(Integer value) {
+                    bh.consume(value);
+                }
+            };
 
-    private static void awaitAllObservers() {
-        try {
-            latch.await();
-        } catch (InterruptedException e) {
-            return;
         }
-    }
 
-    private static final Integer[] intValues = new Integer[1000];
-    static {
-        for (int i = 0; i < intValues.length; i++) {
-            intValues[i] = i;
+        public void awaitCompletion() throws InterruptedException {
+            latch.await();
         }
     }
 
-    private static final Observable<Integer> observableOfInts = Observable.create(new OnSubscribe<Integer>() {
-        @Override
-        public void call(Subscriber<? super Integer> o) {
-            for (int i = 0; i < intValues.length; i++) {
-                if (o.isUnsubscribed())
-                    return;
-                o.onNext(intValues[i]);
-            }
-            o.onCompleted();
-        }
-    });
-    private static final Func1<Integer, Object> ident = new Func1<Integer, Object>() {
-        @Override
-        public Object call(Integer t) {
-            return t;
-        }
-    };
 }