diff --git a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/ScoreStabilityBench.java b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/BurstStabilityBench.java similarity index 98% rename from jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/ScoreStabilityBench.java rename to jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/BurstStabilityBench.java index 8d0110358..ecc39bac3 100644 --- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/ScoreStabilityBench.java +++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/BurstStabilityBench.java @@ -35,7 +35,7 @@ @Measurement(iterations = 50) @Threads(1) @State(Scope.Thread) -public class ScoreStabilityBench { +public class BurstStabilityBench { @Param("10") private int delay; diff --git a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LongStabilityBench.java b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LongStabilityBench.java new file mode 100644 index 000000000..d0ccfd34e --- /dev/null +++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/benchmarks/LongStabilityBench.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jmh.benchmarks; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Threads(1) +@State(Scope.Thread) +public class LongStabilityBench { + + @Benchmark + public void test() { + Blackhole.consumeCPU(1_000_000); + } + +} diff --git a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/Main.java b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/Main.java index 2fa031765..c7babe03d 100644 --- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/Main.java +++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/Main.java @@ -158,8 +158,26 @@ public static void main(String[] args) throws RunnerException, CommandLineOption case timing: new TimingMeasurementsTest().runWith(pw, opts); break; - case stability: - new ScoreStabilityTest().runWith(pw, opts); + case long_stability: + switch (mode) { + case flash: + new LongStabilityTest(3).runWith(pw, opts); + break; + case quick: + new LongStabilityTest(5).runWith(pw, opts); + break; + case normal: + new LongStabilityTest(18).runWith(pw, opts); + break; + case longer: + new LongStabilityTest(60).runWith(pw, opts); + break; + default: + throw new IllegalStateException(); + } + break; + case burst_stability: + new BurstStabilityTest().runWith(pw, opts); break; case compiler_hints: new CompilerHintsTest().runWith(pw, opts); @@ -242,7 +260,8 @@ public enum Test { timing, compiler_hints, thermal, - stability, + long_stability, + burst_stability, thread_scale, helpers, blackhole_cpu, diff --git a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/ScoreStabilityTest.java b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/BurstStabilityTest.java similarity index 94% rename from jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/ScoreStabilityTest.java rename to jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/BurstStabilityTest.java index 508d9b85d..bedc503aa 100644 --- a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/ScoreStabilityTest.java +++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/BurstStabilityTest.java @@ -25,7 +25,7 @@ package org.openjdk.jmh.validation.tests; import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.benchmarks.ScoreStabilityBench; +import org.openjdk.jmh.benchmarks.BurstStabilityBench; import org.openjdk.jmh.results.Result; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; @@ -37,10 +37,10 @@ import java.io.PrintWriter; -public class ScoreStabilityTest extends ValidationTest { +public class BurstStabilityTest extends ValidationTest { @Override public void runWith(PrintWriter pw, Options parent) throws RunnerException { - pw.println("--------- SCORE STABILITY TEST"); + pw.println("--------- BURST STABILITY TEST"); pw.println(); org.openjdk.jmh.util.Utils.reflow(pw, @@ -74,7 +74,7 @@ public void runWith(PrintWriter pw, Options parent) throws RunnerException { Options opts = new OptionsBuilder() .parent(parent) .mode(m) - .include(ScoreStabilityBench.class.getCanonicalName()) + .include(BurstStabilityBench.class.getCanonicalName()) .verbosity(VerboseMode.SILENT) .param("delay", String.valueOf(delay)) .build(); diff --git a/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/LongStabilityTest.java b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/LongStabilityTest.java new file mode 100644 index 000000000..1185fd256 --- /dev/null +++ b/jmh-core-benchmarks/src/main/java/org/openjdk/jmh/validation/tests/LongStabilityTest.java @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jmh.validation.tests; + +import org.openjdk.jmh.benchmarks.LongStabilityBench; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; +import org.openjdk.jmh.runner.options.VerboseMode; +import org.openjdk.jmh.validation.ValidationTest; + +import java.io.PrintWriter; +import java.util.concurrent.ThreadLocalRandom; + +public class LongStabilityTest extends ValidationTest { + private final int tries; + + public LongStabilityTest(int tries) { + this.tries = tries; + } + + @Override + public void runWith(PrintWriter pw, Options parent) throws RunnerException { + pw.println("--------- LONG STABILITY TEST"); + pw.println(); + + org.openjdk.jmh.util.Utils.reflow(pw, + "This test verifies the performance for a single test by running it several times with some " + + "delays between the runs. The performance should be the same across all runs. " + + "If there is a significant difference between the runs, this is usually " + + "indicative of noisy environment, e.g. a busy virtualized node, or background processes " + + "interfering with the run, making the benchmarks unreliable.", + 80, 2); + pw.println(); + + for (int t = 0; t < tries; t++) { + int ms = (t == 0) ? 0 : ThreadLocalRandom.current().nextInt(5_000, 30_000); + + pw.printf(" Sleeping for %6d ms...", ms); + pw.flush(); + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + // Do nothing. + } + + pw.print(" Run: "); + pw.flush(); + + Options opts = new OptionsBuilder() + .parent(parent) + .include(LongStabilityBench.class.getCanonicalName()) + .warmupIterations(5) + .warmupTime(TimeValue.seconds(1)) + .measurementIterations(5) + .measurementTime(TimeValue.seconds(1)) + .forks(1) + .verbosity(VerboseMode.SILENT) + .build(); + + RunResult result = new Runner(opts).runSingle(); + Result r = result.getPrimaryResult(); + pw.printf(" %16s", String.format("%.2f \u00b1 %.2f %s%n", r.getScore(), r.getScoreError(), r.getScoreUnit())); + pw.flush(); + } + } +}