From 5c067a6bdca4aa863c43e6f8209aad81956a7474 Mon Sep 17 00:00:00 2001 From: lkryukova Date: Mon, 21 Dec 2020 09:07:50 -0800 Subject: [PATCH 1/4] Add timeout A timeout is required to prevent freezing and some crashes. --- .../benchmarks/concurrent/RoundTripNotification.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/RoundTripNotification.java b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/RoundTripNotification.java index 4d93868..d40448f 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/RoundTripNotification.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/RoundTripNotification.java @@ -75,7 +75,7 @@ public void close() { terminating = true; receiver.interrupt(); try { - receiver.join(); + receiver.join(3000); } catch (InterruptedException e) { throw new RuntimeException("Thread didn't stop", e); } From 14088d4b352d45084def3db2cfe443f69e6fa3ad Mon Sep 17 00:00:00 2001 From: lkryukova Date: Tue, 22 Dec 2020 00:05:11 -0800 Subject: [PATCH 2/4] Stream stabilization Problems: 1. Using a random number as a thrashhold is an extremely unstable way to estimate. Example: take 1,000,000 numbers and for the first iteration add up all to a random number which turned out to be 10, and on the second it turned out to be 10,000, respectively, and the time on the iteration will differ and variation per iterations jumps 2. Using 1 number to compose an array takes place. But since we operate with large numbers and the largest in this chain do not have time to pass in 1 second. It will be more convenient and efficient to measure the average time. that is, to evaluate not on one number, but to use the range for the trashholds at once and evaluate the average of them. We took it depending on the numbers from 1% for little to 50% for huge ones. 3. Shuffle destabilizes the program only because when adding values, they are stored side by side. But by mixing them, he mixes them in memory, so each time we have a different distance between the necessary cells in memory. That would not remove the possibility of mixing numbers. On the setup, after mixing, we added a pass to stabilize the values in memory, so that they would be consistently next to each other. 4. Make a setup at each iteration on which each time both the array shuffle and the threshhold change, see the description above - this is not advisable from the point of view of variation, which turns out to be more than 100% in many cases. Accordingly, the setup before all iterations reduced the variation several times. 5. Some cases remained unchanged because they are not variable. --- .../collection/StreamBenchmark.java | 96 ++++++++++++++----- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java index 010f361..d40a40d 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java @@ -5,7 +5,7 @@ import org.openjdk.jmh.annotations.OperationsPerInvocation; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.infra.Blackhole; @@ -17,12 +17,16 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; +@State(Scope.Thread) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) @OperationsPerInvocation(1) public class StreamBenchmark extends AbstractCollectionBenchmark { @@ -50,7 +54,6 @@ public Object integerStreamGroupByParallel(MyState state) { public int integerStreamReduceToSum(MyState state) { return state.integers.stream().reduce(0, (i, j) -> i + j); } - @Benchmark public int integerStreamReduceToSumParallel(MyState state) { return state.integers.stream().parallel().reduce(0, (i, j) -> i + j); @@ -68,47 +71,69 @@ public int integerStreamSumParallel(MyState state) { @Benchmark public void integerStreamForEach(MyState state, Blackhole blackhole) { - Collection target = new ArrayList<>(); - state.integers.stream().filter(i -> i < state.integerFilter).forEach(target::add); + final Collection target = new ArrayList<>(); + for (Integer threshold : state.range) { + state.integers.stream().filter(i -> i < threshold).forEach(target::add); + } blackhole.consume(target); } @Benchmark public void integerStreamForEachParallel(MyState state, Blackhole blackhole) { - Set target = new ConcurrentSkipListSet<>(); - state.integers.stream().parallel().filter(i -> i < state.integerFilter).forEach(target::add); + final Set target = new ConcurrentSkipListSet<>(); + for (Integer threshold : state.range) { + state.integers.stream().parallel().filter(i -> i < threshold).forEach(target::add); + } blackhole.consume(target); } @Benchmark public void integerStreamCollect(MyState state, Blackhole blackhole) { - blackhole.consume(state.integers.stream().filter(i -> i < state.integerFilter).collect(toList())); + Collection target = new ArrayList<>(); + for (Integer threshold : state.range) { + target = state.integers.stream().filter(i -> i < threshold).collect(toList()); + } + blackhole.consume(target); } @Benchmark public void integerStreamCollectParallel(MyState state, Blackhole blackhole) { - blackhole.consume(state.integers.stream().parallel().filter(i -> i < state.integerFilter).collect(toList())); + Collection target = new ArrayList<>(); + for (Integer threshold : state.range) { + target = state.integers.stream().parallel().filter(i -> i < threshold).collect(toList()); + } + blackhole.consume(target); } @Benchmark public void integerFor(MyState state, Blackhole blackhole) { Collection source = state.integers; Collection target = new ArrayList<>(); - Integer filter = state.integerFilter; - for (Integer i : source) - if (i < filter) - target.add(i); + for (Integer threshold : state.range) { + Integer filter = threshold; + for (Integer i : source) + if (i < filter) + target.add(i); + } blackhole.consume(target); } @Benchmark public void intStream(MyState state, Blackhole blackhole) { - blackhole.consume(Arrays.stream(state.ints).filter(i -> i < state.intFilter).toArray()); + int[] target = new int[state.collectionSize]; + for (Integer threshold : state.range) { + target = Arrays.stream(state.ints).filter(i -> i < threshold).toArray(); + } + blackhole.consume(target); } @Benchmark public void intStreamParallel(MyState state, Blackhole blackhole) { - blackhole.consume(Arrays.stream(state.ints).parallel().filter(i -> i < state.intFilter).toArray()); + int[] target = new int[state.collectionSize]; + for (Integer threshold : state.range) { + Arrays.stream(state.ints).parallel().filter(i -> i < threshold).toArray(); + } + blackhole.consume(target); } @Benchmark @@ -116,11 +141,14 @@ public void intFor(MyState state, Blackhole blackhole) { int[] source = state.ints; int[] target = new int[source.length]; int targetIndex = 0; - int filter = state.intFilter; - for (int i = 0; i < source.length; i++) { - int candidate = source[i]; - if (candidate < filter) - target[targetIndex++] = candidate; + for (Integer threshold : state.range) { + targetIndex = 0; + int filter = threshold; + for (int i = 0; i < source.length; i++) { + int candidate = source[i]; + if (candidate < filter) + target[targetIndex++] = candidate; + } } int[] trimmedTarget = new int[targetIndex]; System.arraycopy(source, 0, trimmedTarget, 0, targetIndex); @@ -131,20 +159,36 @@ public void intFor(MyState state, Blackhole blackhole) { public static class MyState { static Random random = new Random(); public List integers; + public List range; public int[] ints; - public Integer integerFilter; - public int intFilter; - @Param({"1", "10", "100", "1000", "10000", "100000", "1000000", "10000000"}) + + @Param({"1", "10", "100", "1000", "10000", "100000","1000000", "10000000"}) int collectionSize; - @Setup(Level.Iteration) + @Setup(Level.Trial) public void doSetup() { - integerFilter = random.nextInt(collectionSize); - intFilter = integerFilter; - + if (collectionSize <= 100) { + range = range(0, collectionSize).boxed().collect(toList()); + } + else if (collectionSize <= 100000) { + int step = collectionSize / 100; + range = range(0, 100).map(x -> x * step).boxed().collect(toList()); + } + else { + int step = collectionSize / 2; + range = range(0, 2).map(x -> x * step).boxed().collect(toList()); + } integers = range(0, collectionSize).boxed().collect(toList()); Collections.shuffle(integers, random); ints = integers.stream().mapToInt(Integer::valueOf).toArray(); + + // Fix data layout so that all elements are plced continuously in memory (not in random order ad after shuffle) + List old_int = integers; + integers = new ArrayList(old_int.size()); + for (Integer i: old_int) { + integers.add(Integer.valueOf(i.intValue())); + } } } + } From b25ef058818f7bf84c7d129f0ef61d7e9895c852 Mon Sep 17 00:00:00 2001 From: lkryukova Date: Thu, 31 Dec 2020 06:09:59 -0800 Subject: [PATCH 3/4] Stabilizing collections and run parameter settings 1. For some benchmarks, there was too little warm-up 2. Adding a random number strongly affects the variation. Therefore, we generate an array in advance in the setup, and then simply add elements from this array to other structures. 3. We increased the value and slightly reduced the execution time for blocked structures, since there were regular falls. --- .../benchmarks/AbstractBenchmark.java | 8 +- .../benchmarks/CounterBenchmark.java | 9 +- .../benchmarks/ObjectCacheBenchmark.java | 7 +- .../benchmarks/TimeBenchmark.java | 4 +- .../AbstractCollectionBenchmark.java | 2 +- .../AbstractCollectionBenchmarkState.java | 6 +- .../collection/ArrayAddBenchmark.java | 9 +- .../collection/CollectionAddBenchmark.java | 12 +- .../CollectionIterateBenchmark.java | 13 +- .../collection/StreamBenchmark.java | 4 + .../concurrent/ConcCollectionBenchmark.java | 276 ++++++++++-------- .../concurrent/NotifyBenchmark.java | 11 +- .../fieldaccess/GetterBenchmark.java | 9 +- .../fieldaccess/SetterBenchmark.java | 9 +- 14 files changed, 212 insertions(+), 167 deletions(-) diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/AbstractBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/AbstractBenchmark.java index 1c9a759..75dfbd8 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/AbstractBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/AbstractBenchmark.java @@ -7,12 +7,12 @@ import org.openjdk.jmh.annotations.OutputTimeUnit; import org.openjdk.jmh.annotations.Warmup; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; @BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(MILLISECONDS) -@Warmup(iterations = 5, time = 1000, timeUnit = MILLISECONDS) -@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) +@OutputTimeUnit(SECONDS) +@Warmup(iterations = 50, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) @Fork(1) public abstract class AbstractBenchmark { diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/CounterBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/CounterBenchmark.java index 5586617..209f8da 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/CounterBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/CounterBenchmark.java @@ -2,19 +2,18 @@ import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableLong; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; +import static java.util.concurrent.TimeUnit.*; @OperationsPerInvocation(OPERATIONS_PER_PER_INVOCATION) +@Warmup(iterations = 20, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) public class CounterBenchmark extends AbstractBenchmark { @Benchmark diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/ObjectCacheBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/ObjectCacheBenchmark.java index 9c5067d..b09d9de 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/ObjectCacheBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/ObjectCacheBenchmark.java @@ -1,14 +1,15 @@ package com.github.chrisgleissner.benchmarks; import lombok.Value; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.OperationsPerInvocation; - +import org.openjdk.jmh.annotations.*; +import static java.util.concurrent.TimeUnit.*; import java.util.stream.IntStream; import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; @OperationsPerInvocation(OPERATIONS_PER_PER_INVOCATION) +@Warmup(iterations = 20, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) public class ObjectCacheBenchmark extends AbstractBenchmark { private static Id[] idPool = IntStream.range(0, OPERATIONS_PER_PER_INVOCATION).mapToObj(Id::new).toArray(Id[]::new); diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/TimeBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/TimeBenchmark.java index 3734bc4..f9a537c 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/TimeBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/TimeBenchmark.java @@ -10,8 +10,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; -@Warmup(iterations = 5, time = 300, timeUnit = MILLISECONDS) -@Measurement(iterations = 5, time = 300, timeUnit = MILLISECONDS) +@Warmup(iterations = 20, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) public class TimeBenchmark extends AbstractBenchmark { @Benchmark diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmark.java index 4524b9c..4f3c489 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmark.java @@ -1,7 +1,7 @@ package com.github.chrisgleissner.benchmarks.collection; import com.github.chrisgleissner.benchmarks.AbstractBenchmark; -import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.*; import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmarkState.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmarkState.java index 6622ae1..33fd413 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmarkState.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmarkState.java @@ -67,10 +67,10 @@ public abstract class AbstractCollectionBenchmarkState { @Setup(Level.Iteration) public void doSetup() { - abq = new ArrayBlockingQueue<>(MAX_OPERATIONS_PER_ITERATION); + abq = new ArrayBlockingQueue<>(MAX_OPERATIONS_PER_ITERATION * 2); ad = new ArrayDeque<>(); al = new ArrayList<>(); - alnr = new ArrayList<>(MAX_OPERATIONS_PER_ITERATION); + alnr = new ArrayList<>(MAX_OPERATIONS_PER_ITERATION * 2); chm = new ConcurrentHashMap<>(); cld = new ConcurrentLinkedDeque<>(); clq = new ConcurrentLinkedQueue<>(); @@ -92,7 +92,7 @@ public void doSetup() { tm = new TreeMap<>(); ts = new TreeSet<>(); v = new Vector<>(); - vnr = new Vector<>(MAX_OPERATIONS_PER_ITERATION); + vnr = new Vector<>(MAX_OPERATIONS_PER_ITERATION * 2); } @TearDown(Level.Iteration) diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/ArrayAddBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/ArrayAddBenchmark.java index e0954fa..e105379 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/ArrayAddBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/ArrayAddBenchmark.java @@ -1,16 +1,15 @@ package com.github.chrisgleissner.benchmarks.collection; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.*; import java.util.Arrays; import java.util.stream.IntStream; import java.util.stream.LongStream; +import static java.util.concurrent.TimeUnit.*; import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; - +@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) public class ArrayAddBenchmark extends AbstractCollectionBenchmark { @Benchmark diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionAddBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionAddBenchmark.java index 04eca55..d67e468 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionAddBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionAddBenchmark.java @@ -1,14 +1,18 @@ package com.github.chrisgleissner.benchmarks.collection; -import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.*; import java.util.Collection; import java.util.Map; - +import static java.util.concurrent.TimeUnit.*; import static com.github.chrisgleissner.benchmarks.Constants.MAX_OPERATIONS_PER_ITERATION; import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; import static java.util.stream.IntStream.range; +@BenchmarkMode(Mode.Throughput) +@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) +@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"}) public class CollectionAddBenchmark extends AbstractCollectionBenchmark { static Integer[] integers = range(0, MAX_OPERATIONS_PER_ITERATION).boxed().toArray(Integer[]::new); @@ -34,6 +38,7 @@ public Object ArrayListNoResize(MyState s) { } @Benchmark + @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS) public Object ConcurrentHashMap(MyState s) { return benchmarkPut(s, s.chm); } @@ -49,6 +54,7 @@ public Object ConcurrentLinkedQueue(MyState s) { } @Benchmark + @Measurement(iterations = 100, time = 1000, timeUnit = MILLISECONDS) public Object ConcurrentSkipListMap(MyState s) { return benchmarkPut(s, s.cslm); } @@ -69,6 +75,7 @@ public Object CopyOnWriteArraySet(MyState s) { } @Benchmark + @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS) public Object HashMap(MyState s) { return benchmarkPut(s, s.hm); } @@ -94,6 +101,7 @@ public Object LinkedHashSet(MyState s) { } @Benchmark + @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS) public Object LinkedHashMap(MyState s) { return benchmarkPut(s, s.lhm); } diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionIterateBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionIterateBenchmark.java index f9c048b..f028765 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionIterateBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionIterateBenchmark.java @@ -1,8 +1,6 @@ package com.github.chrisgleissner.benchmarks.collection; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import java.util.Collection; @@ -13,7 +11,12 @@ import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION; import static java.util.function.Function.identity; +import static java.util.concurrent.TimeUnit.*; +@BenchmarkMode(Mode.Throughput) +@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) +@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"}) public class CollectionIterateBenchmark extends AbstractCollectionBenchmark { @Benchmark @@ -23,8 +26,10 @@ public void ArrayBlockingQueue(MyState s, Blackhole bh) { private void iterate(Blackhole bh, Collection target) { Iterator iterator = target.iterator(); + int sum = 0; while (iterator.hasNext()) - bh.consume(iterator.next()); + sum += iterator.next(); + bh.consume(sum); } @Benchmark diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java index d40a40d..d43b4cd 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java @@ -23,10 +23,14 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; +import static java.util.concurrent.TimeUnit.*; @State(Scope.Thread) @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"}) @OperationsPerInvocation(1) public class StreamBenchmark extends AbstractCollectionBenchmark { diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/ConcCollectionBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/ConcCollectionBenchmark.java index 6594176..68f00b6 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/ConcCollectionBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/ConcCollectionBenchmark.java @@ -1,17 +1,11 @@ package com.github.chrisgleissner.benchmarks.concurrent; import com.github.chrisgleissner.benchmarks.collection.AbstractCollectionBenchmark; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Group; -import org.openjdk.jmh.annotations.GroupThreads; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.*; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -19,6 +13,7 @@ import java.util.Random; import java.util.Set; import java.util.Vector; +import static java.util.stream.IntStream.range; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; @@ -28,237 +23,250 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArraySet; import java.util.stream.IntStream; +import java.util.concurrent.TimeUnit; import static java.util.Collections.synchronizedList; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 50, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @OperationsPerInvocation(1) public class ConcCollectionBenchmark extends AbstractCollectionBenchmark { - private static final int GET_THREADS = 10; private static final int ADD_THREADS = 2; - private static final int INITIAL_SIZE = 100_000; + private static final int INITIAL_SIZE = 500_000; private static final int MAX_BOUNDED_SIZE = 10_000_000; @Benchmark - @GroupThreads(GET_THREADS) - @Group("ArrayBlockingQueue") public Object ArrayBlockingQueueGet(ArrayBlockingQueueState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ArrayBlockingQueue") public Object ArrayBlockingQueueAdd(ArrayBlockingQueueState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads - @Group("ArrayBlockingQueue") public Object ArrayBlockingQueueRemove(ArrayBlockingQueueState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ArrayList") public Object ArrayListGet(ArrayListState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ArrayList") public Object ArrayListAdd(ArrayListState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads - @Group("ArrayList") public Integer ArrayListRemove(ArrayListState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ConcurrentHashMap") public Integer ConcurrentHashMapGet(ConcurrentHashMapState s) { - return benchmarkGet(s.source, s.target); + Integer result = 0; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkGet(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ConcurrentHashMap") public Object ConcurrentHashMapAdd(ConcurrentHashMapState s) { - return benchmarkAdd(s.source, s.target); + Map result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads - @Group("ConcurrentHashMap") public Object ConcurrentHashMapRemove(ConcurrentHashMapState s) { - return benchmarkRemove(s.source, s.target); + Map m = null; + for (int i = 0; i < s.source_list.size(); i++) { + m = benchmarkRemove(s.source_list.get(i), s.target); + } + return m; } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ConcurrentLinkedDeque") public Object ConcurrentLinkedDequeGet(ConcurrentLinkedDequeState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ConcurrentLinkedDeque") public Object ConcurrentLinkedDequeAdd(ConcurrentLinkedDequeState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads - @Group("ConcurrentLinkedDeque") public Object ConcurrentLinkedDequeRemove(ConcurrentLinkedDequeState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ConcurrentLinkedQueue") public Integer ConcurrentLinkedQueueGet(ConcurrentLinkedQueueState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ConcurrentLinkedQueue") public Object ConcurrentLinkedQueueAdd(ConcurrentLinkedQueueState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads - @Group("ConcurrentLinkedQueue") public Object ConcurrentLinkedQueueRemove(ConcurrentLinkedQueueState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ConcurrentSkipListMap") + @Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) public Object ConcurrentSkipMapSetGet(ConcurrentSkipListMapState s) { - return benchmarkGet(s.source, s.target); + Integer result = 0; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkGet(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ConcurrentSkipListMap") + @Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) public Object ConcurrentSkipListMapAdd(ConcurrentSkipListMapState s) { - return benchmarkAdd(s.source, s.target); + Map m = null; + for (int i = 0; i < s.source_list.size(); i++) { + m = benchmarkAdd(s.source_list.get(i), s.target); + } + return m; } @Benchmark - @GroupThreads - @Group("ConcurrentSkipListMap") + @Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) public Object ConcurrentSkipListMapRemove(ConcurrentSkipListMapState s) { - return benchmarkRemove(s.source, s.target); + Map result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkRemove(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(GET_THREADS) - @Group("ConcurrentSkipListSet") public Object ConcurrentSkipListSetGet(ConcurrentSkipListSetState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("ConcurrentSkipListSet") + @Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) public Object ConcurrentSkipListSetAdd(ConcurrentSkipListSetState s) { - return benchmarkAdd(s.source, s.target); - } - - @Benchmark - @GroupThreads - @Group("ConcurrentSkipListSet") - public Object ConcurrentSkipListSetRemove(ConcurrentSkipListSetState s) { - return benchmarkRemove(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(GET_THREADS) - @Group("CopyOnWriteArrayList") public Object CopyOnWriteArrayListGet(CopyOnWriteArrayListState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("CopyOnWriteArrayList") public Object CopyOnWriteArrayAdd(CopyOnWriteArrayListState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("CopyOnWriteArrayList") public Object CopyOnWriteArrayRemove(CopyOnWriteArrayListState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("CopyOnWriteArraySet") + @Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) + public Object ConcurrentSkipLPistSetRemove(ConcurrentSkipListSetState s) { + boolean result = false; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkRemove(s.source_list.get(i), s.target); + } + return result; + } + + @Benchmark public Object CopyOnWriteArraySetGet(CopyOnWriteArraySetState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("CopyOnWriteArraySet") public Object CopyOnWriteArraySetAdd(CopyOnWriteArraySetState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("Vector") public Object VectorRemove(VectorState s) { return benchmarkRemove(s.target); } @Benchmark - @GroupThreads(GET_THREADS) - @Group("Vector") public Object VectorGet(VectorState s) { return benchmarkGet(s.target); } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("Vector") public Object VectorAdd(VectorState s) { - return benchmarkAdd(s.source, s.target); + Collection result = null; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkAdd(s.source_list.get(i), s.target); + } + return result; } @Benchmark - @GroupThreads(ADD_THREADS) - @Group("CopyOnWriteArraySet") public Object CopyOnWriteArraySetRemove(CopyOnWriteArraySetState s) { - return benchmarkRemove(s.source, s.target); + boolean result = false; + for (int i = 0; i < s.source_list.size(); i++) { + result = benchmarkRemove(s.source_list.get(i), s.target); + } + return result; } private Integer benchmarkGet(List target) { - return target.get(0); + return (target.size() > 0) ? target.get(0) : 0; } private Integer benchmarkRemove(List target) { - return target.remove(0); + return (target.size() > 0) ? target.remove(0) : 0; } private Integer benchmarkGet(Integer key, Map target) { @@ -266,11 +274,11 @@ private Integer benchmarkGet(Integer key, Map target) { } private Integer benchmarkGet(Set target) { - return target.iterator().next(); + return (target.size() > 0) ? target.iterator().next() : 0; } private Integer benchmarkGet(Queue target) { - return target.peek(); + return (target.size() > 0) ? target.peek() : 0; } private Integer benchmarkRemove(Queue target) { @@ -283,11 +291,14 @@ private Map benchmarkRemove(Integer i, Map t } private Integer benchmarkGet(NavigableSet target) { - return target.first(); + return (target.size() > 0) ? target.first() : 0; } private Collection benchmarkAdd(Integer i, Collection target) { - target.add(i); + if (target.size() == MAX_BOUNDED_SIZE - 1) { + return target; + } + target.add(i); return target; } @@ -300,7 +311,7 @@ private Map benchmarkAdd(Integer i, Map targ return target; } - @State(Scope.Benchmark) + @State(Scope.Benchmark) public static class ArrayBlockingQueueState extends AbstractState { ArrayBlockingQueue target; @@ -310,8 +321,9 @@ public void doSetup() { target.addAll(initialData); } } - - public static class ArrayListState extends AbstractState { + + @State(Scope.Benchmark) + public static class ArrayListState extends AbstractState { List target; @Setup @@ -321,8 +333,9 @@ public void doSetup() { target.addAll(initialData); } } - - public static class ConcurrentHashMapState extends AbstractState { + + @State(Scope.Benchmark) + public static class ConcurrentHashMapState extends AbstractState { ConcurrentHashMap target; @Setup @@ -332,8 +345,9 @@ public void doSetup() { target.putAll(initialData.stream().collect(toMap(identity(), identity()))); } } - - public static class ConcurrentLinkedDequeState extends AbstractState { + + @State(Scope.Benchmark) + public static class ConcurrentLinkedDequeState extends AbstractState { ConcurrentLinkedDeque target; @Setup @@ -343,8 +357,9 @@ public void doSetup() { target.addAll(initialData); } } - - public static class ConcurrentLinkedQueueState extends AbstractState { + + @State(Scope.Benchmark) + public static class ConcurrentLinkedQueueState extends AbstractState { ConcurrentLinkedQueue target; @Setup @@ -354,8 +369,9 @@ public void doSetup() { target.addAll(initialData); } } - - public static class ConcurrentSkipListMapState extends AbstractState { + + @State(Scope.Benchmark) + public static class ConcurrentSkipListMapState extends AbstractState { ConcurrentSkipListMap target; @Setup @@ -365,8 +381,9 @@ public void doSetup() { target.putAll(initialData.stream().collect(toMap(identity(), identity()))); } } - - public static class ConcurrentSkipListSetState extends AbstractState { + + @State(Scope.Benchmark) + public static class ConcurrentSkipListSetState extends AbstractState { ConcurrentSkipListSet target; @Setup @@ -376,8 +393,9 @@ public void doSetup() { target.addAll(initialData); } } - - public static class CopyOnWriteArrayListState extends AbstractState { + + @State(Scope.Benchmark) + public static class CopyOnWriteArrayListState extends AbstractState { CopyOnWriteArrayList target; @Setup @@ -387,19 +405,20 @@ public void doSetup() { target.addAll(initialData); } } - - public static class CopyOnWriteArraySetState extends AbstractState { + + @State(Scope.Benchmark) + public static class CopyOnWriteArraySetState extends AbstractStateSet { CopyOnWriteArraySet target; @Setup public void doSetup() { super.doSetup(); - target = new CopyOnWriteArraySet(); - target.addAll(initialData); + target = new CopyOnWriteArraySet(initialDataSet); } } - - public static class VectorState extends AbstractState { + + @State(Scope.Benchmark) + public static class VectorState extends AbstractState { Vector target; @Setup @@ -410,17 +429,32 @@ public void doSetup() { } } - @State(Scope.Group) + @State(Scope.Benchmark) public static abstract class AbstractState { static Random random = new Random(); - int source; + List source_list = new ArrayList(); Collection initialData; @Setup(Level.Iteration) public void doSetup() { - source = random.nextInt(); - initialData = IntStream.range(MAX_BOUNDED_SIZE / 2, MAX_BOUNDED_SIZE / 2 + INITIAL_SIZE).boxed().collect(toList()); + int step = INITIAL_SIZE / 100; + source_list = range(0, 100).map(x -> x * step).boxed().sorted(Collections.reverseOrder()).collect(toList()); + initialData = IntStream.range(MAX_BOUNDED_SIZE, MAX_BOUNDED_SIZE + INITIAL_SIZE).boxed().collect(toList()); + } + } + + @State(Scope.Benchmark) + public static abstract class AbstractStateSet { + static Random random = new Random(); + CopyOnWriteArraySet initialDataSet = new CopyOnWriteArraySet<>(IntStream.range(MAX_BOUNDED_SIZE, MAX_BOUNDED_SIZE + INITIAL_SIZE).boxed().collect(toList())); + + List source_list = new ArrayList(); + + @Setup(Level.Iteration) + public void doSetup() { + int step = INITIAL_SIZE / 100; + source_list = range(0, 100).map(x -> x * step).boxed().sorted(Collections.reverseOrder()).collect(toList()); } } } diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/NotifyBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/NotifyBenchmark.java index 8ec3cff..560a87f 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/NotifyBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/concurrent/NotifyBenchmark.java @@ -2,20 +2,17 @@ import com.github.chrisgleissner.benchmarks.AbstractBenchmark; import lombok.extern.slf4j.Slf4j; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicReference; +import static java.util.concurrent.TimeUnit.*; @Slf4j +@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS) @OperationsPerInvocation(100) public class NotifyBenchmark extends AbstractBenchmark { diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/GetterBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/GetterBenchmark.java index 8c5161a..e90b3d5 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/GetterBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/GetterBenchmark.java @@ -3,11 +3,7 @@ import com.github.chrisgleissner.benchmarks.AbstractBenchmark; import lombok.AllArgsConstructor; import lombok.Data; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.*; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; @@ -27,7 +23,10 @@ import static java.lang.invoke.MethodHandles.privateLookupIn; import static java.lang.invoke.MethodType.methodType; import static java.util.Collections.shuffle; +import static java.util.concurrent.TimeUnit.SECONDS; +@Warmup(iterations = 50, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) public class GetterBenchmark extends AbstractBenchmark { @Benchmark diff --git a/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/SetterBenchmark.java b/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/SetterBenchmark.java index 179c5b9..1889b95 100644 --- a/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/SetterBenchmark.java +++ b/src/main/java/com/github/chrisgleissner/benchmarks/fieldaccess/SetterBenchmark.java @@ -3,11 +3,7 @@ import com.github.chrisgleissner.benchmarks.AbstractBenchmark; import lombok.AllArgsConstructor; import lombok.Data; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.*; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; @@ -28,7 +24,10 @@ import static java.lang.invoke.MethodType.methodType; import static java.util.Collections.shuffle; import static java.util.stream.Collectors.toList; +import static java.util.concurrent.TimeUnit.*; +@Warmup(iterations = 50, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) public class SetterBenchmark extends AbstractBenchmark { @Benchmark From d97a12aa5fcdaabc2eb5835eb5d238e6cddb6baf Mon Sep 17 00:00:00 2001 From: lkryukova Date: Thu, 31 Dec 2020 07:04:55 -0800 Subject: [PATCH 4/4] Added statistical files with variation of all tests and only unstable tests. --- unstable.log | 17 ++++ variation.log | 257 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 unstable.log create mode 100644 variation.log diff --git a/unstable.log b/unstable.log new file mode 100644 index 0000000..4a3c71a --- /dev/null +++ b/unstable.log @@ -0,0 +1,17 @@ +Benchmark;variation in run 1 (%);variation in run 2 (%);run to run variation (%) +collection.CollectionAddBenchmark.ConcurrentLinkedQueue;39.998;51.732;9.58 +collection.CollectionAddBenchmark.HashMap;15.619;16.734;1.54 +collection.CollectionAddBenchmark.HashSet;40.322;38.234;3.06 +collection.CollectionAddBenchmark.LinkedHashMap;16.818;15.714;1.39 +collection.CollectionAddBenchmark.LinkedHashSet;66.645;54.461;5.82 +collection.CollectionAddBenchmark.LinkedList;96.142;53.016;23.88 +collection.CollectionAddBenchmark.LinkedTransferQueue;50.284;60.076;14.99 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedDequeAdd;35.25;33.276;5.66 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedDequeGet;20.054;29.181;1.33 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedQueueAdd;11.595;20.28;7.54 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedQueueGet;12.407;33.209;3.83 +concurrent.ConcCollectionBenchmark.CopyOnWriteArraySetAdd;8.412;13.392;2.13 +concurrent.NotifyBenchmark.countdownLatch;25.479;38.899;0.86 + + + diff --git a/variation.log b/variation.log new file mode 100644 index 0000000..d9968dc --- /dev/null +++ b/variation.log @@ -0,0 +1,257 @@ +Benchmark;Variation in run 1 (%);Variation in run 2 (%);run to run variation (%) +collection.ArrayAddBenchmark.intArrayAdd;0.57;0.35;0.07 +collection.ArrayAddBenchmark.intArrayClone;1.17;0.96;0.55 +collection.ArrayAddBenchmark.intArrayCopyOf;0.29;1.12;0.29 +collection.ArrayAddBenchmark.intWrapperArrayAdd;0.27;0.87;0.58 +collection.ArrayAddBenchmark.intWrapperArrayClone;0.95;0.27;0.34 +collection.ArrayAddBenchmark.intWrapperArrayCopyOf;0.59;0.70;0.12 +collection.ArrayAddBenchmark.longArrayAdd;0.44;0.72;0.63 +collection.ArrayAddBenchmark.longArrayClone;0.31;0.53;0.17 +collection.ArrayAddBenchmark.longArrayCopyOf;0.59;0.24;0.79 +collection.ArrayAddBenchmark.longWrapperArrayAdd;0.35;0.34;0.05 +collection.ArrayAddBenchmark.longWrapperArrayClone;0.47;0.28;0.14 +collection.ArrayAddBenchmark.longWrapperArrayCopyOf;0.97;0.88;0.06 +collection.CollectionAddBenchmark.ArrayBlockingQueue;0.29;0.30;0.16 +collection.CollectionAddBenchmark.ArrayDeque;1.21;1.28;1.36 +collection.CollectionAddBenchmark.ArrayList;1.44;1.31;0.01 +collection.CollectionAddBenchmark.ArrayListNoResize;0.54;0.69;0.09 +collection.CollectionAddBenchmark.ConcurrentHashMap;4.14;3.98;0.20 +collection.CollectionAddBenchmark.ConcurrentLinkedDeque;0.55;11.21;2.72 +collection.CollectionAddBenchmark.ConcurrentLinkedQueue;40.00;51.73;9.58 +collection.CollectionAddBenchmark.ConcurrentSkipListMap;1.55;1.46;0.55 +collection.CollectionAddBenchmark.ConcurrentSkipListSet;5.78;3.89;2.76 +collection.CollectionAddBenchmark.CopyOnWriteArrayList;0.65;0.78;0.78 +collection.CollectionAddBenchmark.CopyOnWriteArraySet;0.64;0.74;0.06 +collection.CollectionAddBenchmark.HashMap;15.62;16.73;1.54 +collection.CollectionAddBenchmark.HashSet;40.32;38.23;3.06 +collection.CollectionAddBenchmark.LinkedBlockingDeque;0.95;1.75;1.65 +collection.CollectionAddBenchmark.LinkedBlockingQueue;1.63;1.46;9.77 +collection.CollectionAddBenchmark.LinkedHashMap;16.82;15.71;1.39 +collection.CollectionAddBenchmark.LinkedHashSet;66.65;54.46;5.82 +collection.CollectionAddBenchmark.LinkedList;96.14;53.02;23.88 +collection.CollectionAddBenchmark.LinkedTransferQueue;50.28;60.08;14.99 +collection.CollectionAddBenchmark.PriorityBlockingQueue;3.98;4.03;2.92 +collection.CollectionAddBenchmark.PriorityQueue;7.29;8.54;0.29 +collection.CollectionAddBenchmark.Stack;0.44;0.96;0.09 +collection.CollectionAddBenchmark.TreeMap;1.15;2.61;0.76 +collection.CollectionAddBenchmark.TreeSet;2.73;0.88;0.42 +collection.CollectionAddBenchmark.Vector;0.52;0.42;0.26 +collection.CollectionAddBenchmark.VectorNoResize;0.06;0.06;0.00 +collection.CollectionIterateBenchmark.ArrayBlockingQueue;0.10;0.08;1.64 +collection.CollectionIterateBenchmark.ArrayDeque;0.14;0.40;0.17 +collection.CollectionIterateBenchmark.ArrayList;0.68;0.68;0.20 +collection.CollectionIterateBenchmark.ArrayListNoResize;0.64;0.69;0.17 +collection.CollectionIterateBenchmark.ConcurrentHashMap;0.53;0.55;0.43 +collection.CollectionIterateBenchmark.ConcurrentLinkedDeque;0.37;0.21;0.03 +collection.CollectionIterateBenchmark.ConcurrentLinkedQueue;1.73;2.14;0.42 +collection.CollectionIterateBenchmark.ConcurrentSkipListMap;1.23;0.87;0.02 +collection.CollectionIterateBenchmark.ConcurrentSkipListSet;2.93;2.94;0.31 +collection.CollectionIterateBenchmark.CopyOnWriteArrayList;0.12;0.06;0.03 +collection.CollectionIterateBenchmark.CopyOnWriteArraySet;0.15;0.14;0.01 +collection.CollectionIterateBenchmark.HashMap;0.07;0.08;5.29 +collection.CollectionIterateBenchmark.HashSet;0.14;0.09;0.01 +collection.CollectionIterateBenchmark.LinkedBlockingDeque;0.05;0.06;0.03 +collection.CollectionIterateBenchmark.LinkedBlockingQueue;4.26;4.25;0.01 +collection.CollectionIterateBenchmark.LinkedHashMap;0.12;0.18;0.13 +collection.CollectionIterateBenchmark.LinkedHashSet;1.00;0.92;0.18 +collection.CollectionIterateBenchmark.LinkedList;1.37;1.03;0.09 +collection.CollectionIterateBenchmark.LinkedTransferQueue;0.22;0.07;0.16 +collection.CollectionIterateBenchmark.PriorityBlockingQueue;0.45;0.71;0.08 +collection.CollectionIterateBenchmark.PriorityQueue;0.73;0.63;0.17 +collection.CollectionIterateBenchmark.Stack;0.04;0.06;0.01 +collection.CollectionIterateBenchmark.TreeMap;1.07;0.98;0.61 +collection.CollectionIterateBenchmark.TreeSet;5.12;3.81;1.67 +collection.CollectionIterateBenchmark.Vector;0.03;0.03;0.00 +collection.CollectionIterateBenchmark.VectorNoResize;0.03;1.13;0.24 +collection.StreamBenchmark.integerFor;0.34;0.32;0.04 +collection.StreamBenchmark.integerFor;0.15;0.18;2.58 +collection.StreamBenchmark.integerFor;0.14;0.15;0.18 +collection.StreamBenchmark.integerFor;0.09;0.13;6.23 +collection.StreamBenchmark.integerFor;0.31;0.18;3.32 +collection.StreamBenchmark.integerFor;1.98;2.23;0.78 +collection.StreamBenchmark.integerFor;0.65;0.54;1.16 +collection.StreamBenchmark.integerFor;5.14;4.33;0.81 +collection.StreamBenchmark.integerStreamCollect;0.29;0.19;0.26 +collection.StreamBenchmark.integerStreamCollect;0.21;0.20;0.22 +collection.StreamBenchmark.integerStreamCollect;0.14;0.14;1.29 +collection.StreamBenchmark.integerStreamCollect;0.12;0.14;1.65 +collection.StreamBenchmark.integerStreamCollect;0.10;0.14;0.06 +collection.StreamBenchmark.integerStreamCollect;4.96;5.88;1.72 +collection.StreamBenchmark.integerStreamCollect;0.69;0.56;0.26 +collection.StreamBenchmark.integerStreamCollect;4.41;4.62;0.08 +collection.StreamBenchmark.integerStreamCollectParallel;0.32;0.27;0.23 +collection.StreamBenchmark.integerStreamCollectParallel;0.47;0.20;4.63 +collection.StreamBenchmark.integerStreamCollectParallel;0.23;0.35;2.70 +collection.StreamBenchmark.integerStreamCollectParallel;0.23;0.41;1.16 +collection.StreamBenchmark.integerStreamCollectParallel;0.34;0.34;0.00 +collection.StreamBenchmark.integerStreamCollectParallel;2.60;2.78;0.03 +collection.StreamBenchmark.integerStreamCollectParallel;0.61;0.71;0.16 +collection.StreamBenchmark.integerStreamCollectParallel;2.75;2.16;0.10 +collection.StreamBenchmark.integerStreamForEach;0.23;0.21;1.17 +collection.StreamBenchmark.integerStreamForEach;0.16;0.15;1.36 +collection.StreamBenchmark.integerStreamForEach;0.17;0.11;20.14 +collection.StreamBenchmark.integerStreamForEach;0.15;0.13;0.48 +collection.StreamBenchmark.integerStreamForEach;0.19;0.20;0.09 +collection.StreamBenchmark.integerStreamForEach;6.69;6.56;0.03 +collection.StreamBenchmark.integerStreamForEach;0.69;0.83;0.37 +collection.StreamBenchmark.integerStreamForEach;4.24;4.56;0.06 +collection.StreamBenchmark.integerStreamForEachParallel;0.44;0.34;1.01 +collection.StreamBenchmark.integerStreamForEachParallel;1.76;1.84;0.85 +collection.StreamBenchmark.integerStreamForEachParallel;0.13;0.76;0.04 +collection.StreamBenchmark.integerStreamForEachParallel;0.24;0.12;1.56 +collection.StreamBenchmark.integerStreamForEachParallel;2.55;1.32;1.10 +collection.StreamBenchmark.integerStreamForEachParallel;1.50;0.77;0.66 +collection.StreamBenchmark.integerStreamForEachParallel;1.58;1.61;0.83 +collection.StreamBenchmark.integerStreamForEachParallel;4.70;4.11;0.94 +collection.StreamBenchmark.integerStreamGroupBy;0.52;0.47;0.02 +collection.StreamBenchmark.integerStreamGroupBy;0.30;0.23;0.04 +collection.StreamBenchmark.integerStreamGroupBy;0.14;0.11;0.12 +collection.StreamBenchmark.integerStreamGroupBy;0.15;0.18;0.01 +collection.StreamBenchmark.integerStreamGroupBy;0.13;0.14;0.22 +collection.StreamBenchmark.integerStreamGroupBy;2.13;2.17;0.36 +collection.StreamBenchmark.integerStreamGroupBy;0.43;0.50;0.35 +collection.StreamBenchmark.integerStreamGroupBy;3.43;3.71;0.50 +collection.StreamBenchmark.integerStreamGroupByParallel;0.36;0.37;0.66 +collection.StreamBenchmark.integerStreamGroupByParallel;0.40;0.55;0.56 +collection.StreamBenchmark.integerStreamGroupByParallel;0.79;1.25;3.19 +collection.StreamBenchmark.integerStreamGroupByParallel;0.23;0.16;0.23 +collection.StreamBenchmark.integerStreamGroupByParallel;0.38;0.47;0.06 +collection.StreamBenchmark.integerStreamGroupByParallel;1.67;2.36;0.61 +collection.StreamBenchmark.integerStreamGroupByParallel;0.73;1.38;0.41 +collection.StreamBenchmark.integerStreamGroupByParallel;1.53;1.26;0.07 +collection.StreamBenchmark.integerStreamMax;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamMax;0.03;0.00;1.97 +collection.StreamBenchmark.integerStreamMax;0.02;0.50;5.36 +collection.StreamBenchmark.integerStreamMax;0.00;0.01;0.19 +collection.StreamBenchmark.integerStreamMax;0.00;0.00;0.16 +collection.StreamBenchmark.integerStreamMax;0.65;0.91;0.29 +collection.StreamBenchmark.integerStreamMax;1.32;1.39;0.32 +collection.StreamBenchmark.integerStreamMax;0.18;0.18;0.20 +collection.StreamBenchmark.integerStreamMaxParallel;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamMaxParallel;0.58;0.36;2.64 +collection.StreamBenchmark.integerStreamMaxParallel;1.47;0.17;2.84 +collection.StreamBenchmark.integerStreamMaxParallel;0.39;0.32;1.84 +collection.StreamBenchmark.integerStreamMaxParallel;0.25;0.45;1.39 +collection.StreamBenchmark.integerStreamMaxParallel;0.33;0.07;0.46 +collection.StreamBenchmark.integerStreamMaxParallel;0.13;0.60;0.68 +collection.StreamBenchmark.integerStreamMaxParallel;1.29;0.81;1.71 +collection.StreamBenchmark.integerStreamReduceToSum;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamReduceToSum;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamReduceToSum;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamReduceToSum;0.00;0.00;0.04 +collection.StreamBenchmark.integerStreamReduceToSum;0.00;0.00;0.04 +collection.StreamBenchmark.integerStreamReduceToSum;0.52;0.21;1.80 +collection.StreamBenchmark.integerStreamReduceToSum;0.31;0.33;0.20 +collection.StreamBenchmark.integerStreamReduceToSum;0.28;0.27;0.07 +collection.StreamBenchmark.integerStreamReduceToSumParallel;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamReduceToSumParallel;1.02;0.48;0.82 +collection.StreamBenchmark.integerStreamReduceToSumParallel;0.49;0.37;4.10 +collection.StreamBenchmark.integerStreamReduceToSumParallel;0.32;0.19;2.81 +collection.StreamBenchmark.integerStreamReduceToSumParallel;0.34;0.37;0.81 +collection.StreamBenchmark.integerStreamReduceToSumParallel;0.60;0.61;1.15 +collection.StreamBenchmark.integerStreamReduceToSumParallel;1.81;1.67;1.10 +collection.StreamBenchmark.integerStreamReduceToSumParallel;1.61;1.68;0.58 +collection.StreamBenchmark.integerStreamSum;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamSum;0.00;0.21;0.09 +collection.StreamBenchmark.integerStreamSum;0.07;0.00;0.02 +collection.StreamBenchmark.integerStreamSum;0.01;0.01;0.11 +collection.StreamBenchmark.integerStreamSum;0.00;0.00;0.03 +collection.StreamBenchmark.integerStreamSum;4.76;2.98;7.04 +collection.StreamBenchmark.integerStreamSum;3.97;4.30;1.73 +collection.StreamBenchmark.integerStreamSum;0.09;0.09;15.34 +collection.StreamBenchmark.integerStreamSumParallel;0.00;0.00;0.00 +collection.StreamBenchmark.integerStreamSumParallel;0.42;0.51;0.93 +collection.StreamBenchmark.integerStreamSumParallel;1.67;0.44;3.78 +collection.StreamBenchmark.integerStreamSumParallel;0.51;0.34;2.02 +collection.StreamBenchmark.integerStreamSumParallel;0.14;0.14;1.29 +collection.StreamBenchmark.integerStreamSumParallel;0.13;0.06;1.32 +collection.StreamBenchmark.integerStreamSumParallel;0.13;0.17;2.24 +collection.StreamBenchmark.integerStreamSumParallel;0.82;0.57;2.81 +collection.StreamBenchmark.intFor;0.63;0.81;0.05 +collection.StreamBenchmark.intFor;0.64;0.19;2.50 +collection.StreamBenchmark.intFor;0.24;0.24;1.59 +collection.StreamBenchmark.intFor;0.02;0.02;1.31 +collection.StreamBenchmark.intFor;0.04;0.05;0.45 +collection.StreamBenchmark.intFor;0.02;0.02;0.02 +collection.StreamBenchmark.intFor;0.35;0.59;0.49 +collection.StreamBenchmark.intFor;0.16;0.17;0.03 +collection.StreamBenchmark.intStream;0.26;0.26;0.43 +collection.StreamBenchmark.intStream;0.45;0.32;1.07 +collection.StreamBenchmark.intStream;0.13;0.14;2.00 +collection.StreamBenchmark.intStream;0.10;0.07;8.23 +collection.StreamBenchmark.intStream;0.05;0.06;0.14 +collection.StreamBenchmark.intStream;8.12;9.05;0.60 +collection.StreamBenchmark.intStream;0.07;0.07;0.27 +collection.StreamBenchmark.intStream;0.16;0.13;0.08 +collection.StreamBenchmark.intStreamParallel;0.31;0.27;0.14 +collection.StreamBenchmark.intStreamParallel;0.51;0.63;0.08 +collection.StreamBenchmark.intStreamParallel;0.35;0.35;4.24 +collection.StreamBenchmark.intStreamParallel;0.20;0.14;0.93 +collection.StreamBenchmark.intStreamParallel;0.27;0.29;6.85 +collection.StreamBenchmark.intStreamParallel;0.29;0.32;3.08 +collection.StreamBenchmark.intStreamParallel;2.08;1.96;1.14 +collection.StreamBenchmark.intStreamParallel;0.87;1.19;3.59 +concurrent.ConcCollectionBenchmark.ArrayBlockingQueueAdd;0.81;0.65;0.16 +concurrent.ConcCollectionBenchmark.ArrayBlockingQueueGet;1.26;2.56;0.80 +concurrent.ConcCollectionBenchmark.ArrayBlockingQueueRemove;0.19;0.23;0.00 +concurrent.ConcCollectionBenchmark.ArrayListAdd;0.12;0.18;0.08 +concurrent.ConcCollectionBenchmark.ArrayListGet;0.56;0.68;0.04 +concurrent.ConcCollectionBenchmark.ArrayListRemove;5.05;2.07;0.51 +concurrent.ConcCollectionBenchmark.ConcurrentHashMapAdd;0.44;0.26;0.01 +concurrent.ConcCollectionBenchmark.ConcurrentHashMapGet;1.21;0.70;0.24 +concurrent.ConcCollectionBenchmark.ConcurrentHashMapRemove;0.09;4.44;0.89 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedDequeAdd;35.25;33.28;5.66 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedDequeGet;20.05;29.18;1.33 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedDequeRemove;0.02;0.90;2.80 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedQueueAdd;11.60;20.28;7.54 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedQueueGet;12.41;33.21;3.83 +concurrent.ConcCollectionBenchmark.ConcurrentLinkedQueueRemove;0.24;0.01;0.04 +concurrent.ConcCollectionBenchmark.ConcurrentSkipListMapAdd;2.04;2.13;0.21 +concurrent.ConcCollectionBenchmark.ConcurrentSkipListMapRemove;4.70;4.96;2.95 +concurrent.ConcCollectionBenchmark.ConcurrentSkipListSetAdd;1.52;2.21;2.93 +concurrent.ConcCollectionBenchmark.ConcurrentSkipListSetGet;0.75;1.30;0.44 +concurrent.ConcCollectionBenchmark.ConcurrentSkipLPistSetRemove;3.39;3.00;0.75 +concurrent.ConcCollectionBenchmark.ConcurrentSkipMapSetGet;3.62;3.12;0.37 +concurrent.ConcCollectionBenchmark.CopyOnWriteArrayAdd;5.40;8.14;1.05 +concurrent.ConcCollectionBenchmark.CopyOnWriteArrayListGet;0.35;0.31;0.20 +concurrent.ConcCollectionBenchmark.CopyOnWriteArrayRemove;4.58;4.13;1.05 +concurrent.ConcCollectionBenchmark.CopyOnWriteArraySetAdd;8.41;13.39;2.13 +concurrent.ConcCollectionBenchmark.CopyOnWriteArraySetGet;0.47;0.74;0.13 +concurrent.ConcCollectionBenchmark.CopyOnWriteArraySetRemove;1.71;0.86;5.30 +concurrent.ConcCollectionBenchmark.VectorAdd;14.40;1.87;8.21 +concurrent.ConcCollectionBenchmark.VectorGet;0.31;1.05;0.54 +concurrent.ConcCollectionBenchmark.VectorRemove;2.18;2.14;1.49 +concurrent.NotifyBenchmark.countdownLatch;25.48;38.90;0.86 +concurrent.NotifyBenchmark.cyclicBarrier;7.65;6.30;6.11 +concurrent.NotifyBenchmark.phaser;4.33;8.19;0.92 +concurrent.NotifyBenchmark.waitNotify;4.81;6.20;1.98 +CounterBenchmark.atomicInteger;0.00;0.00;0.00 +CounterBenchmark.atomicLong;0.01;0.01;0.00 +CounterBenchmark.mutableInt;0.03;0.03;0.02 +CounterBenchmark.mutableLong;0.00;0.01;0.00 +CounterBenchmark.primitiveInt;0.02;0.07;0.02 +CounterBenchmark.primitiveLong;0.04;0.00;5.69 +fieldaccess.GetterBenchmark.direct;0.06;0.06;0.00 +fieldaccess.GetterBenchmark.lambdaMetaFactoryForGetter;0.02;0.07;0.04 +fieldaccess.GetterBenchmark.methodHandleForField;0.26;0.41;0.65 +fieldaccess.GetterBenchmark.methodHandleForGetter;0.06;0.09;0.00 +fieldaccess.GetterBenchmark.reflectionField;0.03;0.09;0.11 +fieldaccess.GetterBenchmark.reflectionGetter;1.38;1.59;0.24 +fieldaccess.GetterBenchmark.varHandle;1.00;0.54;0.97 +fieldaccess.SetterBenchmark.direct;0.25;0.23;0.03 +fieldaccess.SetterBenchmark.lambdaMetaFactoryForSetter;0.25;0.22;0.03 +fieldaccess.SetterBenchmark.methodHandleForField;0.39;0.21;0.06 +fieldaccess.SetterBenchmark.methodHandleForSetter;0.49;0.47;0.53 +fieldaccess.SetterBenchmark.reflectionField;0.09;0.08;0.02 +fieldaccess.SetterBenchmark.reflectionSetter;1.03;1.34;0.13 +fieldaccess.SetterBenchmark.varHandle;0.09;0.06;0.05 +ObjectCacheBenchmark.cache;0.27;0.12;1.34 +ObjectCacheBenchmark.constructor;0.96;0.06;0.58 +ObjectCacheBenchmark.primitive;0.12;1.14;0.43 +TimeBenchmark.currentTimeMillis;0.05;0.03;0.02 +TimeBenchmark.localDate;1.83;2.22;0.32 +TimeBenchmark.localDateTime;2.63;1.78;0.17 +TimeBenchmark.nanoTime;0.01;0.00;0.01 +TimeBenchmark.zonedDateTime;1.05;2.59;0.28 + +