Benchmarks

.bsp/sbt.json

@@ -0,0 +1 @@
+{"name":"sbt","version":"1.4.7","bspVersion":"2.0.0-M5","languages":["scala"],"argv":["/Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/paualarco/Library/Application Support/JetBrains/IdeaIC2020.3/plugins/Scala/launcher/sbt-launch.jar","xsbt.boot.Boot","-bsp"]}

benchmarks/consumer-1.txt

@@ -0,0 +1,142 @@
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_auto_commit
+
+# Run progress: 0.00% complete, ETA 00:07:20
+# Fork: 1 of 1
+# Warmup Iteration   1: 9.643 ops/s
+Iteration   1: 13.072 ops/s
+Iteration   2: 12.147 ops/s
+Iteration   3: 11.022 ops/s
+Iteration   4: 8.540 ops/s
+Iteration   5: 9.754 ops/s
+Iteration   6: 10.663 ops/s
+Iteration   7: 11.656 ops/s
+Iteration   8: 12.694 ops/s
+Iteration   9: 11.146 ops/s
+Iteration  10: 11.898 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_auto_commit":
+  11.259 ±(99.9%) 2.065 ops/s [Average]
+  (min, avg, max) = (8.540, 11.259, 13.072), stdev = 1.366
+  CI (99.9%): [9.195, 13.324] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit
+
+# Run progress: 25.00% complete, ETA 00:06:06
+# Fork: 1 of 1
+# Warmup Iteration   1: <failure>
+
+java.lang.NoSuchMethodError: monix.kafka.benchmarks.generated.ConsumerBenchmark_jmhType.monix_manual_commit()V
+	at monix.kafka.benchmarks.generated.ConsumerBenchmark_monix_manual_commit_jmhTest.monix_manual_commit_thrpt_jmhStub(ConsumerBenchmark_monix_manual_commit_jmhTest.java:119)
+	at monix.kafka.benchmarks.generated.ConsumerBenchmark_monix_manual_commit_jmhTest.monix_manual_commit_Throughput(ConsumerBenchmark_monix_manual_commit_jmhTest.java:83)
+	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+	at java.lang.reflect.Method.invoke(Method.java:498)
+	at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:453)
+	at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:437)
+	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
+	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
+	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
+	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+	at java.lang.Thread.run(Thread.java:748)
+
+
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1
+
+# Run progress: 50.00% complete, ETA 00:02:13
+# Fork: 1 of 1
+# Warmup Iteration   1: 7.288 ops/s
+Iteration   1: 9.133 ops/s
+Iteration   2: 9.689 ops/s
+Iteration   3: 10.091 ops/s
+Iteration   4: 10.151 ops/s
+Iteration   5: 10.438 ops/s
+Iteration   6: 10.438 ops/s
+Iteration   7: 10.539 ops/s
+Iteration   8: 10.347 ops/s
+Iteration   9: 8.787 ops/s
+Iteration  10: 9.061 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1":
+  9.867 ±(99.9%) 0.989 ops/s [Average]
+  (min, avg, max) = (8.787, 9.867, 10.539), stdev = 0.654
+  CI (99.9%): [8.878, 10.856] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1000
+
+# Run progress: 75.00% complete, ETA 00:01:24
+# Fork: 1 of 1
+# Warmup Iteration   1: 0.980 ops/s
+Iteration   1: 0.991 ops/s
+Iteration   2: 0.991 ops/s
+Iteration   3: 0.993 ops/s
+Iteration   4: 0.993 ops/s
+Iteration   5: 0.993 ops/s
+Iteration   6: 0.993 ops/s
+Iteration   7: 0.994 ops/s
+Iteration   8: 0.994 ops/s
+Iteration   9: 0.994 ops/s
+Iteration  10: 0.994 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1000":
+  0.993 ±(99.9%) 0.002 ops/s [Average]
+  (min, avg, max) = (0.991, 0.993, 0.994), stdev = 0.001
+  CI (99.9%): [0.991, 0.995] (assumes normal distribution)
+
+
+# Run complete. Total time: 00:06:15
+
+REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on
+why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial
+experiments, perform baseline and negative tests that provide experimental control, make sure
+the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts.
+Do not assume the numbers tell you what you want them to tell.
+
+Benchmark                                             Mode  Cnt   Score   Error  Units
+ConsumerBenchmark.monix_auto_commit                  thrpt   10  11.259 ± 2.065  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat1     thrpt   10   9.867 ± 0.989  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat1000  thrpt   10   0.993 ± 0.002  ops/s

benchmarks/consumer-master.txt

@@ -0,0 +1,42 @@
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit
+
+# Run progress: 0.00% complete, ETA 00:01:50
+# Fork: 1 of 1
+# Warmup Iteration   1: 9.483 ops/s
+Iteration   1: 12.133 ops/s
+Iteration   2: 10.040 ops/s
+Iteration   3: 7.764 ops/s
+Iteration   4: 11.043 ops/s
+Iteration   5: 11.178 ops/s
+Iteration   6: 11.964 ops/s
+Iteration   7: 13.490 ops/s
+Iteration   8: 13.681 ops/s
+Iteration   9: 13.426 ops/s
+Iteration  10: 13.665 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit":
+  11.838 ±(99.9%) 2.894 ops/s [Average]
+  (min, avg, max) = (7.764, 11.838, 13.681), stdev = 1.914
+  CI (99.9%): [8.944, 14.733] (assumes normal distribution)
+
+
+# Run complete. Total time: 00:01:57
+
+REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on
+why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial
+experiments, perform baseline and negative tests that provide experimental control, make sure
+the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts.
+Do not assume the numbers tell you what you want them to tell.
+
+Benchmark                               Mode  Cnt   Score   Error  Units
+ConsumerBenchmark.monix_manual_commit  thrpt   10  11.838 ± 2.894  ops/s

benchmarks/consumer2.txt

@@ -0,0 +1,174 @@
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_auto_commit
+
+# Run progress: 0.00% complete, ETA 00:09:10
+# Fork: 1 of 1
+# Warmup Iteration   1: 8.696 ops/s
+Iteration   1: 9.784 ops/s
+Iteration   2: 12.588 ops/s
+Iteration   3: 12.075 ops/s
+Iteration   4: 8.459 ops/s
+Iteration   5: 10.894 ops/s
+Iteration   6: 9.174 ops/s
+Iteration   7: 11.828 ops/s
+Iteration   8: 10.768 ops/s
+Iteration   9: 9.139 ops/s
+Iteration  10: 10.985 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_auto_commit":
+  10.569 ±(99.9%) 2.096 ops/s [Average]
+  (min, avg, max) = (8.459, 10.569, 12.588), stdev = 1.387
+  CI (99.9%): [8.473, 12.666] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1
+
+# Run progress: 20.00% complete, ETA 00:08:09
+# Fork: 1 of 1
+# Warmup Iteration   1: 6.852 ops/s
+Iteration   1: 11.218 ops/s
+Iteration   2: 9.514 ops/s
+Iteration   3: 10.431 ops/s
+Iteration   4: 9.078 ops/s
+Iteration   5: 9.250 ops/s
+Iteration   6: 10.335 ops/s
+Iteration   7: 9.936 ops/s
+Iteration   8: 12.642 ops/s
+Iteration   9: 9.415 ops/s
+Iteration  10: 11.382 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1":
+  10.320 ±(99.9%) 1.720 ops/s [Average]
+  (min, avg, max) = (9.078, 10.320, 12.642), stdev = 1.137
+  CI (99.9%): [8.601, 12.040] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat100
+
+# Run progress: 40.00% complete, ETA 00:06:06
+# Fork: 1 of 1
+# Warmup Iteration   1: 4.483 ops/s
+Iteration   1: 4.252 ops/s
+Iteration   2: 4.466 ops/s
+Iteration   3: 4.261 ops/s
+Iteration   4: 4.578 ops/s
+Iteration   5: 4.676 ops/s
+Iteration   6: 4.380 ops/s
+Iteration   7: 4.597 ops/s
+Iteration   8: 4.703 ops/s
+Iteration   9: 4.526 ops/s
+Iteration  10: 4.745 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat100":
+  4.518 ±(99.9%) 0.266 ops/s [Average]
+  (min, avg, max) = (4.252, 4.518, 4.745), stdev = 0.176
+  CI (99.9%): [4.252, 4.784] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1000
+
+# Run progress: 60.00% complete, ETA 00:04:04
+# Fork: 1 of 1
+# Warmup Iteration   1: 0.977 ops/s
+Iteration   1: 0.992 ops/s
+Iteration   2: 0.994 ops/s
+Iteration   3: 0.992 ops/s
+Iteration   4: 0.992 ops/s
+Iteration   5: 0.995 ops/s
+Iteration   6: 0.994 ops/s
+Iteration   7: 0.994 ops/s
+Iteration   8: 0.995 ops/s
+Iteration   9: 0.995 ops/s
+Iteration  10: 0.994 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat1000":
+  0.994 ±(99.9%) 0.002 ops/s [Average]
+  (min, avg, max) = (0.992, 0.994, 0.995), stdev = 0.001
+  CI (99.9%): [0.992, 0.995] (assumes normal distribution)
+
+
+# JMH version: 1.21
+# VM version: JDK 1.8.0_221, Java HotSpot(TM) 64-Bit Server VM, 25.221-b11
+# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/bin/java
+# VM options: <none>
+# Warmup: 1 iterations, 10 s each
+# Measurement: 10 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat3000
+
+# Run progress: 80.00% complete, ETA 00:02:02
+# Fork: 1 of 1
+# Warmup Iteration   1: 0.329 ops/s
+Iteration   1: 0.332 ops/s
+Iteration   2: 0.331 ops/s
+Iteration   3: 0.332 ops/s
+Iteration   4: 0.332 ops/s
+Iteration   5: 0.332 ops/s
+Iteration   6: 0.333 ops/s
+Iteration   7: 0.333 ops/s
+Iteration   8: 0.333 ops/s
+Iteration   9: 0.332 ops/s
+Iteration  10: 0.333 ops/s
+
+
+Result "monix.kafka.benchmarks.ConsumerBenchmark.monix_manual_commit_heartbeat3000":
+  0.332 ±(99.9%) 0.001 ops/s [Average]
+  (min, avg, max) = (0.331, 0.332, 0.333), stdev = 0.001
+  CI (99.9%): [0.332, 0.333] (assumes normal distribution)
+
+
+# Run complete. Total time: 00:10:31
+
+REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on
+why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial
+experiments, perform baseline and negative tests that provide experimental control, make sure
+the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts.
+Do not assume the numbers tell you what you want them to tell.
+
+Benchmark                                             Mode  Cnt   Score   Error  Units
+ConsumerBenchmark.monix_auto_commit                  thrpt   10  10.569 ± 2.096  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat1     thrpt   10  10.320 ± 1.720  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat100   thrpt   10   4.518 ± 0.266  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat1000  thrpt   10   0.994 ± 0.002  ops/s
+ConsumerBenchmark.monix_manual_commit_heartbeat3000  thrpt   10   0.332 ± 0.001  ops/s