Added Memory Limitation Exploration

.gitignore

@@ -8,6 +8,7 @@ graph.xml
 gsql_client.*
 *.txt
 *.csv
+.local
 /workspace
 /results
 

build.sbt

@@ -69,8 +69,6 @@ publishMavenStyle := true
 
 lazy val datasetDir = taskKey[File]("Dataset directory")
 datasetDir := baseDirectory.value / "workspace" / "defects4j"
-lazy val driversToBenchmark = taskKey[Seq[String]]("Drivers to benchmark")
-driversToBenchmark := Seq("overflowdb", "tinkergraph", "neo4j-embedded")
 
 lazy val defect4jDataset = taskKey[Seq[(String, String)]]("JARs for projects used in `defects4j`")
 defect4jDataset :=

runBenchmarks.sc

@@ -8,31 +8,41 @@ import scala.jdk.CollectionConverters.*
   "sbt compile benchmarkDownloadTask".!
 
   val datasetDir = Path.of("workspace", "defects4j")
-  val resultsDir = Path.of("results")
+  val resultsDir = Path.of("results").createIfNotExists
 
-  if (!Files.exists(resultsDir)) Files.createDirectory(resultsDir)
-
-  def benchmarkArgs(driver: String, project: String): String = {
+  def benchmarkArgs(driver: String, project: String, memGb: Int): String = {
     val projectDir = Path.of(datasetDir.toString, project)
     val projectName = project.toLowerCase.stripSuffix(".jar")
-    val resultsPath = Path.of(resultsDir.toString, s"results-$driver-$projectName")
-    val outputPath = Path.of(resultsDir.toString, s"output-$driver-$projectName")
-    s"Jmh/runMain com.github.plume.oss.Benchmark $driver $projectDir -o ${outputPath.toAbsolutePath} -r ${resultsPath.toAbsolutePath}"
+    val driverResultsDir = Path.of(resultsDir.toString, driver, projectName).createIfNotExists
+    val resultsPath = Path.of(driverResultsDir.toString, s"results-Xmx${memGb}G")
+    val outputPath = Path.of(driverResultsDir.toString, s"output-Xmx${memGb}G")
+    s"Jmh/runMain com.github.plume.oss.Benchmark $driver $projectDir -o ${outputPath.toAbsolutePath} -r ${resultsPath.toAbsolutePath} -m $memGb"
   }
 
   println("[info] Available projects:")
   val projects = Files.list(datasetDir).filter(_.toString.endsWith(".jar")).toList.asScala.toList
   projects.foreach(p => println(s" - ${p.getFileName.toString}"))
 
   println("[info] Available drivers:")
-  val drivers = Seq("overflowdb")
+  val drivers = Seq("overflowdb", "tinkergraph", "neo4j-embedded")
   drivers.foreach(d => println(s" - $d"))
 
-  drivers.foreach { driver =>
-    projects.foreach { project =>
-      val cmd = benchmarkArgs(driver, project.getFileName.toString)
-      println(s"[info] Benchmarking '$driver' on project '$project'")
-      s"sbt \"$cmd\"".!
+  val memoryConfigs = Seq(4, 8, 16)
+
+  memoryConfigs.foreach { memConfig =>
+    drivers.foreach { driver =>
+      projects.foreach { project =>
+        val cmd = benchmarkArgs(driver, project.getFileName.toString, memConfig)
+        println(s"[info] Benchmarking '$driver' on project '$project' with `-Xmx${memConfig}G`")
+        s"sbt \"$cmd\"".!
+      }
     }
   }
 }
+
+implicit class PathExt(x: Path) {
+  def createIfNotExists: Path = {
+    if (!Files.exists(x)) Files.createDirectories(x)
+    x
+  }
+}

src/main/scala/com/github/plume/oss/Benchmark.scala

@@ -72,14 +72,14 @@ object Benchmark {
       .warmupIterations(1)
       .warmupTime(TimeValue.seconds(1))
       .measurementTime(TimeValue.seconds(2))
-      .measurementIterations(3)
+      .measurementIterations(5)
       .mode(Mode.AverageTime)
       .timeUnit(TimeUnit.NANOSECONDS)
       .forks(2)
       .output(s"${config.jmhOutputFile}-$benchmarkType.txt")
       .result(s"${config.jmhResultFile}-$benchmarkType.csv")
       .param("configStr", write(config))
-      .detectJvmArgs() // inherit stuff like max heap size
+      .jvmArgsAppend(s"-Xmx${config.jmhMemoryGb}G", "-XX:+UseZGC")
   }
 
   enum BenchmarkType {

src/main/scala/com/github/plume/oss/Plume.scala

@@ -44,6 +44,15 @@ object Plume {
         .text("The target application to parse.")
         .action((x, c) => c.copy(inputDir = x))
 
+      opt[Int]('m', "jmh-memory")
+        .text(s"The -Xmx JVM argument in Gb for JMH. Default is 4 (-Xmx4G).")
+        .hidden()
+        .validate {
+          case x if x < 1 => failure("Consider at least 1Gb")
+          case _          => success
+        }
+        .action((x, c) => c.copy(jmhMemoryGb = x))
+
       opt[String]('o', "jmh-output-file")
         .text(s"The JMH output file path. Exclude file extensions.")
         .hidden()

src/main/scala/com/github/plume/oss/benchmarking/TinkerGraphReadBenchmark.scala

@@ -1,19 +1,18 @@
 package com.github.plume.oss.benchmarking
 
-import com.github.plume.oss.benchmarking.GraphReadBenchmark
 import com.github.plume.oss.drivers.TinkerGraphDriver
 import io.shiftleft.codepropertygraph.generated.EdgeTypes.AST
 import io.shiftleft.codepropertygraph.generated.NodeTypes.{CALL, METHOD}
 import io.shiftleft.codepropertygraph.generated.PropertyNames.{FULL_NAME, ORDER}
 import org.apache.tinkerpop.gremlin.process.traversal.P
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.{GraphTraversalSource, __}
-import org.openjdk.jmh.annotations.{Benchmark, Scope, Setup, State}
+import org.openjdk.jmh.annotations.*
 import org.openjdk.jmh.infra.{BenchmarkParams, Blackhole}
+import overflowdb.traversal.*
 
-import scala.jdk.CollectionConverters.*
 import scala.compiletime.uninitialized
+import scala.jdk.CollectionConverters.*
 import scala.util.Random
-import overflowdb.traversal.*
 
 @State(Scope.Benchmark)
 class TinkerGraphReadBenchmark extends GraphReadBenchmark {

src/main/scala/com/github/plume/oss/package.scala

@@ -8,6 +8,7 @@ package object oss {
 
   case class PlumeConfig(
     inputDir: String = "",
+    jmhMemoryGb: Int = 4,
     jmhOutputFile: String = File.newTemporaryFile("plume-jmh-output-").pathAsString,
     jmhResultFile: String = File.newTemporaryFile("plume-jmh-result-").pathAsString,
     dbConfig: DatabaseConfig = OverflowDbConfig()