Pass & Filter pattern (sbt#1323)

Filter pattern in GlobalBenchmarkSetup

Test that -Dbenchmark.pattern works in CI

Pass pattern to GlobalBenchmarkSetup

Run benchmarks in parallel

Renaming job names

Author: Friendseeker

.github/workflows/ci.yml

@@ -27,6 +27,9 @@ jobs:
           - os: ubuntu-latest
             java: 21
             jobtype: 4
+          - os: ubuntu-latest
+            java: 21
+            jobtype: 5
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -50,20 +53,31 @@ jobs:
       if: ${{ matrix.jobtype == 3 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 "runBenchmarks"
-    - name: Checkout Develop Branch (3)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 3 }}
+        sbt -v -Dfile.encoding=UTF-8 scalafmtCheckAll scalafmtSbtCheck
+    - name: Benchmark (Scalac) (4)
+      if: ${{ matrix.jobtype == 4 }}
+      shell: bash
+      run: |
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
+    - name: Benchmark (Shapeless) (5)
+      if: ${{ matrix.jobtype == 5 }}
+      shell: bash
+      run: |
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
+    - name: Checkout Develop Branch (4, 5)
+      if: ${{ github.event_name == 'pull_request' && (matrix.jobtype == 4 || matrix.jobtype == 5) }}
       uses: actions/checkout@v4
       with:
         clean: false
         ref: develop
-    - name: Build and test against Develop Branch (3)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 3 }}
+    - name: Benchmark (Scalac) against Develop Branch (4)
+      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 "runBenchmarks"
-    - name: Build and test (4)
-      if: ${{ matrix.jobtype == 4 }}
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
+    - name: Benchmark (Shapeless) against Develop Branch (5)
+      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 scalafmtCheckAll scalafmtSbtCheck
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
+        

build.sbt

@@ -697,11 +697,12 @@ crossTestBridges := (compilerBridgeTest.jvm(scala213) / Test / test).dependsOn(p
 addCommandAlias(
   "runBenchmarks", {
     val dir = IO.createTemporaryDirectory.getAbsolutePath
+    val pattern = sys.props.getOrElse("benchmark.pattern", "")
     Seq(
       s"${compilerBridge213.id}/packageBin",
       s"${compilerBridge212.id}/packageBin",
-      s"${zincBenchmarks.jvm(scala212).id}/Test/run $dir",
-      s"${zincBenchmarks.jvm(scala212).id}/jmh:run -p _tempDir=$dir -prof gc -foe true",
+      s"${zincBenchmarks.jvm(scala212).id}/Test/run $dir $pattern",
+      s"${zincBenchmarks.jvm(scala212).id}/jmh:run -p _tempDir=$dir -prof gc -foe true $pattern",
       s"""eval IO.delete(file("$dir"))""",
     ).mkString(";", ";", "")
   }

internal/zinc-benchmarks/src/test/scala/xsbt/GlobalBenchmarkSetup.scala

@@ -18,13 +18,15 @@ import xsbt.BenchmarkProjects.{ Scalac, Shapeless }
 object GlobalBenchmarkSetup {
 
   /** Update this list every time you add a new benchmark. */
-  val projects = List(Scalac, Shapeless)
+  val projects = Map("Scalac" -> Scalac, "Shapeless" -> Shapeless)
 
-  def runSetup(setupDir: File): (Int, String) = {
-    val projectsPreparation = projects.map { project =>
-      val benchmark = new ZincBenchmark(project)
-      project -> benchmark.writeSetup(new File(setupDir, project.repo))
-    }
+  def runSetup(setupDir: File, pattern: String): (Int, String) = {
+    val projectsPreparation = projects
+      .filterKeys { _.matches(pattern) }
+      .map { case (_, project) =>
+        val benchmark = new ZincBenchmark(project)
+        project -> benchmark.writeSetup(new File(setupDir, project.repo))
+      }
 
     val failedToPrepare = projectsPreparation.filter(_._2.isLeft)
     if (failedToPrepare.isEmpty)
@@ -43,11 +45,12 @@ object GlobalBenchmarkSetup {
 
     if (args.isEmpty)
       fail("Missing directory to host project setups.")
-    else if (args.length > 1)
+    else if (args.length > 2)
       fail("Too many arguments. Pass the directory to host project setups.")
     else {
       val setupDir = new File(args(0))
-      val (exitCode, status) = runSetup(setupDir)
+      val pattern = if (args.length == 1) ".*" else args(1)
+      val (exitCode, status) = runSetup(setupDir, pattern)
       println(status)
       println("The benchmark setup has finished.")
       System.exit(exitCode)