Add Protobuf/gRPC-based EtsIR loader

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' && github.ref != 'ref/heads/neo' }}
 
       - name: Build and run tests
-        run: ./gradlew --scan build -x :jacodb-ets:build
+        run: ./gradlew --scan build
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
@@ -122,11 +122,12 @@ jobs:
 
       - name: Set up ArkAnalyzer
         run: |
-          REPO_URL="https://gitcode.com/Lipen/arkanalyzer"
+          REPO_URL="https://gitcode.com/Lipen/arkanalyzer.git"
           DEST_DIR="arkanalyzer"
           MAX_RETRIES=10
           RETRY_DELAY=3  # Delay between retries in seconds  
-          BRANCH="neo/2025-05-30b"
+          #BRANCH="neo/2025-05-30b"
+          BRANCH="lipen/grpc"
 
           for ((i=1; i<=MAX_RETRIES; i++)); do
               git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break
@@ -147,8 +148,11 @@ jobs:
           npm install
           npm run build
 
+      - name: Enable ETS modules
+        run: echo "enableEts=true" >> local.properties
+
       - name: Run ETS tests
-        run: ./gradlew --scan :jacodb-ets:generateTestResources :jacodb-ets:test
+        run: ./gradlew --scan --stacktrace :jacodb-ets:generateTestResources :jacodb-ets:test
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3

.gitignore

@@ -1,7 +1,8 @@
 .idea/
 .gradle/
-build/
 .kotlin/
+build/
+generated/
 idea-community
 *.db
-/generated/
+/local.properties

build.gradle.kts

@@ -147,6 +147,7 @@ allprojects {
     license {
         include("**/*.kt")
         include("**/*.java")
+        exclude { it.file.startsWith(layout.buildDirectory.asFile.get()) }
         header(rootProject.file("docs/copyright/COPYRIGHT_HEADER.txt"))
     }
 }
@@ -173,7 +174,6 @@ if (!repoUrl.isNullOrEmpty()) {
             project(":jacodb-storage"),
             project(":jacodb-approximations"),
             project(":jacodb-taint-configuration"),
-            project(":jacodb-ets"),
         )
     ) {
         tasks {

buildSrc/src/main/kotlin/Dependencies.kt

@@ -6,12 +6,13 @@ object Versions {
     const val asm = "9.7.1"
     const val dokka = "1.9.20" // note: must be compatible with kotlin version
     const val gradle_download = "5.3.0"
+    const val gradle_node = "7.1.0"
+    const val gradle_protobuf = "0.9.5"
     const val gradle_versions = "0.47.0"
-
-    // hikaricp version compatible with Java 8
-    const val hikaricp = "4.0.3"
-
+    const val grpc = "1.72.0"
+    const val grpc_kotlin = "1.4.3"
     const val guava = "31.1-jre"
+    const val hikaricp = "4.0.3" // compatible with Java 8
     const val javax_activation = "1.1"
     const val javax_mail = "1.4.7"
     const val javax_servlet_api = "2.5"
@@ -24,23 +25,26 @@ object Versions {
     const val junit = "5.9.2"
     const val kotlin = "2.1.0"
     const val kotlin_logging = "1.8.3"
+    const val kotlin_metadata = kotlin
     const val kotlinx_benchmark = "0.4.6"
     const val kotlinx_cli = "0.3.5"
     const val kotlinx_collections_immutable = "0.3.5"
     const val kotlinx_coroutines = "1.6.4"
-    const val kotlin_metadata = kotlin
     const val kotlinx_serialization = "1.8.0"
     const val licenser = "0.6.1"
+    const val lmdb_java = "0.9.0"
     const val mockk = "1.13.3"
+    const val protobuf = "4.30.2"
+    const val rocks_db = "9.1.1"
     const val sarif4k = "0.5.0"
     const val shadow = "8.1.1"
-    const val slf4j = "1.7.36"
+    const val slf4j = "2.0.17"
     const val soot_utbot_fork = "4.4.0-FORK-2"
     const val sootup = "1.0.0"
     const val sqlite = "3.41.2.2"
     const val xodus = "2.0.1"
-    const val rocks_db = "9.1.1"
-    const val lmdb_java = "0.9.0"
+    const val wire = "5.3.1"
+    const val wire_grpc_server = "1.0.0-alpha04"
 
     // libs for tests only
     const val jgit_test_only_version = "5.9.0.202009080501-r"
@@ -144,11 +148,6 @@ object Libs {
     )
 
     // https://github.com/Kotlin/kotlinx.serialization
-    val kotlinx_serialization_core = dep(
-        group = "org.jetbrains.kotlinx",
-        name = "kotlinx-serialization-core",
-        version = Versions.kotlinx_serialization
-    )
     val kotlinx_serialization_json = dep(
         group = "org.jetbrains.kotlinx",
         name = "kotlinx-serialization-json",
@@ -339,6 +338,69 @@ object Libs {
         name = "commons-compress",
         version = Versions.commons_compress_test_only_version
     )
+
+    // https://protobuf.dev/
+    val protobuf_protoc = dep(
+        group = "com.google.protobuf",
+        name = "protoc",
+        version = Versions.protobuf
+    )
+    val protobuf_java = dep(
+        group = "com.google.protobuf",
+        name = "protobuf-java",
+        version = Versions.protobuf
+    )
+    val protobuf_kotlin = dep(
+        group = "com.google.protobuf",
+        name = "protobuf-kotlin",
+        version = Versions.protobuf
+    )
+
+    // https://github.com/grpc/grpc-java
+    val grpc_api = dep(
+        group = "io.grpc",
+        name = "grpc-api",
+        version = Versions.grpc
+    )
+    val grpc_protobuf = dep(
+        group = "io.grpc",
+        name = "grpc-protobuf",
+        version = Versions.grpc
+    )
+    val grpc_services = dep(
+        group = "io.grpc",
+        name = "grpc-services",
+        version = Versions.grpc
+    )
+    val grpc_netty_shaded = dep(
+        group = "io.grpc",
+        name = "grpc-netty-shaded",
+        version = Versions.grpc
+    )
+
+    // https://github.com/square/wire
+    val wire_runtime = dep(
+        group = "com.squareup.wire",
+        name = "wire-runtime",
+        version = Versions.wire
+    )
+    val wire_grpc_client = dep(
+        group = "com.squareup.wire",
+        name = "wire-grpc-client",
+        version = Versions.wire
+    )
+
+    // https://github.com/square/wire-grpc-server
+    val wire_grpc_server = dep(
+        group = "com.squareup.wiregrpcserver",
+        name = "server",
+        version = Versions.wire_grpc_server
+    )
+    val wire_grpc_server_generator = dep(
+        group = "com.squareup.wiregrpcserver",
+        name = "server-generator",
+        version = Versions.wire_grpc_server
+    )
 }
 
 object Plugins {
@@ -357,6 +419,18 @@ object Plugins {
         id = "de.undercouch.download"
     )
 
+    // https://github.com/node-gradle/gradle-node-plugin
+    object GradleNode : ProjectPlugin(
+        version = Versions.gradle_node,
+        id = "com.github.node-gradle.node"
+    )
+
+    // https://github.com/google/protobuf-gradle-plugin
+    object GradleProtobuf : ProjectPlugin(
+        version = Versions.gradle_protobuf,
+        id = "com.google.protobuf"
+    )
+
     // https://github.com/ben-manes/gradle-versions-plugin
     object GradleVersions : ProjectPlugin(
         version = Versions.gradle_versions,
@@ -380,6 +454,12 @@ object Plugins {
         version = Versions.shadow,
         id = "com.github.johnrengelman.shadow"
     )
+
+    // https://github.com/square/wire
+    object Wire : ProjectPlugin(
+        version = Versions.wire,
+        id = "com.squareup.wire"
+    )
 }
 
 fun PluginDependenciesSpec.id(plugin: Plugins.ProjectPlugin) {

buildSrc/src/main/kotlin/ProcessUtil.kt

@@ -0,0 +1,88 @@
+import java.io.Reader
+import java.util.concurrent.TimeUnit
+import kotlin.concurrent.thread
+import kotlin.time.Duration
+
+object ProcessUtil {
+    data class Result(
+        val exitCode: Int,
+        val stdout: String,
+        val stderr: String,
+        val isTimeout: Boolean, // true if the process was terminated due to timeout
+    )
+
+    fun run(
+        command: List<String>,
+        input: String? = null,
+        timeout: Duration? = null,
+        builder: ProcessBuilder.() -> Unit = {},
+    ): Result {
+        val reader = input?.reader() ?: "".reader()
+        return run(command, reader, timeout, builder)
+    }
+
+    fun run(
+        command: List<String>,
+        input: Reader,
+        timeout: Duration? = null,
+        builder: ProcessBuilder.() -> Unit = {},
+    ): Result {
+        val process = ProcessBuilder(command).apply(builder).start()
+        return communicate(process, input, timeout)
+    }
+
+    private fun communicate(
+        process: Process,
+        input: Reader,
+        timeout: Duration?,
+    ): Result {
+        // Handle process input (stdin)
+        val stdinThread = thread {
+            process.outputStream.bufferedWriter().use { writer ->
+                input.copyTo(writer)
+                writer.flush()
+            }
+        }
+
+        // Capture process output (stdout)
+        val stdout = StringBuilder()
+        val stdoutThread = thread {
+            process.inputStream.bufferedReader().useLines { lines ->
+                lines.forEach { stdout.appendLine(it) }
+            }
+        }
+
+        // Capture process error output (stderr)
+        val stderr = StringBuilder()
+        val stderrThread = thread {
+            process.errorStream.bufferedReader().useLines { lines ->
+                lines.forEach { stderr.appendLine(it) }
+            }
+        }
+
+        // Wait for completion
+        val isTimeout = if (timeout != null) {
+            !process.waitFor(timeout.inWholeMilliseconds, TimeUnit.MILLISECONDS)
+        } else {
+            process.waitFor()
+            false
+        }
+
+        // If timeout occurred, destroy the process forcibly
+        if (isTimeout) {
+            process.destroyForcibly()
+        }
+
+        // Wait for stream threads to finish
+        stdinThread.join()
+        stdoutThread.join()
+        stderrThread.join()
+
+        return Result(
+            exitCode = if (isTimeout) -1 else process.exitValue(),
+            stdout = stdout.toString(),
+            stderr = stderr.toString(),
+            isTimeout = isTimeout
+        )
+    }
+}

jacodb-ets/.gitignore

@@ -1,3 +1,4 @@
+arkanalyzer
 /src/test/resources/samples/etsir
 /src/test/resources/projects
 

jacodb-ets/build.gradle.kts

@@ -1,12 +1,16 @@
+import com.github.gradle.node.npm.task.NpmTask
 import java.io.FileNotFoundException
+import kotlin.time.Duration.Companion.minutes
 
 plugins {
     kotlin("plugin.serialization")
-    `java-test-fixtures`
+    id(Plugins.GradleNode)
 }
 
 dependencies {
     api(project(":jacodb-api-common"))
+    api(project(":jacodb-ets:wire-client"))
+    api(project(":jacodb-ets:wire-server"))
 
     implementation(Libs.kotlin_logging)
     implementation(Libs.slf4j_simple)
@@ -21,6 +25,16 @@ dependencies {
     testFixturesImplementation(Libs.junit_jupiter_api)
 }
 
+node {
+    download = true
+    nodeProjectDir.set(file("arkanalyzer"))
+}
+
+tasks.register<NpmTask>("runArkAnalyzerServer") {
+    dependsOn(tasks.npmInstall)
+    args = listOf("run", "server")
+}
+
 // Example usage:
 // ```
 // export ARKANALYZER_DIR=~/dev/arkanalyzer
@@ -74,21 +88,20 @@ tasks.register("generateTestResources") {
             "-t",
         )
         println("Running: '${cmd.joinToString(" ")}'")
-        val process = ProcessBuilder(cmd).directory(resources).start()
-        val ok = process.waitFor(10, TimeUnit.MINUTES)
-
-        val stdout = process.inputStream.bufferedReader().readText().trim()
-        if (stdout.isNotBlank()) {
-            println("[STDOUT]:\n--------\n$stdout\n--------")
+        val result = ProcessUtil.run(cmd, timeout = 1.minutes) {
+            directory(resources)
         }
-        val stderr = process.errorStream.bufferedReader().readText().trim()
-        if (stderr.isNotBlank()) {
-            println("[STDERR]:\n--------\n$stderr\n--------")
+        if (result.stdout.isNotBlank()) {
+            println("[STDOUT]:\n--------\n${result.stdout}\n--------")
         }
-
-        if (!ok) {
+        if (result.stderr.isNotBlank()) {
+            println("[STDERR]:\n--------\n${result.stderr}\n--------")
+        }
+        if (result.isTimeout) {
             println("Timeout!")
-            process.destroy()
+        }
+        if (result.exitCode != 0) {
+            println("Exit code: ${result.exitCode}")
         }
 
         println("Done generating test resources in %.1fs".format((System.currentTimeMillis() - startTime) / 1000.0))