[WIP] Upgrade Kafka client library to v2.8.1 / 3.0.0 (+ update related Kafka client dependencies)

build.gradle

@@ -39,16 +39,12 @@ repositories {
     mavenCentral()
 }
 
-task wrapper(type: Wrapper) {
-    gradleVersion = '4.0'
-}
-
 dependencies {
-    compile 'org.apache.kafka:kafka-clients:2.3.0'
-    compile 'com.github.luben:zstd-jni:1.4.2-1'
-    compile 'org.slf4j:slf4j-api:1.7.26'
-    compile 'org.lz4:lz4-java:1.6.0'
-    compile 'org.xerial.snappy:snappy-java:1.1.7.3'
+    compile 'org.apache.kafka:kafka-clients:2.8.1'
+    compile 'com.github.luben:zstd-jni:1.5.0-2'
+    compile 'org.slf4j:slf4j-api:1.7.30'
+    compile 'org.lz4:lz4-java:1.7.1'
+    compile 'org.xerial.snappy:snappy-java:1.1.8.1'
 
 }
 task generateGemJarRequiresFile {

build.gradle.gradle73

@@ -0,0 +1,86 @@
+import java.nio.file.Files
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+apply plugin: "java"
+apply plugin: 'maven-publish'
+apply plugin: "distribution"
+apply plugin: "idea"
+
+group "org.logstash.inputs"
+
+sourceCompatibility = JavaVersion.VERSION_1_8
+
+buildscript {
+    repositories {
+        mavenCentral()
+        jcenter()
+    }
+
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compileOnly 'org.apache.kafka:kafka-clients:2.8.1'
+    compileOnly 'com.github.luben:zstd-jni:1.5.0-2'
+    compileOnly 'org.slf4j:slf4j-api:1.7.30'
+    compileOnly 'org.lz4:lz4-java:1.7.1'
+    compileOnly 'org.xerial.snappy:snappy-java:1.1.8.1'
+
+}
+
+configurations {
+    compileOnlyClasspath {
+        // https://discuss.gradle.org/t/what-is-a-configuration-which-cant-be-directly-resolved/30721
+        extendsFrom compileOnly
+        canBeConsumed false
+        canBeResolved true
+    }
+}
+
+task generateGemJarRequiresFile {
+    doLast {
+        File jars_file = file('lib/logstash-input-kafka_jars.rb')
+        jars_file.newWriter().withWriter { w ->
+            w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n"
+            w << "require \'jar_dependencies\'\n"
+            configurations.runtimeClasspath.allDependencies.each {
+                w << "require_jar(\'${it.group}\', \'${it.name}\', \'${it.version}\')\n"
+            }
+        }
+    }
+}
+
+task vendor {
+    doLast {
+        String vendorPathPrefix = "vendor/jar-dependencies"
+        configurations.compileOnlyClasspath.allDependencies.each { dep ->
+            File f = configurations.compileOnlyClasspath.filter { it.absolutePath.contains("${dep.group}/${dep.name}/${dep.version}") }.singleFile
+            String groupPath = dep.group.replaceAll('\\.', '/')
+            File newJarFile = file("${vendorPathPrefix}/${groupPath}/${dep.name}/${dep.version}/${dep.name}-${dep.version}.jar")
+            newJarFile.mkdirs()
+            Files.copy(f.toPath(), newJarFile.toPath(), REPLACE_EXISTING)
+        }
+    }
+}
+
+vendor.dependsOn(generateGemJarRequiresFile)

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,6 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.0-all.zip
+# NOTE: For building this gem under JDK 17 we need patched ruby-maven gaven and
+# gradle >= 7.3
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-all.zip

gradle/wrapper/gradle-wrapper.properties.gradle73

@@ -0,0 +1,6 @@
+#Wed Jun 21 11:39:16 CEST 2017
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-all.zip

lib/logstash-input-kafka_jars.rb

@@ -1,8 +1,8 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.apache.kafka', 'kafka-clients', '2.1.0')
-require_jar('com.github.luben', 'zstd-jni', '1.3.7-3')
-require_jar('org.slf4j', 'slf4j-api', '1.7.25')
-require_jar('org.lz4', 'lz4-java', '1.5.0')
-require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.2')
+require_jar('org.apache.kafka', 'kafka-clients', '2.8.1')
+require_jar('com.github.luben', 'zstd-jni', '1.5.0-2')
+require_jar('org.slf4j', 'slf4j-api', '1.7.30')
+require_jar('org.lz4', 'lz4-java', '1.7.1')
+require_jar('org.xerial.snappy', 'snappy-java', '1.1.8.1')

logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '9.1.0'
+  s.version         = '9.2.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Reads events from a Kafka topic"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"