Break the build for classpath collisions in Kafka images (strimzi#2253)

This detects when the 3rd-party jars mechanism adds jar dependencies which collide
with either other 3rd party jars or with the Kafka jars/dependencies.

This is, in essence a regression test for strimzi#2245.

Signed-off-by: Tom Bentley <[email protected]>

docker-images/build.sh

@@ -119,6 +119,17 @@ function fetch_and_unpack_kafka_binaries {
         # Store the folder address for use in the image build 
         version_dist_dirs["$kafka_version"]="./tmp/$kafka_version"
 
+        # create a file listing all the jars with colliding class files in the Kafka dist
+        # (on the assumption that this is OK). This file will be used after building the images to detect any collisions
+        # added by the third-party jars mechanism.
+        whilelist_file="$binary_file_dir/$kafka_version.whitelist"
+        if [ ! -e $whilelist_file ]
+        then
+            unzipped_dir=`mktemp -d`
+            ./extract-jars.sh "$dist_dir/libs" "$unzipped_dir"
+            ./find-colliding-classes.sh "$unzipped_dir" | awk '{print $1}' | sort | uniq > "$whilelist_file" || true
+            rm -rf $unzipped_dir
+        fi
     done
 
 }

docker-images/extract-jars.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+jars_dir=$1
+classes_root=$2
+
+for jar in $(find $jars_dir -type f -name '*.jar')
+do
+  extracted=$classes_root/$(basename "$jar")
+  mkdir -p "$extracted"
+  unzip -qq "$jar" -d "$extracted" >/dev/null
+done

docker-images/find-classpath-collision.sh

@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+image=$1 #strimzi/kafka:latest-kafka-2.2.1
+image_jar_dir=$2 #/opt/kafka/libs
+whilelist_file=$3
+
+jars_dir=`mktemp -d`
+trap "[ -e $jars_dir ] && rm -rf $jars_dir" EXIT
+
+docker run --name temp-container-name "$image" /bin/true || exit 2
+docker cp "temp-container-name:$image_jar_dir" "$jars_dir"
+docker rm temp-container-name > /dev/null
+
+classes_root=`mktemp -d`
+trap "[ -e $classes_root ] && rm -rf $classes_root" EXIT
+
+$(dirname $0)/extract-jars.sh "$jars_dir" "$classes_root"
+
+collisions=$($(dirname "$0")/find-colliding-classes.sh "$classes_root" | awk '{printf("%s\t%s\n",$1,$2);}' | \
+    grep -vFf "$whilelist_file")
+
+if [ "$collisions" != "" ] ; then
+  echo "ERROR: Different class files with same name from different jars found!"
+  echo "$collisions"
+  echo "(Ignoring jars from Kafka distribution containing different class files with same name:"
+  sed -e 's/^/  /' "$whilelist_file"
+  echo ")"
+  echo "It's likely that either two third party jars are using different versions "
+  echo "of a common (transitive) dependency or a single third party jar is using a"
+  echo "dependency which is also a (transitive) dependency of Kafka."
+  echo "In either case the solution is a judicious <exclude> of the dependency."
+  exit 1
+fi

docker-images/find-colliding-classes.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+# $1: Directory containing (directories containing) .class files
+unzipped_dir=$1
+
+# compute list of <md5> <classfile>
+find $unzipped_dir ! -empty -type f -name '*.class' -a ! -name 'module-info.class' -exec md5sum {} + | \
+  # split into <md5> <jarname> <classname>
+  sed -E 's#([a-z0-9]+).*/([^/]*[.]jar)/(.*)#\1\t\2\t\3#' | \
+  # sort by classname
+  sort -r -k 3 | \
+  # find duplicate classname
+  uniq -D -f2 | \
+  # swap column order => <classname> <jarname> <md5>
+  awk '{printf("%s\t%s\t%s\n",$2,$3,$1);}' | \
+  # find unique md5 (i.e. classfiles differ)
+  uniq -u -f2

docker-images/kafka/Makefile

@@ -5,6 +5,7 @@ clean:
 	rm -rf tmp
 	rm -f .*.tmp
 	rm -rf kafka-thirdparty-libs/*/target
+	rm -f tmp/*.whitelist
 
 .kafka-agent.tmp: ../../kafka-agent/target/kafka-agent-$(RELEASE_VERSION).jar
 	test -d tmp || mkdir tmp
@@ -31,4 +32,7 @@ docker_build: .kafka-agent.tmp .tracing-agent.tmp .mirror-maker-agent.tmp .third
 
 include ../../Makefile.docker
 
+docker_build:
+	../find-classpath-collision.sh strimzi/$(PROJECT_NAME):$(BUILD_TAG) /opt/kafka/libs "tmp/$(KAFKA_VERSION).whitelist"
+
 .PHONY: build clean release

docker-images/kafka/kafka-thirdparty-libs/2.2.x/pom.xml

@@ -82,6 +82,10 @@
                     <groupId>com.github.spotbugs</groupId>
                     <artifactId>spotbugs-annotations</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

docker-images/kafka/kafka-thirdparty-libs/2.3.x/pom.xml

@@ -82,6 +82,10 @@
                     <groupId>com.github.spotbugs</groupId>
                     <artifactId>spotbugs-annotations</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>