diff --git a/docker-images/build.sh b/docker-images/build.sh index 747b6ca6e75..82c37df9da5 100755 --- a/docker-images/build.sh +++ b/docker-images/build.sh @@ -119,6 +119,17 @@ function fetch_and_unpack_kafka_binaries { # Store the folder address for use in the image build version_dist_dirs["$kafka_version"]="./tmp/$kafka_version" + # create a file listing all the jars with colliding class files in the Kafka dist + # (on the assumption that this is OK). This file will be used after building the images to detect any collisions + # added by the third-party jars mechanism. + whilelist_file="$binary_file_dir/$kafka_version.whitelist" + if [ ! -e $whilelist_file ] + then + unzipped_dir=`mktemp -d` + ./extract-jars.sh "$dist_dir/libs" "$unzipped_dir" + ./find-colliding-classes.sh "$unzipped_dir" | awk '{print $1}' | sort | uniq > "$whilelist_file" || true + rm -rf $unzipped_dir + fi done } diff --git a/docker-images/extract-jars.sh b/docker-images/extract-jars.sh new file mode 100755 index 00000000000..cf21589d95b --- /dev/null +++ b/docker-images/extract-jars.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +jars_dir=$1 +classes_root=$2 + +for jar in $(find $jars_dir -type f -name '*.jar') +do + extracted=$classes_root/$(basename "$jar") + mkdir -p "$extracted" + unzip -qq "$jar" -d "$extracted" >/dev/null +done \ No newline at end of file diff --git a/docker-images/find-classpath-collision.sh b/docker-images/find-classpath-collision.sh new file mode 100755 index 00000000000..9ccde91aeec --- /dev/null +++ b/docker-images/find-classpath-collision.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +image=$1 #strimzi/kafka:latest-kafka-2.2.1 +image_jar_dir=$2 #/opt/kafka/libs +whilelist_file=$3 + +jars_dir=`mktemp -d` +trap "[ -e $jars_dir ] && rm -rf $jars_dir" EXIT + +docker run --name temp-container-name "$image" /bin/true || exit 2 +docker cp "temp-container-name:$image_jar_dir" "$jars_dir" +docker rm temp-container-name > /dev/null + +classes_root=`mktemp -d` +trap "[ -e $classes_root ] && rm -rf $classes_root" EXIT + +$(dirname $0)/extract-jars.sh "$jars_dir" "$classes_root" + +collisions=$($(dirname "$0")/find-colliding-classes.sh "$classes_root" | awk '{printf("%s\t%s\n",$1,$2);}' | \ + grep -vFf "$whilelist_file") + +if [ "$collisions" != "" ] ; then + echo "ERROR: Different class files with same name from different jars found!" + echo "$collisions" + echo "(Ignoring jars from Kafka distribution containing different class files with same name:" + sed -e 's/^/ /' "$whilelist_file" + echo ")" + echo "It's likely that either two third party jars are using different versions " + echo "of a common (transitive) dependency or a single third party jar is using a" + echo "dependency which is also a (transitive) dependency of Kafka." + echo "In either case the solution is a judicious of the dependency." + exit 1 +fi diff --git a/docker-images/find-colliding-classes.sh b/docker-images/find-colliding-classes.sh new file mode 100755 index 00000000000..8a58ae3512f --- /dev/null +++ b/docker-images/find-colliding-classes.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash +# $1: Directory containing (directories containing) .class files +unzipped_dir=$1 + +# compute list of +find $unzipped_dir ! -empty -type f -name '*.class' -a ! -name 'module-info.class' -exec md5sum {} + | \ + # split into + sed -E 's#([a-z0-9]+).*/([^/]*[.]jar)/(.*)#\1\t\2\t\3#' | \ + # sort by classname + sort -r -k 3 | \ + # find duplicate classname + uniq -D -f2 | \ + # swap column order => + awk '{printf("%s\t%s\t%s\n",$2,$3,$1);}' | \ + # find unique md5 (i.e. classfiles differ) + uniq -u -f2 \ No newline at end of file diff --git a/docker-images/kafka/Makefile b/docker-images/kafka/Makefile index 99966d4b04e..09dd8014fbe 100644 --- a/docker-images/kafka/Makefile +++ b/docker-images/kafka/Makefile @@ -5,6 +5,7 @@ clean: rm -rf tmp rm -f .*.tmp rm -rf kafka-thirdparty-libs/*/target + rm -f tmp/*.whitelist .kafka-agent.tmp: ../../kafka-agent/target/kafka-agent-$(RELEASE_VERSION).jar test -d tmp || mkdir tmp @@ -31,4 +32,7 @@ docker_build: .kafka-agent.tmp .tracing-agent.tmp .mirror-maker-agent.tmp .third include ../../Makefile.docker +docker_build: + ../find-classpath-collision.sh strimzi/$(PROJECT_NAME):$(BUILD_TAG) /opt/kafka/libs "tmp/$(KAFKA_VERSION).whitelist" + .PHONY: build clean release diff --git a/docker-images/kafka/kafka-thirdparty-libs/2.2.x/pom.xml b/docker-images/kafka/kafka-thirdparty-libs/2.2.x/pom.xml index f694645e74c..4cc9e2ac1e0 100644 --- a/docker-images/kafka/kafka-thirdparty-libs/2.2.x/pom.xml +++ b/docker-images/kafka/kafka-thirdparty-libs/2.2.x/pom.xml @@ -82,6 +82,10 @@ com.github.spotbugs spotbugs-annotations + + com.fasterxml.jackson.core + jackson-core + diff --git a/docker-images/kafka/kafka-thirdparty-libs/2.3.x/pom.xml b/docker-images/kafka/kafka-thirdparty-libs/2.3.x/pom.xml index d68ce41ddb9..e89a784b671 100644 --- a/docker-images/kafka/kafka-thirdparty-libs/2.3.x/pom.xml +++ b/docker-images/kafka/kafka-thirdparty-libs/2.3.x/pom.xml @@ -82,6 +82,10 @@ com.github.spotbugs spotbugs-annotations + + com.fasterxml.jackson.core + jackson-core +