From c709c8032a7fe6d7a6bbded57a039f2c2bab10a6 Mon Sep 17 00:00:00 2001 From: Ben Fradet Date: Mon, 7 May 2018 11:41:17 +0100 Subject: [PATCH 1/2] Add Stream Enrich 0.16.1 image (closes #46) --- .travis.yml | 8 +++--- README.md | 8 +++--- stream-enrich/0.11.1/Dockerfile | 2 +- stream-enrich/0.12.0/Dockerfile | 2 +- stream-enrich/0.13.0/Dockerfile | 2 +- stream-enrich/0.14.0/Dockerfile | 2 +- stream-enrich/0.15.0/google-pubsub/Dockerfile | 2 +- stream-enrich/0.15.0/kafka/Dockerfile | 2 +- stream-enrich/0.15.0/kinesis/Dockerfile | 2 +- stream-enrich/0.15.0/nsq/Dockerfile | 2 +- stream-enrich/0.16.0/google-pubsub/Dockerfile | 2 +- stream-enrich/0.16.0/kafka/Dockerfile | 2 +- stream-enrich/0.16.0/kinesis/Dockerfile | 2 +- stream-enrich/0.16.0/nsq/Dockerfile | 2 +- stream-enrich/0.16.1/google-pubsub/Dockerfile | 26 +++++++++++++++++++ .../0.16.1/google-pubsub/docker-entrypoint.sh | 15 +++++++++++ stream-enrich/0.16.1/kafka/Dockerfile | 26 +++++++++++++++++++ .../0.16.1/kafka/docker-entrypoint.sh | 15 +++++++++++ stream-enrich/0.16.1/kinesis/Dockerfile | 26 +++++++++++++++++++ .../0.16.1/kinesis/docker-entrypoint.sh | 15 +++++++++++ stream-enrich/0.16.1/nsq/Dockerfile | 26 +++++++++++++++++++ stream-enrich/0.16.1/nsq/docker-entrypoint.sh | 15 +++++++++++ stream-enrich/README.md | 2 +- 23 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 stream-enrich/0.16.1/google-pubsub/Dockerfile create mode 100755 stream-enrich/0.16.1/google-pubsub/docker-entrypoint.sh create mode 100644 stream-enrich/0.16.1/kafka/Dockerfile create mode 100755 stream-enrich/0.16.1/kafka/docker-entrypoint.sh create mode 100644 stream-enrich/0.16.1/kinesis/Dockerfile create mode 100755 stream-enrich/0.16.1/kinesis/docker-entrypoint.sh create mode 100644 stream-enrich/0.16.1/nsq/Dockerfile create mode 100755 stream-enrich/0.16.1/nsq/docker-entrypoint.sh diff --git a/.travis.yml b/.travis.yml index fdc9404..67fb2cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,10 @@ env: - IMG=scala-stream-collector/0.13.0/kafka - IMG=scala-stream-collector/0.13.0/kinesis - IMG=scala-stream-collector/0.13.0/nsq - - IMG=stream-enrich/0.16.0/google-pubsub - - IMG=stream-enrich/0.16.0/kafka - - IMG=stream-enrich/0.16.0/kinesis - - IMG=stream-enrich/0.16.0/nsq + - IMG=stream-enrich/0.16.1/google-pubsub + - IMG=stream-enrich/0.16.1/kafka + - IMG=stream-enrich/0.16.1/kinesis + - IMG=stream-enrich/0.16.1/nsq - IMG=s3-loader/0.6.0 - IMG=elasticsearch-loader/0.10.1 - IMG=iglu-server/0.3.0 diff --git a/README.md b/README.md index 432fcc4..d45b961 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You can pull the images from the registry directly: docker pull snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0 # NSQ Stream Enrich image, there are others available for Kinesis, Kafka and Google PubSub -docker pull snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.0 +docker pull snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1 # Elasticsearch Loader image docker pull snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1 @@ -48,7 +48,7 @@ docker pull snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 docker build -t snowplow/scala-stream-collector-nsq:0.13.0 scala-stream-collector/0.13.0/nsq # NSQ Stream Enrich image, there are others available for Kinesis, Kafka and Google PubSub -docker build -t snowplow/stream-enrich-nsq:0.16.0 stream-enrich/0.16.0/nsq +docker build -t snowplow/stream-enrich-nsq:0.16.1 stream-enrich/0.16.1/nsq # Elasticsearch Loader image docker build -t snowplow/elasticsearch-loader:0.10.1 elasticsearch-loader/0.10.1 @@ -84,8 +84,8 @@ docker run \ # NSQ Stream Enrich container, there are others available for Kinesis, Kafka and Google PubSub docker run \ -v $PWD/stream-enrich-config:/snowplow/config \ - snowplow/stream-enrich-nsq:0.16.0 \ # if you have built the image - # snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.0 if you have pulled the image + snowplow/stream-enrich-nsq:0.16.1 \ # if you have built the image + # snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1 if you have pulled the image --config /snowplow/config/config.hocon \ --resolver file:/snowplow/config/resolver.json \ --enrichments file:/snowplow/config/enrichments/ \ diff --git a/stream-enrich/0.11.1/Dockerfile b/stream-enrich/0.11.1/Dockerfile index 927a5cf..b4e0c2a 100644 --- a/stream-enrich/0.11.1/Dockerfile +++ b/stream-enrich/0.11.1/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.11.1" # The name of the archive to download. diff --git a/stream-enrich/0.12.0/Dockerfile b/stream-enrich/0.12.0/Dockerfile index f6e4eae..7c8ab15 100644 --- a/stream-enrich/0.12.0/Dockerfile +++ b/stream-enrich/0.12.0/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.12.0" # The name of the archive to download. diff --git a/stream-enrich/0.13.0/Dockerfile b/stream-enrich/0.13.0/Dockerfile index efbda13..aa71e6e 100644 --- a/stream-enrich/0.13.0/Dockerfile +++ b/stream-enrich/0.13.0/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.13.0" # The name of the archive to download. diff --git a/stream-enrich/0.14.0/Dockerfile b/stream-enrich/0.14.0/Dockerfile index d409c20..2bbf10f 100644 --- a/stream-enrich/0.14.0/Dockerfile +++ b/stream-enrich/0.14.0/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.14.0" # The name of the archive to download. diff --git a/stream-enrich/0.15.0/google-pubsub/Dockerfile b/stream-enrich/0.15.0/google-pubsub/Dockerfile index b6b7f77..541ec0e 100644 --- a/stream-enrich/0.15.0/google-pubsub/Dockerfile +++ b/stream-enrich/0.15.0/google-pubsub/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.15.0" # The targeted platform diff --git a/stream-enrich/0.15.0/kafka/Dockerfile b/stream-enrich/0.15.0/kafka/Dockerfile index 2fd878e..ca741de 100644 --- a/stream-enrich/0.15.0/kafka/Dockerfile +++ b/stream-enrich/0.15.0/kafka/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.15.0" # The targeted platform diff --git a/stream-enrich/0.15.0/kinesis/Dockerfile b/stream-enrich/0.15.0/kinesis/Dockerfile index 8bc08fb..64a1829 100644 --- a/stream-enrich/0.15.0/kinesis/Dockerfile +++ b/stream-enrich/0.15.0/kinesis/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.15.0" # The targeted platform diff --git a/stream-enrich/0.15.0/nsq/Dockerfile b/stream-enrich/0.15.0/nsq/Dockerfile index edd0628..36e957d 100644 --- a/stream-enrich/0.15.0/nsq/Dockerfile +++ b/stream-enrich/0.15.0/nsq/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.15.0" # The targeted platform diff --git a/stream-enrich/0.16.0/google-pubsub/Dockerfile b/stream-enrich/0.16.0/google-pubsub/Dockerfile index f413156..3212d66 100644 --- a/stream-enrich/0.16.0/google-pubsub/Dockerfile +++ b/stream-enrich/0.16.0/google-pubsub/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.16.0" # The targeted platform diff --git a/stream-enrich/0.16.0/kafka/Dockerfile b/stream-enrich/0.16.0/kafka/Dockerfile index 63dc2ff..6f99782 100644 --- a/stream-enrich/0.16.0/kafka/Dockerfile +++ b/stream-enrich/0.16.0/kafka/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.16.0" # The targeted platform diff --git a/stream-enrich/0.16.0/kinesis/Dockerfile b/stream-enrich/0.16.0/kinesis/Dockerfile index b2b9760..24aeab6 100644 --- a/stream-enrich/0.16.0/kinesis/Dockerfile +++ b/stream-enrich/0.16.0/kinesis/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.16.0" # The targeted platform diff --git a/stream-enrich/0.16.0/nsq/Dockerfile b/stream-enrich/0.16.0/nsq/Dockerfile index eb6924b..16d232c 100644 --- a/stream-enrich/0.16.0/nsq/Dockerfile +++ b/stream-enrich/0.16.0/nsq/Dockerfile @@ -1,7 +1,7 @@ FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 LABEL maintainer="Snowplow Analytics Ltd. " -# The version of the collector to download. +# The version of stream enrich to download. ENV ENRICH_VERSION="0.16.0" # The targeted platform diff --git a/stream-enrich/0.16.1/google-pubsub/Dockerfile b/stream-enrich/0.16.1/google-pubsub/Dockerfile new file mode 100644 index 0000000..4c00988 --- /dev/null +++ b/stream-enrich/0.16.1/google-pubsub/Dockerfile @@ -0,0 +1,26 @@ +FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 +LABEL maintainer="Snowplow Analytics Ltd. " + +# The version of stream enrich to download. +ENV ENRICH_VERSION="0.16.1" + +# The targeted platform +ENV PLATFORM="google_pubsub" + +# The name of the archive to download. +ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip" + +# Install the Scala Stream Collector. +RUN mkdir -p /tmp/build && \ + cd /tmp/build && \ + wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \ + unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \ + cd /tmp && \ + rm -rf /tmp/build + +# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user. +# The script uses dumb-init as the top-level process. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT [ "docker-entrypoint.sh" ] + +CMD [ "--help" ] diff --git a/stream-enrich/0.16.1/google-pubsub/docker-entrypoint.sh b/stream-enrich/0.16.1/google-pubsub/docker-entrypoint.sh new file mode 100755 index 0000000..26b7b52 --- /dev/null +++ b/stream-enrich/0.16.1/google-pubsub/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/dumb-init /bin/sh +set -e + +# If the config directory has been mounted through -v, we chown it. +if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then + chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH} +fi + +# Needed because of SCE's ./ip_geo file +cd $(eval echo ~snowplow) + +# Make sure we run the collector as the snowplow user +exec su-exec snowplow:snowplow /usr/bin/java \ + $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ + -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM//_/-}-${ENRICH_VERSION}.jar "$@" diff --git a/stream-enrich/0.16.1/kafka/Dockerfile b/stream-enrich/0.16.1/kafka/Dockerfile new file mode 100644 index 0000000..56659f4 --- /dev/null +++ b/stream-enrich/0.16.1/kafka/Dockerfile @@ -0,0 +1,26 @@ +FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 +LABEL maintainer="Snowplow Analytics Ltd. " + +# The version of stream enrich to download. +ENV ENRICH_VERSION="0.16.1" + +# The targeted platform +ENV PLATFORM="kafka" + +# The name of the archive to download. +ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip" + +# Install the Scala Stream Collector. +RUN mkdir -p /tmp/build && \ + cd /tmp/build && \ + wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \ + unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \ + cd /tmp && \ + rm -rf /tmp/build + +# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user. +# The script uses dumb-init as the top-level process. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT [ "docker-entrypoint.sh" ] + +CMD [ "--help" ] diff --git a/stream-enrich/0.16.1/kafka/docker-entrypoint.sh b/stream-enrich/0.16.1/kafka/docker-entrypoint.sh new file mode 100755 index 0000000..916b910 --- /dev/null +++ b/stream-enrich/0.16.1/kafka/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/dumb-init /bin/sh +set -e + +# If the config directory has been mounted through -v, we chown it. +if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then + chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH} +fi + +# Needed because of SCE's ./ip_geo file +cd $(eval echo ~snowplow) + +# Make sure we run the collector as the snowplow user +exec su-exec snowplow:snowplow /usr/bin/java \ + $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ + -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM}-${ENRICH_VERSION}.jar "$@" diff --git a/stream-enrich/0.16.1/kinesis/Dockerfile b/stream-enrich/0.16.1/kinesis/Dockerfile new file mode 100644 index 0000000..2f6e3e8 --- /dev/null +++ b/stream-enrich/0.16.1/kinesis/Dockerfile @@ -0,0 +1,26 @@ +FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 +LABEL maintainer="Snowplow Analytics Ltd. " + +# The version of stream enrich to download. +ENV ENRICH_VERSION="0.16.1" + +# The targeted platform +ENV PLATFORM="kinesis" + +# The name of the archive to download. +ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip" + +# Install the Scala Stream Collector. +RUN mkdir -p /tmp/build && \ + cd /tmp/build && \ + wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \ + unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \ + cd /tmp && \ + rm -rf /tmp/build + +# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user. +# The script uses dumb-init as the top-level process. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT [ "docker-entrypoint.sh" ] + +CMD [ "--help" ] diff --git a/stream-enrich/0.16.1/kinesis/docker-entrypoint.sh b/stream-enrich/0.16.1/kinesis/docker-entrypoint.sh new file mode 100755 index 0000000..916b910 --- /dev/null +++ b/stream-enrich/0.16.1/kinesis/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/dumb-init /bin/sh +set -e + +# If the config directory has been mounted through -v, we chown it. +if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then + chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH} +fi + +# Needed because of SCE's ./ip_geo file +cd $(eval echo ~snowplow) + +# Make sure we run the collector as the snowplow user +exec su-exec snowplow:snowplow /usr/bin/java \ + $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ + -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM}-${ENRICH_VERSION}.jar "$@" diff --git a/stream-enrich/0.16.1/nsq/Dockerfile b/stream-enrich/0.16.1/nsq/Dockerfile new file mode 100644 index 0000000..fad0634 --- /dev/null +++ b/stream-enrich/0.16.1/nsq/Dockerfile @@ -0,0 +1,26 @@ +FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0 +LABEL maintainer="Snowplow Analytics Ltd. " + +# The version of stream enrich to download. +ENV ENRICH_VERSION="0.16.1" + +# The targeted platform +ENV PLATFORM="nsq" + +# The name of the archive to download. +ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip" + +# Install the Scala Stream Collector. +RUN mkdir -p /tmp/build && \ + cd /tmp/build && \ + wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \ + unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \ + cd /tmp && \ + rm -rf /tmp/build + +# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user. +# The script uses dumb-init as the top-level process. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT [ "docker-entrypoint.sh" ] + +CMD [ "--help" ] diff --git a/stream-enrich/0.16.1/nsq/docker-entrypoint.sh b/stream-enrich/0.16.1/nsq/docker-entrypoint.sh new file mode 100755 index 0000000..916b910 --- /dev/null +++ b/stream-enrich/0.16.1/nsq/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/dumb-init /bin/sh +set -e + +# If the config directory has been mounted through -v, we chown it. +if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then + chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH} +fi + +# Needed because of SCE's ./ip_geo file +cd $(eval echo ~snowplow) + +# Make sure we run the collector as the snowplow user +exec su-exec snowplow:snowplow /usr/bin/java \ + $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ + -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM}-${ENRICH_VERSION}.jar "$@" diff --git a/stream-enrich/README.md b/stream-enrich/README.md index e7ce9ca..221bf75 100644 --- a/stream-enrich/README.md +++ b/stream-enrich/README.md @@ -32,7 +32,7 @@ Additional JVM options can be set through the `SP_JAVA_OPTS` environment variabl Running the container without arguments will print out its usage: ```bash -$ VERSION=0.16.0 +$ VERSION=0.16.1 $ docker run snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:${VERSION} snowplow-stream-enrich $VERSION From 7891352c4bd647ac748770d48bb99f92cc00ceac Mon Sep 17 00:00:00 2001 From: Ben Fradet Date: Wed, 9 May 2018 16:29:34 +0100 Subject: [PATCH 2/2] Prepare for release --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 16c9f8e..acf445d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +Release 5 (2018-05-09) +---------------------- +Add Stream Enrich 0.16.1 image (#46) + Release 4 (2018-04-20) ---------------------- Add Dockerfile for Iglu Server (#2)