Merge branch 'release-6'

.travis.yml

@@ -10,14 +10,15 @@ script:
 env:
   matrix:
   - IMG=base
+  - IMG=base-debian
   - IMG=scala-stream-collector/0.13.0/google-pubsub
   - IMG=scala-stream-collector/0.13.0/kafka
   - IMG=scala-stream-collector/0.13.0/kinesis
   - IMG=scala-stream-collector/0.13.0/nsq
-  - IMG=stream-enrich/0.16.1/google-pubsub
-  - IMG=stream-enrich/0.16.1/kafka
-  - IMG=stream-enrich/0.16.1/kinesis
-  - IMG=stream-enrich/0.16.1/nsq
+  - IMG=stream-enrich/0.17.0/google-pubsub
+  - IMG=stream-enrich/0.17.0/kafka
+  - IMG=stream-enrich/0.17.0/kinesis
+  - IMG=stream-enrich/0.17.0/nsq
   - IMG=s3-loader/0.6.0
   - IMG=elasticsearch-loader/0.10.1
   - IMG=iglu-server/0.3.0
@@ -33,6 +34,12 @@ deploy:
   on:
     tags: true
     condition: '"$(.travis/is_release_tag.sh base $TRAVIS_TAG)" == "" && $? == 0 && $IMG = base'
+- provider: script
+  script: ./.travis/deploy.sh base-debian $TRAVIS_TAG
+  skip_cleanup: true
+  on:
+    tags: true
+    condition: '"$(.travis/is_release_tag.sh base-debian $TRAVIS_TAG)" == "" && $? == 0 && $IMG = base-debian'
 - provider: script
   script: ./.travis/all_platforms_deploy.sh scala-stream-collector $TRAVIS_TAG
   skip_cleanup: true

.travis/deploy.sh

@@ -27,7 +27,7 @@ curl -X GET \
 
 cd $project
 # except for the base image, there is one folder per image version
-if [ "${project}" != "base" ]; then
+if [[ "${project}" != base* ]]; then
     if [ -d "${release}" ]; then
         cd $release
     else
@@ -39,4 +39,4 @@ fi
 img_tag="snowplow/${project}:${release}"
 final_tag="${docker_repo}/${img_tag}"
 docker build -t $final_tag .
-docker push $final_tag
+docker push $final_tag

README.md

@@ -24,7 +24,7 @@ You can pull the images from the registry directly:
 docker pull snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0
 
 # NSQ Stream Enrich image, there are others available for Kinesis, Kafka and Google PubSub
-docker pull snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1
+docker pull snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.17.0
 
 # Elasticsearch Loader image
 docker pull snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1
@@ -41,14 +41,15 @@ docker pull snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.3.0
 Alternatively, you can build them yourself:
 
 ```bash
-# All images are based on the base image
+# All images are based on the base images
 docker pull snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+docker pull snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
 
 # NSQ Scala Stream Collector image, there are others available for Kinesis, Kafka and Google PubSub
 docker build -t snowplow/scala-stream-collector-nsq:0.13.0 scala-stream-collector/0.13.0/nsq
 
 # NSQ Stream Enrich image, there are others available for Kinesis, Kafka and Google PubSub
-docker build -t snowplow/stream-enrich-nsq:0.16.1 stream-enrich/0.16.1/nsq
+docker build -t snowplow/stream-enrich-nsq:0.17.0 stream-enrich/0.17.0/nsq
 
 # Elasticsearch Loader image
 docker build -t snowplow/elasticsearch-loader:0.10.1 elasticsearch-loader/0.10.1
@@ -84,8 +85,8 @@ docker run \
 # NSQ Stream Enrich container, there are others available for Kinesis, Kafka and Google PubSub
 docker run \
   -v $PWD/stream-enrich-config:/snowplow/config \
-  snowplow/stream-enrich-nsq:0.16.1 \ # if you have built the image
-  # snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1 if you have pulled the image
+  snowplow/stream-enrich-nsq:0.17.0 \ # if you have built the image
+  # snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.17.0 if you have pulled the image
   --config /snowplow/config/config.hocon \
   --resolver file:/snowplow/config/resolver.json \
   --enrichments file:/snowplow/config/enrichments/ \

base-debian/Dockerfile

@@ -0,0 +1,44 @@
+FROM openjdk:8u171-jre-slim
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# Snowplow components will be installed in this folder.
+ENV SNOWPLOW_PATH="/snowplow"
+ENV SNOWPLOW_CONFIG_PATH="${SNOWPLOW_PATH}/config" \
+    SNOWPLOW_BIN_PATH="${SNOWPLOW_PATH}/bin"
+
+# Create a snowplow group and user.
+RUN addgroup snowplow && \
+    adduser --system --ingroup snowplow snowplow
+
+# Install the components common to all apps.
+# https://github.com/yelp/dumb-init: lightweight init system
+# https://github.com/tianon/gosu/: sudo replacement
+RUN apt-get update && \
+    apt-get install -y ca-certificates wget gnupg && \
+    rm -rf /var/lib/apt/lists && \
+    wget https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    export GOSU_VERSION=1.10 && \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64.asc" && \
+    export GNUPGHOME="$(mktemp -d)" && \
+    for server in $(shuf -e ha.pool.sks-keyservers.net \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
+    done && \
+    gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \
+    rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc && \
+    chmod +x /usr/local/bin/gosu && \
+    apt-get purge -y --auto-remove gnupg
+
+# /snowplow/bin is meant to contain the application jar.
+# /snowplow/config is meant to contain the necessary configuration.
+RUN mkdir -p ${SNOWPLOW_BIN_PATH} && \
+    mkdir -p ${SNOWPLOW_CONFIG_PATH} && \
+    chown -R snowplow:snowplow ${SNOWPLOW_PATH}
+
+# Expose the configuration directory.
+VOLUME ${SNOWPLOW_CONFIG_PATH}

base/Dockerfile

@@ -12,7 +12,7 @@ RUN addgroup snowplow && \
 
 # Install the components common to all apps.
 # https://github.com/yelp/dumb-init: lightweight init system
-# su-exec: sudo replacement
+# https://github.com/ncopa/su-exec: sudo replacement
 RUN apk add --no-cache dumb-init su-exec ca-certificates wget
 
 # /snowplow/bin is meant to contain the application jar.

example/docker-compose.yml

@@ -37,7 +37,7 @@ services:
     logging:
       options:
         max-size: 1M
-        max-file: 10
+        max-file: "10"
     deploy:
       resources:
         limits:
@@ -65,7 +65,7 @@ services:
     logging:
       options:
         max-size: 1M
-        max-file: 10
+        max-file: "10"
     deploy:
       resources:
         limits:

scala-stream-collector/0.13.0/google-pubsub/Dockerfile

@@ -1,4 +1,4 @@
-FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
 LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
 
 # The version of the collector to download.

scala-stream-collector/0.13.0/google-pubsub/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/dumb-init /bin/sh
+#!/usr/bin/dumb-init /bin/bash
 set -e
 
 # If the config directory has been mounted through -v, we chown it.
@@ -7,6 +7,6 @@ if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
 fi
 
 # Make sure we run the collector as the snowplow user
-exec su-exec snowplow:snowplow /usr/bin/java \
+exec gosu snowplow:snowplow /usr/bin/java \
   $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
   -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-collector-${PLATFORM//_/-}-${COLLECTOR_VERSION}.jar "$@"

scala-stream-collector/README.md

@@ -12,7 +12,8 @@ There is one image per targeted platform:
 ## Introduction
 
 This image is based on [the base image][base-image] which leverages
-[the Java 8 Alpine image][alpine-image].
+[the Java 8 Alpine image][alpine-image] (the Google PubSub image leverages
+[the Java 8 Debian image][debian-image] due to incompatibilities with Alpine).
 
 The Scala Stream Collector runs under [dumb-init][dumb-init] which handles reaping zombie processes
 and forwards signals on to all processes running in the container. This image also uses
@@ -86,6 +87,7 @@ limitations under the License.
 [base-image]: https://github.com/snowplow/snowplow-docker/tree/master/base
 [docker-compose-example]: https://github.com/snowplow/snowplow-docker/tree/master/example
 [alpine-image]: https://github.com/docker-library/openjdk/blob/master/8-jre/alpine/Dockerfile
+[debian-image]: https://github.com/docker-library/openjdk/blob/master/8-jre/slim/Dockerfile
 
 [ssc]: https://github.com/snowplow/snowplow/tree/master/2-collectors/scala-stream-collector
 [dumb-init]: https://github.com/Yelp/dumb-init

stream-enrich/0.15.0/google-pubsub/Dockerfile

@@ -1,4 +1,4 @@
-FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
 LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
 
 # The version of stream enrich to download.

stream-enrich/0.15.0/google-pubsub/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/dumb-init /bin/sh
+#!/usr/bin/dumb-init /bin/bash
 set -e
 
 # If the config directory has been mounted through -v, we chown it.
@@ -10,6 +10,6 @@ fi
 cd $(eval echo ~snowplow)
 
 # Make sure we run the collector as the snowplow user
-exec su-exec snowplow:snowplow /usr/bin/java \
+exec gosu snowplow:snowplow /usr/bin/java \
   $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
   -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM//_/-}-${ENRICH_VERSION}.jar "$@"

stream-enrich/0.16.0/google-pubsub/Dockerfile

@@ -1,4 +1,4 @@
-FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
 LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
 
 # The version of stream enrich to download.

stream-enrich/0.16.0/google-pubsub/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/dumb-init /bin/sh
+#!/usr/bin/dumb-init /bin/bash
 set -e
 
 # If the config directory has been mounted through -v, we chown it.
@@ -10,6 +10,6 @@ fi
 cd $(eval echo ~snowplow)
 
 # Make sure we run the collector as the snowplow user
-exec su-exec snowplow:snowplow /usr/bin/java \
+exec gosu snowplow:snowplow /usr/bin/java \
   $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
   -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM//_/-}-${ENRICH_VERSION}.jar "$@"

stream-enrich/0.16.1/google-pubsub/Dockerfile

@@ -1,4 +1,4 @@
-FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
 LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
 
 # The version of stream enrich to download.

stream-enrich/0.16.1/google-pubsub/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/dumb-init /bin/sh
+#!/usr/bin/dumb-init /bin/bash
 set -e
 
 # If the config directory has been mounted through -v, we chown it.
@@ -10,6 +10,6 @@ fi
 cd $(eval echo ~snowplow)
 
 # Make sure we run the collector as the snowplow user
-exec su-exec snowplow:snowplow /usr/bin/java \
+exec gosu snowplow:snowplow /usr/bin/java \
   $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
   -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM//_/-}-${ENRICH_VERSION}.jar "$@"

stream-enrich/0.17.0/google-pubsub/Dockerfile

@@ -0,0 +1,26 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of stream enrich to download.
+ENV ENRICH_VERSION="0.17.0"
+
+# The targeted platform
+ENV PLATFORM="google_pubsub"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

stream-enrich/0.17.0/google-pubsub/docker-entrypoint.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/dumb-init /bin/bash
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Needed because of SCE's ./ip_geo file
+cd $(eval echo ~snowplow)
+
+# Make sure we run the collector as the snowplow user
+exec gosu snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM//_/-}-${ENRICH_VERSION}.jar "$@"

stream-enrich/0.17.0/kafka/Dockerfile

@@ -0,0 +1,26 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of stream enrich to download.
+ENV ENRICH_VERSION="0.17.0"
+
+# The targeted platform
+ENV PLATFORM="kafka"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

stream-enrich/0.17.0/kafka/docker-entrypoint.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Needed because of SCE's ./ip_geo file
+cd $(eval echo ~snowplow)
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM}-${ENRICH_VERSION}.jar "$@"

stream-enrich/0.17.0/kinesis/Dockerfile

@@ -0,0 +1,26 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base:0.1.0
+LABEL maintainer="Snowplow Analytics Ltd. <[email protected]>"
+
+# The version of stream enrich to download.
+ENV ENRICH_VERSION="0.17.0"
+
+# The targeted platform
+ENV PLATFORM="kinesis"
+
+# The name of the archive to download.
+ENV ARCHIVE="snowplow_stream_enrich_${PLATFORM}_${ENRICH_VERSION}.zip"
+
+# Install the Scala Stream Collector.
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q http://dl.bintray.com/snowplow/snowplow-generic/${ARCHIVE} && \
+    unzip -d ${SNOWPLOW_BIN_PATH} ${ARCHIVE} && \
+    cd /tmp && \
+    rm -rf /tmp/build
+
+# Defines an entrypoint script delegating the lauching of stream enrich to the snowplow user.
+# The script uses dumb-init as the top-level process.
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "--help" ]

stream-enrich/0.17.0/kinesis/docker-entrypoint.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/dumb-init /bin/sh
+set -e
+
+# If the config directory has been mounted through -v, we chown it.
+if [ "$(stat -c %u ${SNOWPLOW_CONFIG_PATH})" != "$(id -u snowplow)" ]; then
+  chown snowplow:snowplow ${SNOWPLOW_CONFIG_PATH}
+fi
+
+# Needed because of SCE's ./ip_geo file
+cd $(eval echo ~snowplow)
+
+# Make sure we run the collector as the snowplow user
+exec su-exec snowplow:snowplow /usr/bin/java \
+  $SP_JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \
+  -jar ${SNOWPLOW_BIN_PATH}/snowplow-stream-enrich-${PLATFORM}-${ENRICH_VERSION}.jar "$@"