Fix permission issue in k8s-dataflow (close #140)

.travis.yml

@@ -25,6 +25,7 @@ env:
   - IMG=piinguin-server/0.1.1
   - IMG=emr-etl-runner/r114_polonnaruwa
   - IMG=k8s-dataflow/0.1.1
+  - IMG=k8s-dataflow/0.2.0
   global:
   # BINTRAY_SNOWPLOW_DOCKER_USER
   - secure: IcBMTopBAIzGlWNgS7dw0iNmvFhd2dgSn/Y1mt6hXnzLrr7SDryRuxPqemsCBLIlf4xtpgHvuOoo+7jYywtAOqyHQ+EOnR5eU8/28v+Z8zd/Djnw0Sv10RV5oMS6ejh9gGS4WDNPvf9M1sfP5p+0gA0p68dNeObLDT4TwpG2pdpeEuqQ3RMGCQc+tb3Q26K6aUPXeIgAHUyIEgHtzxxfgrGNwSmxdb6npsWTUttKpQkJudGRk17xXW67Dd0PKLWJkDpien/lbkLIBK8MlwaH03NMawGcTChgM2njcqElqz1b46wxl1ATcVV2T7A6YPYZxb012P8j/KUZ6MLhdDFNhmz/jIk6caEZ3x4AH2Q0qN8C9Hn70yMN8gAliBeUN4pjLpKnD8t5HgCE+90EufppcQRCfCJhSk9xmegTaikan3QZCN09vwSctIBu4AeWYqctql6bLhNWguakbRIyk9feogmxqbn2HSoFpRrAEqXXv9jFKJrKm2UNKuLbXCvyoYqjAhxTPn7OMAY30wqsVSUu5gSt1fO0Pd+5H8yq44Ul3jROgYKM5NlLWUY+u8qp6KKm2zXQ2LFlr8rzZ2ar5pI9zZiABDMnRRUdAXDTnpDw2HtBvngGXhyfr/irrzach0z/f5YwHL7WVf9Wu7MqoVpexdDqThNGip4ansIikrC57LE=
@@ -96,4 +97,4 @@ deploy:
   skip_cleanup: true
   on:
     tags: true
-    condition: '"$(.travis/is_release_tag.sh k8s-dataflow $TRAVIS_TAG)" == "" && $? == 0 && $IMG = k8s-dataflow/0.1.0'
+    condition: '"$(.travis/is_release_tag.sh k8s-dataflow $TRAVIS_TAG)" == "" && $? == 0'

k8s-dataflow/0.2.0/Dockerfile

@@ -0,0 +1,19 @@
+FROM snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0
+LABEL MAINTAINER="Snowplow Analytics Ltd. <[email protected]>"
+
+RUN \
+  mkdir -p /var/lib/apt/lists/partial &&\
+  echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list &&\
+  apt-get update &&\
+  apt-get install -y apt-transport-https ca-certificates wget gnupg &&\
+  wget -O - http://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - &&\
+  apt-get update &&\
+  apt-get install -y python google-cloud-sdk=302.0.0-0 &&\
+  apt-get purge -y --auto-remove gnupg
+
+USER snowplow
+
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+
+CMD [ "" ]

k8s-dataflow/0.2.0/docker-entrypoint.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+
+DELAY=3
+THRESHOLD=5
+BUCKET=""
+
+ITER=0
+PARAMS=""
+
+# extract params
+for opt in "$@"; do
+  case "$opt" in
+      --tempLocation=*)
+          BUCKET="${opt#*=}"
+          PARAMS="$PARAMS $opt"
+          shift
+          ;;
+      --gcpTempLocation=*)
+          BUCKET="${opt#*=}"
+          PARAMS="$PARAMS $opt"
+          shift
+          ;;
+      --gcsThreshold=*)
+          THRESHOLD="${opt#*=}"
+          shift
+          ;;
+      --gcsDelay=*)
+          DELAY="${opt#*=}"
+          shift
+          ;;
+      *) # preserve positional arguments
+          PARAMS="$PARAMS $opt"
+          shift
+          ;;
+  esac
+done
+
+if [ -z "$BUCKET" ]; then
+    echo "Missing --tempLocation flag. Exiting."
+    exit 1
+fi
+
+echo "params: ${PARAMS} threshold: ${THRESHOLD} delay: ${DELAY} GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-Not Available}"
+
+# enable service account if credentials are available
+if [ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+  echo "GOOGLE_APPLICATION_CREDENTIALS is not defined, gcloud isn't authenticated!"
+  exit 1
+else
+  gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
+fi
+
+# wait for GCS bucket to be available
+while [ "$ITER" -le "$THRESHOLD" ]; do
+  if gsutil ls -b "${BUCKET}";
+  then
+    echo "Bucket ${BUCKET} exists! Proceeding."
+    break
+  else
+    echo "Bucket ${BUCKET} does not exist. Retry: ${ITER}/${THRESHOLD}"
+    sleep "${DELAY}"
+    ITER=$(( ITER+1 ))
+  fi
+done
+
+# check if retry limit was reached or not
+if [ "$ITER" -le "$THRESHOLD" ]; then
+  $PARAMS
+else
+  echo "Bucket ${BUCKET} does not exist. Not retrying anymore."
+  exit 1
+fi