[#77] Introduce Fig - isolated snakebite testing

Notes:
 * Fig is not integrated with setuptools, running python setup.py test
   will use Tox for testing
 * Fig still uses standard python setup.py test (including Tox) but
   inside separate/isolated OS/container
 * Requires Docker installed on your machine(!)

PR provides fig configuration for:

+-------------------- -+-------------+
|  description         |   command   |
+----------------------+-------------+
|  dirty testing image |    dirty    |
|    py2.7 and HDP     | testPy27hdp |
|    py2.7 and CDH     | testPy27cdh |
|    py2.6 and HDP     | testPy26hdp |
|    py2.6 and CDH     | testPy26cdh |
+----------------------+-------------+

So how to start:
 * fig build ( should build your current dirty image)
 * fig run testPy27hdp (to run isolated py2.7 tests on HDP)

To run all tests at the same time run:
 * fig up
Not tho this is actually requires so resources.

Details:

1. Improve setup_env

 * handle only-download, only-extract, distro-select
 * add nice flags and nice flag handler

2. Add fig to dev requirements

3. Add Dockerfile for dirty testing image

 * image used to build dirty testing images
 * based on ravwojdyla/snakebite_test:base
 * copies your local dirty code base
 * install all dev requirements from your dev requirements

4. Add fig configuration file

5. Add Dockerfile for snakebite_test:base

 * base image for dirty testing images
 * based on dockerfile/java:oracle-java7
 * downloads cdh/hdp tarballs
 * install python 2.6 and 2.7
 * install pip

6. Add script to build snakebite_test:base

Dockerfile

@@ -0,0 +1,5 @@
+FROM ravwojdyla/snakebite_test:base
+
+ADD . /snakebite
+WORKDIR /snakebite
+RUN pip install -r requirements-dev.txt

fig.yml

@@ -0,0 +1,34 @@
+dirty:
+    build: .
+
+testPy27cdh:
+    image: snakebite_dirty
+    working_dir: /snakebite
+    environment:
+        - JAVA_HOME=/usr
+        - ONLY_EXTRACT=true
+    command: python2.7 setup.py test --tox-args='-e py27-cdh --recreate'
+
+testPy27hdp:
+    image: snakebite_dirty
+    working_dir: /snakebite
+    environment:
+        - JAVA_HOME=/usr
+        - ONLY_EXTRACT=true
+    command: python2.7 setup.py test --tox-args='-e py27-hdp --recreate'
+
+testPy26hdp:
+    image: snakebite_dirty
+    working_dir: /snakebite
+    environment:
+        - JAVA_HOME=/usr
+        - ONLY_EXTRACT=true
+    command: python2.7 setup.py test --tox-args='-e py26-hdp --recreate'
+
+testPy26cdh:
+    image: snakebite_dirty
+    working_dir: /snakebite
+    environment:
+        - JAVA_HOME=/usr
+        - ONLY_EXTRACT=true
+    command: python2.7 setup.py test --tox-args='-e py26-cdh --recreate'

requirements-dev.txt

@@ -5,3 +5,4 @@ mock
 virtualenv>=1.11.2
 tox>=1.7.2
 sphinx
+fig

scripts/Dockerfile

@@ -0,0 +1,15 @@
+FROM dockerfile/java:oracle-java7
+
+# Push setup_env script
+ADD ./ci/setup_env.sh /setup_env.sh
+# Download tarballs for both hdp and cdh
+RUN /setup_env.sh --only-download --distro hdp
+RUN /setup_env.sh --only-download --distro cdh
+# since py2.6 is deprecated in Ubuntu image, we need to add
+# ppa to install it
+RUN add-apt-repository --yes ppa:fkrull/deadsnakes
+RUN apt-get update
+RUN apt-get install --yes python2.7 python2.6
+# Fetch pip for python requirments
+RUN curl -SL 'https://bootstrap.pypa.io/get-pip.py' | python2.7
+WORKDIR /

scripts/build-base-test-docker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Build snakebite_test:base
+docker build -t ravwojdyla/snakebite_test:base .

scripts/ci/setup_env.sh

@@ -1,25 +1,77 @@
 #!/usr/bin/env bash
 
 HADOOP_DISTRO=${HADOOP_DISTRO:-"hdp"}
+
+ONLY_DOWNLOAD=${ONLY_DOWNLOAD:-false}
+ONLY_EXTRACT=${ONLY_EXTRACT:-false}
+
+while test $# -gt 0; do
+    case "$1" in
+        -h|--help)
+            echo "Setup environment for snakebite tests"
+            echo " "
+            echo "options:"
+            echo -e "\t-h, --help            show brief help"
+            echo -e "\t-o, --only-download   just download hadoop tar(s)"
+            echo -e "\t-e, --only-extract    just extract hadoop tar(s)"
+            echo -e "\t-d, --distro          select distro (hdp|cdh)"
+            exit 0
+            ;;
+        -o|--only-download)
+            shift
+            ONLY_DOWNLOAD=true
+            ;;
+        -e|--only-extract)
+            shift
+            ONLY_EXTRACT=true
+            ;;
+        -d|--distro)
+            shift
+            if test $# -gt 0; then
+                HADOOP_DISTRO=$1
+            else
+                echo "No Hadoop distro specified - abort" >&2
+                exit 1
+            fi
+            shift
+            ;;
+        *)
+            echo "Unknown options: $1" >&2
+            exit 1
+            ;;
+    esac
+done
+
 HADOOP_HOME=/tmp/hadoop-${HADOOP_DISTRO}
 
+if $ONLY_DOWNLOAD && $ONLY_EXTRACT; then
+    echo "Both only-download and only-extract specified - abort" >&2
+    exit 1
+fi
+
 mkdir -p $HADOOP_HOME
 
 if [ $HADOOP_DISTRO = "cdh" ]; then
     URL="http://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz"
 elif [ $HADOOP_DISTRO = "hdp" ]; then
     URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0/tars/hadoop-2.2.0.2.0.6.0-76.tar.gz"
 else
-    echo "No HADOOP_DISTRO specified"
+    echo "No/bad HADOOP_DISTRO='${HADOOP_DISTRO}' specified" >&2
     exit 1
 fi
 
-echo "Downloading Hadoop from $URL to ${HADOOP_HOME}/hadoop.tar.gz"
-curl -z ${HADOOP_HOME}/hadoop.tar.gz -o ${HADOOP_HOME}/hadoop.tar.gz -L $URL
+if ! $ONLY_EXTRACT; then
+    echo "Downloading Hadoop from $URL to ${HADOOP_HOME}/hadoop.tar.gz"
+    curl -z ${HADOOP_HOME}/hadoop.tar.gz -o ${HADOOP_HOME}/hadoop.tar.gz -L $URL
 
-if [ $? != 0 ]; then
-    echo "Failed to download Hadoop from $URL - abort" >&2
-    exit 1
+    if [ $? != 0 ]; then
+        echo "Failed to download Hadoop from $URL - abort" >&2
+        exit 1
+    fi
+fi
+
+if $ONLY_DOWNLOAD; then
+    exit 0
 fi
 
 echo "Extracting ${HADOOP_HOME}/hadoop.tar.gz into $HADOOP_HOME"