From f897c37b108da966de75564a9d2677d6278725cc Mon Sep 17 00:00:00 2001 From: Rafal Wojdyla Date: Mon, 8 Sep 2014 04:09:03 +0200 Subject: [PATCH] [#77] Introduce Fig - isolated snakebite testing Notes: * Fig is not integrated with setuptools, running python setup.py test will use Tox for testing * Fig still uses standard python setup.py test (including Tox) but inside separate/isolated OS/container * Requires Docker installed on your machine(!) PR provides fig configuration for: +-------------------- -+-------------+ | description | command | +----------------------+-------------+ | dirty testing image | dirty | | py2.7 and HDP | testPy27hdp | | py2.7 and CDH | testPy27cdh | | py2.6 and HDP | testPy26hdp | | py2.6 and CDH | testPy26cdh | +----------------------+-------------+ So how to start: * fig build ( should build your current dirty image) * fig run testPy27hdp (to run isolated py2.7 tests on HDP) To run all tests at the same time run: * fig up Not tho this is actually requires so resources. Details: 1. Improve setup_env * handle only-download, only-extract, distro-select * add nice flags and nice flag handler 2. Add fig to dev requirements 3. Add Dockerfile for dirty testing image * image used to build dirty testing images * based on ravwojdyla/snakebite_test:base * copies your local dirty code base * install all dev requirements from your dev requirements 4. Add fig configuration file 5. Add Dockerfile for snakebite_test:base * base image for dirty testing images * based on dockerfile/java:oracle-java7 * downloads cdh/hdp tarballs * install python 2.6 and 2.7 * install pip 6. Add script to build snakebite_test:base --- Dockerfile | 5 +++ fig.yml | 34 ++++++++++++++++ requirements-dev.txt | 1 + scripts/Dockerfile | 15 ++++++++ scripts/build-base-test-docker.sh | 4 ++ scripts/ci/setup_env.sh | 64 ++++++++++++++++++++++++++++--- 6 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 Dockerfile create mode 100644 fig.yml create mode 100644 scripts/Dockerfile create mode 100755 scripts/build-base-test-docker.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6482b4e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM ravwojdyla/snakebite_test:base + +ADD . /snakebite +WORKDIR /snakebite +RUN pip install -r requirements-dev.txt diff --git a/fig.yml b/fig.yml new file mode 100644 index 0000000..83dfcf9 --- /dev/null +++ b/fig.yml @@ -0,0 +1,34 @@ +dirty: + build: . + +testPy27cdh: + image: snakebite_dirty + working_dir: /snakebite + environment: + - JAVA_HOME=/usr + - ONLY_EXTRACT=true + command: python2.7 setup.py test --tox-args='-e py27-cdh --recreate' + +testPy27hdp: + image: snakebite_dirty + working_dir: /snakebite + environment: + - JAVA_HOME=/usr + - ONLY_EXTRACT=true + command: python2.7 setup.py test --tox-args='-e py27-hdp --recreate' + +testPy26hdp: + image: snakebite_dirty + working_dir: /snakebite + environment: + - JAVA_HOME=/usr + - ONLY_EXTRACT=true + command: python2.7 setup.py test --tox-args='-e py26-hdp --recreate' + +testPy26cdh: + image: snakebite_dirty + working_dir: /snakebite + environment: + - JAVA_HOME=/usr + - ONLY_EXTRACT=true + command: python2.7 setup.py test --tox-args='-e py26-cdh --recreate' diff --git a/requirements-dev.txt b/requirements-dev.txt index 8d0cfff..8c66d75 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,4 @@ mock virtualenv>=1.11.2 tox>=1.7.2 sphinx +fig diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 0000000..beea0ba --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,15 @@ +FROM dockerfile/java:oracle-java7 + +# Push setup_env script +ADD ./ci/setup_env.sh /setup_env.sh +# Download tarballs for both hdp and cdh +RUN /setup_env.sh --only-download --distro hdp +RUN /setup_env.sh --only-download --distro cdh +# since py2.6 is deprecated in Ubuntu image, we need to add +# ppa to install it +RUN add-apt-repository --yes ppa:fkrull/deadsnakes +RUN apt-get update +RUN apt-get install --yes python2.7 python2.6 +# Fetch pip for python requirments +RUN curl -SL 'https://bootstrap.pypa.io/get-pip.py' | python2.7 +WORKDIR / diff --git a/scripts/build-base-test-docker.sh b/scripts/build-base-test-docker.sh new file mode 100755 index 0000000..97de7fb --- /dev/null +++ b/scripts/build-base-test-docker.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Build snakebite_test:base +docker build -t ravwojdyla/snakebite_test:base . diff --git a/scripts/ci/setup_env.sh b/scripts/ci/setup_env.sh index 9a80bb9..eaf2305 100755 --- a/scripts/ci/setup_env.sh +++ b/scripts/ci/setup_env.sh @@ -1,8 +1,54 @@ #!/usr/bin/env bash HADOOP_DISTRO=${HADOOP_DISTRO:-"hdp"} + +ONLY_DOWNLOAD=${ONLY_DOWNLOAD:-false} +ONLY_EXTRACT=${ONLY_EXTRACT:-false} + +while test $# -gt 0; do + case "$1" in + -h|--help) + echo "Setup environment for snakebite tests" + echo " " + echo "options:" + echo -e "\t-h, --help show brief help" + echo -e "\t-o, --only-download just download hadoop tar(s)" + echo -e "\t-e, --only-extract just extract hadoop tar(s)" + echo -e "\t-d, --distro select distro (hdp|cdh)" + exit 0 + ;; + -o|--only-download) + shift + ONLY_DOWNLOAD=true + ;; + -e|--only-extract) + shift + ONLY_EXTRACT=true + ;; + -d|--distro) + shift + if test $# -gt 0; then + HADOOP_DISTRO=$1 + else + echo "No Hadoop distro specified - abort" >&2 + exit 1 + fi + shift + ;; + *) + echo "Unknown options: $1" >&2 + exit 1 + ;; + esac +done + HADOOP_HOME=/tmp/hadoop-${HADOOP_DISTRO} +if $ONLY_DOWNLOAD && $ONLY_EXTRACT; then + echo "Both only-download and only-extract specified - abort" >&2 + exit 1 +fi + mkdir -p $HADOOP_HOME if [ $HADOOP_DISTRO = "cdh" ]; then @@ -10,16 +56,22 @@ if [ $HADOOP_DISTRO = "cdh" ]; then elif [ $HADOOP_DISTRO = "hdp" ]; then URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0/tars/hadoop-2.2.0.2.0.6.0-76.tar.gz" else - echo "No HADOOP_DISTRO specified" + echo "No/bad HADOOP_DISTRO='${HADOOP_DISTRO}' specified" >&2 exit 1 fi -echo "Downloading Hadoop from $URL to ${HADOOP_HOME}/hadoop.tar.gz" -curl -z ${HADOOP_HOME}/hadoop.tar.gz -o ${HADOOP_HOME}/hadoop.tar.gz -L $URL +if ! $ONLY_EXTRACT; then + echo "Downloading Hadoop from $URL to ${HADOOP_HOME}/hadoop.tar.gz" + curl -z ${HADOOP_HOME}/hadoop.tar.gz -o ${HADOOP_HOME}/hadoop.tar.gz -L $URL -if [ $? != 0 ]; then - echo "Failed to download Hadoop from $URL - abort" >&2 - exit 1 + if [ $? != 0 ]; then + echo "Failed to download Hadoop from $URL - abort" >&2 + exit 1 + fi +fi + +if $ONLY_DOWNLOAD; then + exit 0 fi echo "Extracting ${HADOOP_HOME}/hadoop.tar.gz into $HADOOP_HOME"