diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..afbb9a7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,280 @@ +FROM nvidia/cuda:8.0-cudnn6-devel +LABEL maintainer="Ming " + +# ================================= +# cuda 8.0 +# cudnn v6 +# --------------------------------- +# python 3.5 +# opencv latest (git) +# --------------------------------- +# tensorflow latest (pip) +# sonnet latest (pip) +# pytorch 0.2.0 (pip) +# keras latest (pip) +# mxnet latest (pip) +# cntk 2.2 (pip) +# chainer latest (pip) +# theano latest (git) +# lasagne latest (git) +# caffe latest (git) +# torch latest (git) +# --------------------------------- + +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + +# ================================= +# apt +# ================================= + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + apt-utils \ + && \ + +# ================================= +# common tools +# ================================= + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + wget \ + git \ + vim \ + && \ + +# ================================= +# cmake +# ================================= + + # fix boost-not-found issue caused by the `apt-get` version of cmake + $GIT_CLONE https://github.com/Kitware/CMake ~/cmake && \ + cd ~/cmake && \ + ./bootstrap --prefix=/usr/local && \ + make -j"$(nproc)" install && \ + +# ================================= +# python3 +# ================================= + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3-pip \ + python3-dev \ + && \ + ln -s /usr/bin/python3 /usr/local/bin/python && \ + pip3 --no-cache-dir install --upgrade pip && \ + $PIP_INSTALL \ + setuptools \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================= +# opencv +# ================================= + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libboost-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ + + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j"$(nproc)" install && \ + +# ================================= +# tensorflow +# ================================= + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================= +# sonnet +# ================================= + + $PIP_INSTALL \ + dm-sonnet \ + && \ + +# ================================= +# mxnet +# ================================= + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + graphviz \ + && \ + + $PIP_INSTALL \ + mxnet-cu80 \ + graphviz \ + && \ + +# ================================= +# cntk +# ================================= + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + openmpi-bin \ + libjasper-dev \ + && \ + + $PIP_INSTALL \ + https://cntk.ai/PythonWheel/GPU/cntk-2.2-cp35-cp35m-linux_x86_64.whl \ + && \ + +# ================================= +# keras +# ================================= + + $PIP_INSTALL \ + h5py \ + keras \ + && \ + +# ================================= +# pytorch +# ================================= + + $PIP_INSTALL \ + http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp35-cp35m-manylinux1_x86_64.whl \ + torchvision \ + && \ + +# ================================= +# chainer +# ================================= + + $PIP_INSTALL \ + cupy \ + chainer \ + && \ + +# ================================= +# theano +# ================================= + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' \ + > ~/.theanorc && \ + +# ================================= +# lasagne +# ================================= + + $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ + cd ~/lasagne && \ + $PIP_INSTALL \ + . && \ + +# ================================= +# caffe +# ================================= + + $GIT_CLONE https://github.com/NVIDIA/nccl && \ + cd nccl; make -j"$(nproc)" install; cd ..; rm -rf nccl && \ + + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + mkdir ~/caffe/build && cd ~/caffe/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D USE_CUDNN=1 \ + -D USE_NCCL=1 \ + -D python_version=3 \ + -D CUDA_NVCC_FLAGS=--Wno-deprecated-gpu-targets \ + -Wno-dev \ + .. && \ + make -j"$(nproc)" install && \ + + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ + + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + + mv /usr/local/python/caffe /usr/local/lib/python3.5/dist-packages/ && \ + rm -rf /usr/local/python && \ + +# ================================= +# torch +# ================================= + + $GIT_CLONE https://github.com/torch/distro.git ~/torch --recursive&& \ + + cd ~/torch/exe/luajit-rocks && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_LUAJIT21=ON \ + .. && \ + make -j"$(nproc)" install && \ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libreadline-dev \ + && \ + + $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ + cd ~/moses && \ + luarocks install rockspec/moses-1.6.1-1.rockspec && \ + + cd ~/torch && \ + sed -i 's/extra\/cudnn/extra\/cudnn \&\& git checkout R6/' install.sh && \ + sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ + sed -i '/qt/d' install.sh && \ + sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ + sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ + sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ + sed -i '/You might want to/,/^fi$/d' install.sh && \ + yes no | ./install.sh && \ + + +# ================================= +# config & cleanup +# ================================= + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..df7c0b0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Ming + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b56629b --- /dev/null +++ b/README.md @@ -0,0 +1,200 @@ +![deepo](https://user-images.githubusercontent.com/2270240/32102393-aecf573c-bb4e-11e7-811c-dc673cae7b9c.png) + +[![CircleCI](https://img.shields.io/circleci/project/github/ufoym/deepo.svg)](https://circleci.com/gh/ufoym/deepo) +![license](https://img.shields.io/github/license/ufoym/deepo.svg) + + +***Deepo*** is a [*Docker*](http://www.docker.com/) image with a full reproducible deep learning research environment. It contains most popular deep learning frameworks: +[theano](http://deeplearning.net/software/theano), +[tensorflow](http://www.tensorflow.org), +[sonnet](https://github.com/deepmind/sonnet), +[pytorch](http://pytorch.org), +[keras](https://keras.io), +[lasagne](http://lasagne.readthedocs.io), +[mxnet](http://mxnet.incubator.apache.org), +[cntk](https://www.microsoft.com/en-us/cognitive-toolkit), +[chainer](https://chainer.org), +[caffe](http://caffe.berkeleyvision.org), +[torch](http://torch.ch/). + +- [Quick Start](#Quick-Start) + - [Installation](#Installation) + - [Usage](#Usage) +- [Comparison to Alternatives](#Comparison) +- [Licensing](#Licensing) + +--- + + + +## Quick Start + + + + +### Installation + +#### Step 1. Install [Docker](https://docs.docker.com/engine/installation/) and [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). + +#### Step 2. Obtain the Deepo image + +You can either directly download the image from Docker Hub, or build the image yourself. + +##### Option 1: Get the image from Docker Hub (recommended) +```bash +docker pull ufoym/deepo +``` +##### Option 2: Build the Docker image locally +```bash +git clone https://github.com/ufoym/deepo.git +cd deepo && docker build -t ufoym/deepo . +``` +Note that this may take several hours as it compiles a few libraries from scratch. + + + +### Usage + +Now you can try this command: +```bash +nvidia-docker run --rm ufoym/deepo nvidia-smi +``` +This should work and enables Deepo to use the GPU from inside a docker container. +If this does not work, search [the issues section on the nvidia-docker GitHub](https://github.com/NVIDIA/nvidia-docker/issues) -- many solutions are already documented. To get an interactive shell to a container that will not be automatically deleted after you exit do + +```bash +nvidia-docker run -it ufoym/deepo bash +``` + +If you want to share your data and configurations between the host (your machine or VM) and the container in which you are using Deepo, use the -v option, e.g. +```bash +nvidia-docker run -it -v /host/data:/data -v /host/config:/config ufoym/deepo bash +``` +This will make `/host/data` from the host visible as `/data` in the container, and `/host/config` as `/config`. Such isolation reduces the chances of your containerized experiments overwriting or using wrong data. + + +_You are now ready to begin your journey._ + + +#### tensorflow +```$ python``` +```python +>>> import tensorflow +>>> print(tensorflow.__name__, tensorflow.__version__) +tensorflow 1.3.0 +``` + +#### sonnet +```$ python``` +```python +>>> import sonnet +>>> print(sonnet.__name__, sonnet.__path__) +sonnet ['/usr/local/lib/python3.5/dist-packages/sonnet'] +``` + +#### pytorch +```$ python``` +```python +>>> import torch +>>> print(torch.__name__, torch.__version__) +torch 0.2.0_3 +``` + +#### keras +```$ python``` +```python +>>> import keras +>>> print(keras.__name__, keras.__version__) +keras 2.0.8 +``` + +#### mxnet +```$ python``` +```python +>>> import mxnet +>>> print(mxnet.__name__, mxnet.__version__) +mxnet 0.11.0 +``` + +#### cntk +```$ python``` +```python +>>> import cntk +>>> print(cntk.__name__, cntk.__version__) +cntk 2.2 +``` + +#### chainer +```$ python``` +```python +>>> import chainer +>>> print(chainer.__name__, chainer.__version__) +chainer 3.0.0 +``` + +#### theano +```$ python``` +```python +>>> import theano +>>> print(theano.__name__, theano.__version__) +theano 0.10.0beta4+14.gb6e3768 +``` + +#### lasagne +```$ python``` +```python +>>> import lasagne +>>> print(lasagne.__name__, lasagne.__version__) +lasagne 0.2.dev1 +``` + +#### caffe +```$ python``` +```python +>>> import caffe +>>> print(caffe.__name__, caffe.__version__) +caffe 1.0.0 +``` + +```$ caffe --version``` +``` +caffe version 1.0.0 +``` + +#### torch +```$ th``` +``` + │ ______ __ | Torch7 + │ /_ __/__ ________/ / | Scientific computing for Lua. + │ / / / _ \/ __/ __/ _ \ | Type ? for help + │ /_/ \___/_/ \__/_//_/ | https://github.com/torch + │ | http://torch.ch + │ + │th> +``` + + + +## Comparison to alternatives +. | modern-deep-learning | dl-docker | jupyter-deeplearning | Deepo +:------------------------------------------------: | :------------------: | :----------------: | :------------------: | :----------------: + [ubuntu](https://www.ubuntu.com) | 16.04 | 14.04 | 14.04 | 16.04 + [cuda](https://developer.nvidia.com/cuda-zone) | :x: | 8.0 | 6.5-8.0 | 8.0 + [cudnn](https://developer.nvidia.com/cudnn) | :x: | v5 | v2-5 | v6 + [theano](http://deeplearning.net/software/theano) | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + [tensorflow](http://www.tensorflow.org) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + [sonnet](https://github.com/deepmind/sonnet) | :x: | :x: | :x: | :heavy_check_mark: + [pytorch](http://pytorch.org) | :x: | :x: | :x: | :heavy_check_mark: + [keras](https://keras.io) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + [lasagne](http://lasagne.readthedocs.io) | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + [mxnet](http://mxnet.incubator.apache.org) | :x: | :x: | :x: | :heavy_check_mark: + [cntk](http://cntk.ai) | :x: | :x: | :x: | :heavy_check_mark: + [chainer](https://chainer.org) | :x: | :x: | :x: | :heavy_check_mark: + [caffe](http://caffe.berkeleyvision.org) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + [torch](http://torch.ch/) | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + + + +## Licensing + +Deepo is [MIT licensed](https://github.com/ufoym/deepo/blob/master/LICENSE). diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..cd4e26b --- /dev/null +++ b/circle.yml @@ -0,0 +1,13 @@ +version: 2 +jobs: + build: + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO . + - run: | + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker push $DOCKER_REPO + + +