diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71e6852 --- /dev/null +++ b/.gitignore @@ -0,0 +1,62 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv python configuration file +.python-version diff --git a/README.md b/README.md index b56629b..c4b78e7 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,53 @@ ![deepo](https://user-images.githubusercontent.com/2270240/32102393-aecf573c-bb4e-11e7-811c-dc673cae7b9c.png) [![CircleCI](https://img.shields.io/circleci/project/github/ufoym/deepo.svg)](https://circleci.com/gh/ufoym/deepo) +[![docker](https://img.shields.io/docker/pulls/ufoym/deepo.svg)](https://hub.docker.com/r/ufoym/deepo) ![license](https://img.shields.io/github/license/ufoym/deepo.svg) -***Deepo*** is a [*Docker*](http://www.docker.com/) image with a full reproducible deep learning research environment. It contains most popular deep learning frameworks: -[theano](http://deeplearning.net/software/theano), -[tensorflow](http://www.tensorflow.org), -[sonnet](https://github.com/deepmind/sonnet), -[pytorch](http://pytorch.org), -[keras](https://keras.io), -[lasagne](http://lasagne.readthedocs.io), -[mxnet](http://mxnet.incubator.apache.org), -[cntk](https://www.microsoft.com/en-us/cognitive-toolkit), -[chainer](https://chainer.org), -[caffe](http://caffe.berkeleyvision.org), -[torch](http://torch.ch/). +***Deepo*** is a series of [*Docker*](http://www.docker.com/) images that +- allows you to quickly set up your deep learning research environment +- supports almost all commonly used deep learning frameworks: [theano](http://deeplearning.net/software/theano), [tensorflow](http://www.tensorflow.org), [sonnet](https://github.com/deepmind/sonnet), [pytorch](http://pytorch.org), [keras](https://keras.io), [lasagne](http://lasagne.readthedocs.io), [mxnet](http://mxnet.incubator.apache.org), [cntk](https://www.microsoft.com/en-us/cognitive-toolkit), [chainer](https://chainer.org), [caffe](http://caffe.berkeleyvision.org), [torch](http://torch.ch/) +and their Dockerfile generator that +- allows you to customize your own environment with Lego-like modules +- automatically resolves the dependencies for you + + +# Table of contents - [Quick Start](#Quick-Start) - [Installation](#Installation) - [Usage](#Usage) +- [Customization](#Customization) + - [I hate all-in-one solution](#One) + - [Other python versions](#Python) + - [Build your own customized image](#Build) - [Comparison to Alternatives](#Comparison) +- [Contributing](#Contributing) - [Licensing](#Licensing) --- -## Quick Start +# Quick Start -### Installation +## Installation #### Step 1. Install [Docker](https://docs.docker.com/engine/installation/) and [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). -#### Step 2. Obtain the Deepo image - -You can either directly download the image from Docker Hub, or build the image yourself. +#### Step 2. Obtain the all-in-one image from [Docker Hub](https://hub.docker.com/r/ufoym/deepo) -##### Option 1: Get the image from Docker Hub (recommended) ```bash docker pull ufoym/deepo ``` -##### Option 2: Build the Docker image locally -```bash -git clone https://github.com/ufoym/deepo.git -cd deepo && docker build -t ufoym/deepo . -``` -Note that this may take several hours as it compiles a few libraries from scratch. -### Usage +## Usage Now you can try this command: ```bash @@ -72,88 +66,27 @@ nvidia-docker run -it -v /host/data:/data -v /host/config:/config ufoym/deepo ba ``` This will make `/host/data` from the host visible as `/data` in the container, and `/host/config` as `/config`. Such isolation reduces the chances of your containerized experiments overwriting or using wrong data. +Please note that some frameworks (e.g. PyTorch) use shared memory to share data between processes, so if multiprocessing is used the default shared memory segment size that container runs with is not enough, and you should increase shared memory size either with `--ipc=host` or `--shm-size` command line options to `nvidia-docker run`. +```bash +nvidia-docker run -it --ipc=host ufoym/deepo bash +``` + _You are now ready to begin your journey._ -#### tensorflow ```$ python``` ```python >>> import tensorflow ->>> print(tensorflow.__name__, tensorflow.__version__) -tensorflow 1.3.0 -``` - -#### sonnet -```$ python``` -```python >>> import sonnet ->>> print(sonnet.__name__, sonnet.__path__) -sonnet ['/usr/local/lib/python3.5/dist-packages/sonnet'] -``` - -#### pytorch -```$ python``` -```python >>> import torch ->>> print(torch.__name__, torch.__version__) -torch 0.2.0_3 -``` - -#### keras -```$ python``` -```python >>> import keras ->>> print(keras.__name__, keras.__version__) -keras 2.0.8 -``` - -#### mxnet -```$ python``` -```python >>> import mxnet ->>> print(mxnet.__name__, mxnet.__version__) -mxnet 0.11.0 -``` - -#### cntk -```$ python``` -```python >>> import cntk ->>> print(cntk.__name__, cntk.__version__) -cntk 2.2 -``` - -#### chainer -```$ python``` -```python >>> import chainer ->>> print(chainer.__name__, chainer.__version__) -chainer 3.0.0 -``` - -#### theano -```$ python``` -```python >>> import theano ->>> print(theano.__name__, theano.__version__) -theano 0.10.0beta4+14.gb6e3768 -``` - -#### lasagne -```$ python``` -```python >>> import lasagne ->>> print(lasagne.__name__, lasagne.__version__) -lasagne 0.2.dev1 -``` - -#### caffe -```$ python``` -```python >>> import caffe ->>> print(caffe.__name__, caffe.__version__) -caffe 1.0.0 ``` ```$ caffe --version``` @@ -161,7 +94,6 @@ caffe 1.0.0 caffe version 1.0.0 ``` -#### torch ```$ th``` ``` │ ______ __ | Torch7 @@ -173,9 +105,78 @@ caffe version 1.0.0 │th> ``` + + + +# Customization + +Note that `docker pull ufoym/deepo` mentioned in [Quick Start](#Quick-Start) will give you a standard image containing all available deep learning frameworks. You can customize your own environment as well. + + + +## I hate all-in-one solution + +If you prefer a specific framework rather than an all-in-one image, just append a tag with the name of the framework. +Take tensorflow for example: +```bash +docker pull ufoym/deepo:tensorflow +``` + + + +## Other python versions + +Note that all python-related images use `Python 3.6` by default. If you are unhappy with `Python 3.6`, you can also specify other python versions: +```bash +docker pull ufoym/deepo:py27 +``` + +```bash +docker pull ufoym/deepo:tensorflow-py27 +``` + +Currently, we support `Python 2.7` and `Python 3.6`. + +See [https://hub.docker.com/r/ufoym/deepo/tags/](https://hub.docker.com/r/ufoym/deepo/tags/) for a complete list of all available tags. These pre-built images are all built from `docker/Dockerfile.*` and `circle.yml`. See [How to generate `docker/Dockerfile.*` and `circle.yml`](https://github.com/ufoym/deepo/tree/master/scripts) if you are interested in how these files are generated. + + + +## Build your own customized image with Lego-like modules + +#### Step 1. prepare generator + +```bash +git clone https://github.com/ufoym/deepo.git +cd deepo/generator +pip install -r requirements.txt +``` + +#### Step 2. generate your customized Dockerfile + +For example, if you like `pytorch` and `lasagne`, then +```bash +python generate.py Dockerfile pytorch lasagne +``` + +This should generate a Dockerfile that contains everything for building `pytorch` and `lasagne`. Note that the generator can handle automatic dependency processing and topologically sort the lists. So you don't need to worry about missing dependencies and the list order. + +You can also specify the version of Python: +```bash +python generate.py Dockerfile pytorch lasagne python==3.6 +``` + +#### Step 3. build your Dockerfile + +```bash +docker build -t my/deepo . +``` + +This may take several minutes as it compiles a few libraries from scratch. + + -## Comparison to alternatives +# Comparison to alternatives . | modern-deep-learning | dl-docker | jupyter-deeplearning | Deepo :------------------------------------------------: | :------------------: | :----------------: | :------------------: | :----------------: [ubuntu](https://www.ubuntu.com) | 16.04 | 14.04 | 14.04 | 16.04 @@ -193,8 +194,15 @@ caffe version 1.0.0 [caffe](http://caffe.berkeleyvision.org) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: [torch](http://torch.ch/) | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: + + + +# Contributing + +We appreciate all contributions. If you are planning to contribute back bug-fixes, please do so without any further discussion. If you plan to contribute new features, utility functions or extensions, please first open an issue and discuss the feature with us. + -## Licensing +# Licensing Deepo is [MIT licensed](https://github.com/ufoym/deepo/blob/master/LICENSE). diff --git a/circle.yml b/circle.yml index cd4e26b..55580a5 100644 --- a/circle.yml +++ b/circle.yml @@ -1,13 +1,228 @@ -version: 2 +version: 2.0 jobs: - build: - machine: true - steps: - - checkout - - run: docker build -t $DOCKER_REPO . - - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker push $DOCKER_REPO - - + + "all-py27_py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:all-py27 -t $DOCKER_REPO:py27 -f docker/Dockerfile.all-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:all-py27 + - run: docker push $DOCKER_REPO:py27 + + "all-py36_all_py36_latest": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:all-py36 -t $DOCKER_REPO:all -t $DOCKER_REPO:py36 -t $DOCKER_REPO:latest -f docker/Dockerfile.all-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:all-py36 + - run: docker push $DOCKER_REPO:all + - run: docker push $DOCKER_REPO:py36 + - run: docker push $DOCKER_REPO:latest + + "caffe-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:caffe-py27 -f docker/Dockerfile.caffe-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:caffe-py27 + + "caffe-py36_caffe": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:caffe-py36 -t $DOCKER_REPO:caffe -f docker/Dockerfile.caffe-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:caffe-py36 + - run: docker push $DOCKER_REPO:caffe + + "chainer-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:chainer-py27 -f docker/Dockerfile.chainer-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:chainer-py27 + + "chainer-py36_chainer": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:chainer-py36 -t $DOCKER_REPO:chainer -f docker/Dockerfile.chainer-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:chainer-py36 + - run: docker push $DOCKER_REPO:chainer + + "cntk-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:cntk-py27 -f docker/Dockerfile.cntk-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:cntk-py27 + + "cntk-py36_cntk": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:cntk-py36 -t $DOCKER_REPO:cntk -f docker/Dockerfile.cntk-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:cntk-py36 + - run: docker push $DOCKER_REPO:cntk + + "keras-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:keras-py27 -f docker/Dockerfile.keras-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:keras-py27 + + "keras-py36_keras": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:keras-py36 -t $DOCKER_REPO:keras -f docker/Dockerfile.keras-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:keras-py36 + - run: docker push $DOCKER_REPO:keras + + "lasagne-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:lasagne-py27 -f docker/Dockerfile.lasagne-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:lasagne-py27 + + "lasagne-py36_lasagne": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:lasagne-py36 -t $DOCKER_REPO:lasagne -f docker/Dockerfile.lasagne-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:lasagne-py36 + - run: docker push $DOCKER_REPO:lasagne + + "mxnet-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:mxnet-py27 -f docker/Dockerfile.mxnet-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:mxnet-py27 + + "mxnet-py36_mxnet": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:mxnet-py36 -t $DOCKER_REPO:mxnet -f docker/Dockerfile.mxnet-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:mxnet-py36 + - run: docker push $DOCKER_REPO:mxnet + + "pytorch-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:pytorch-py27 -f docker/Dockerfile.pytorch-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:pytorch-py27 + + "pytorch-py36_pytorch": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:pytorch-py36 -t $DOCKER_REPO:pytorch -f docker/Dockerfile.pytorch-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:pytorch-py36 + - run: docker push $DOCKER_REPO:pytorch + + "sonnet-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:sonnet-py27 -f docker/Dockerfile.sonnet-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:sonnet-py27 + + "sonnet-py36_sonnet": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:sonnet-py36 -t $DOCKER_REPO:sonnet -f docker/Dockerfile.sonnet-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:sonnet-py36 + - run: docker push $DOCKER_REPO:sonnet + + "tensorflow-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:tensorflow-py27 -f docker/Dockerfile.tensorflow-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:tensorflow-py27 + + "tensorflow-py36_tensorflow": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:tensorflow-py36 -t $DOCKER_REPO:tensorflow -f docker/Dockerfile.tensorflow-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:tensorflow-py36 + - run: docker push $DOCKER_REPO:tensorflow + + "theano-py27": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:theano-py27 -f docker/Dockerfile.theano-py27 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:theano-py27 + + "theano-py36_theano": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:theano-py36 -t $DOCKER_REPO:theano -f docker/Dockerfile.theano-py36 . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:theano-py36 + - run: docker push $DOCKER_REPO:theano + + "torch": + machine: true + steps: + - checkout + - run: docker build -t $DOCKER_REPO:torch -f docker/Dockerfile.torch . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push $DOCKER_REPO:torch +workflows: + version: 2 + build: + jobs: + - "all-py27_py27" + - "all-py36_all_py36_latest" + - "caffe-py27" + - "caffe-py36_caffe" + - "chainer-py27" + - "chainer-py36_chainer" + - "cntk-py27" + - "cntk-py36_cntk" + - "keras-py27" + - "keras-py36_keras" + - "lasagne-py27" + - "lasagne-py36_lasagne" + - "mxnet-py27" + - "mxnet-py36_mxnet" + - "pytorch-py27" + - "pytorch-py36_pytorch" + - "sonnet-py27" + - "sonnet-py36_sonnet" + - "tensorflow-py27" + - "tensorflow-py36_tensorflow" + - "theano-py27" + - "theano-py36_theano" + - "torch" \ No newline at end of file diff --git a/Dockerfile b/docker/Dockerfile.all-py27 similarity index 54% rename from Dockerfile rename to docker/Dockerfile.all-py27 index afbb9a7..899d7aa 100644 --- a/Dockerfile +++ b/docker/Dockerfile.all-py27 @@ -1,77 +1,58 @@ -FROM nvidia/cuda:8.0-cudnn6-devel -LABEL maintainer="Ming " - -# ================================= -# cuda 8.0 -# cudnn v6 -# --------------------------------- -# python 3.5 -# opencv latest (git) -# --------------------------------- -# tensorflow latest (pip) -# sonnet latest (pip) -# pytorch 0.2.0 (pip) -# keras latest (pip) -# mxnet latest (pip) -# cntk 2.2 (pip) +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# torch latest (git) # chainer latest (pip) +# cntk 2.3 (pip) +# mxnet latest (pip) +# pytorch 0.2.0 (pip) +# tensorflow latest (pip) # theano latest (git) +# keras latest (pip) # lasagne latest (git) +# opencv latest (git) +# sonnet latest (pip) # caffe latest (git) -# torch latest (git) -# --------------------------------- +# ================================================================== +FROM nvidia/cuda:8.0-cudnn6-devel RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ GIT_CLONE="git clone --depth 10" && \ -# ================================= -# apt -# ================================= - rm -rf /var/lib/apt/lists/* \ /etc/apt/sources.list.d/cuda.list \ /etc/apt/sources.list.d/nvidia-ml.list && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - apt-utils \ - && \ -# ================================= -# common tools -# ================================= +# ================================================================== +# tools +# ------------------------------------------------------------------ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ build-essential \ ca-certificates \ + cmake \ wget \ git \ vim \ && \ -# ================================= -# cmake -# ================================= - - # fix boost-not-found issue caused by the `apt-get` version of cmake - $GIT_CLONE https://github.com/Kitware/CMake ~/cmake && \ - cd ~/cmake && \ - ./bootstrap --prefix=/usr/local && \ - make -j"$(nproc)" install && \ - -# ================================= -# python3 -# ================================= +# ================================================================== +# python +# ------------------------------------------------------------------ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - python3-pip \ - python3-dev \ + python-pip \ + python-dev \ && \ - ln -s /usr/bin/python3 /usr/local/bin/python && \ - pip3 --no-cache-dir install --upgrade pip && \ $PIP_INSTALL \ setuptools \ + pip \ + && \ + $PIP_INSTALL \ numpy \ scipy \ pandas \ @@ -80,107 +61,104 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ Cython \ && \ -# ================================= -# opencv -# ================================= +# ================================================================== +# torch +# ------------------------------------------------------------------ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - libatlas-base-dev \ - libboost-all-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libhdf5-serial-dev \ - libleveldb-dev \ - liblmdb-dev \ - libprotobuf-dev \ - libsnappy-dev \ - protobuf-compiler \ - && \ + $GIT_CLONE https://github.com/torch/distro.git ~/torch --recursive && \ - $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ - mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cd ~/torch/exe/luajit-rocks && \ + mkdir build && cd build && \ cmake -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D WITH_IPP=OFF \ - -D WITH_CUDA=OFF \ - -D WITH_OPENCL=OFF \ - -D BUILD_TESTS=OFF \ - -D BUILD_PERF_TESTS=OFF \ + -D WITH_LUAJIT21=ON \ .. && \ make -j"$(nproc)" install && \ -# ================================= -# tensorflow -# ================================= - - $PIP_INSTALL \ - tensorflow_gpu \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libreadline-dev \ && \ -# ================================= -# sonnet -# ================================= + $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ + cd ~/moses && \ + luarocks install rockspec/moses-1.6.1-1.rockspec && \ - $PIP_INSTALL \ - dm-sonnet \ - && \ + cd ~/torch && \ + sed -i 's/extra\/cudnn/extra\/cudnn \&\& git checkout R6/' install.sh && \ + sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ + sed -i '/qt/d' install.sh && \ + sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ + sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ + sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ + sed -i '/You might want to/,/^fi$/d' install.sh && \ + yes no | ./install.sh && \ -# ================================= -# mxnet -# ================================= +# ================================================================== +# boost +# ------------------------------------------------------------------ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - graphviz \ + libboost-all-dev \ && \ +# ================================================================== +# chainer +# ------------------------------------------------------------------ + $PIP_INSTALL \ - mxnet-cu80 \ - graphviz \ + cupy \ + chainer \ && \ -# ================================= +# ================================================================== # cntk -# ================================= +# ------------------------------------------------------------------ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ openmpi-bin \ + libpng-dev \ + libtiff-dev \ libjasper-dev \ && \ $PIP_INSTALL \ - https://cntk.ai/PythonWheel/GPU/cntk-2.2-cp35-cp35m-linux_x86_64.whl \ + https://cntk.ai/PythonWheel/GPU/cntk-2.3-cp27-cp27mu-linux_x86_64.whl \ && \ -# ================================= -# keras -# ================================= +# ================================================================== +# mxnet +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + graphviz \ + && \ $PIP_INSTALL \ - h5py \ - keras \ + mxnet-cu80 \ + graphviz \ && \ -# ================================= +# ================================================================== # pytorch -# ================================= +# ------------------------------------------------------------------ $PIP_INSTALL \ - http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp35-cp35m-manylinux1_x86_64.whl \ + http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp27-cp27mu-manylinux1_x86_64.whl \ torchvision \ && \ -# ================================= -# chainer -# ================================= +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ $PIP_INSTALL \ - cupy \ - chainer \ + tensorflow_gpu \ && \ -# ================================= +# ================================================================== # theano -# ================================= +# ------------------------------------------------------------------ $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ cd ~/theano && \ @@ -197,84 +175,99 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ python setup.py build && \ python setup.py install && \ - printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' \ - > ~/.theanorc && \ + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# keras +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + h5py \ + keras \ + && \ -# ================================= +# ================================================================== # lasagne -# ================================= +# ------------------------------------------------------------------ $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ cd ~/lasagne && \ $PIP_INSTALL \ . && \ -# ================================= -# caffe -# ================================= +# ================================================================== +# opencv +# ------------------------------------------------------------------ - $GIT_CLONE https://github.com/NVIDIA/nccl && \ - cd nccl; make -j"$(nproc)" install; cd ..; rm -rf nccl && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ - $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ - mkdir ~/caffe/build && cd ~/caffe/build && \ + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ cmake -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D USE_CUDNN=1 \ - -D USE_NCCL=1 \ - -D python_version=3 \ - -D CUDA_NVCC_FLAGS=--Wno-deprecated-gpu-targets \ - -Wno-dev \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ .. && \ make -j"$(nproc)" install && \ - # fix ValueError caused by python-dateutil 1.x - sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ +# ================================================================== +# sonnet +# ------------------------------------------------------------------ $PIP_INSTALL \ - -r ~/caffe/python/requirements.txt && \ - - mv /usr/local/python/caffe /usr/local/lib/python3.5/dist-packages/ && \ - rm -rf /usr/local/python && \ - -# ================================= -# torch -# ================================= + dm-sonnet \ + && \ - $GIT_CLONE https://github.com/torch/distro.git ~/torch --recursive&& \ +# ================================================================== +# caffe +# ------------------------------------------------------------------ - cd ~/torch/exe/luajit-rocks && \ - mkdir build && cd build && \ - cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D WITH_LUAJIT21=ON \ - .. && \ + $GIT_CLONE https://github.com/NVIDIA/nccl ~/nccl && \ + cd ~/nccl && \ make -j"$(nproc)" install && \ - DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ - libreadline-dev \ - && \ + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + cp ~/caffe/Makefile.config.example ~/caffe/Makefile.config && \ + sed -i 's/# USE_CUDNN/USE_CUDNN/g' ~/caffe/Makefile.config && \ + sed -i 's/# WITH_PYTHON_LAYER/WITH_PYTHON_LAYER/g' ~/caffe/Makefile.config && \ + sed -i 's/# OPENCV_VERSION/OPENCV_VERSION/g' ~/caffe/Makefile.config && \ + sed -i 's/# USE_NCCL/USE_NCCL/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/lib\/python/\/usr\/local\/lib\/python/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/local\/include/\/usr\/local\/include \/usr\/include\/hdf5\/serial/g' ~/caffe/Makefile.config && \ + sed -i 's/hdf5/hdf5_serial/g' ~/caffe/Makefile && \ + cd ~/caffe && \ + make -j"$(nproc)" -Wno-deprecated-gpu-targets distribute && \ - $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ - cd ~/moses && \ - luarocks install rockspec/moses-1.6.1-1.rockspec && \ + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ - cd ~/torch && \ - sed -i 's/extra\/cudnn/extra\/cudnn \&\& git checkout R6/' install.sh && \ - sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ - sed -i '/qt/d' install.sh && \ - sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ - sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ - sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ - sed -i '/You might want to/,/^fi$/d' install.sh && \ - yes no | ./install.sh && \ + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + cd ~/caffe/distribute/bin && \ + for file in *.bin; do mv "$file" "${file%%.bin}"; done && \ + cd ~/caffe/distribute && \ + cp -r bin include lib proto /usr/local/ && \ + cp -r python/caffe /usr/local/lib/python2.7/dist-packages/ && \ -# ================================= +# ================================================================== # config & cleanup -# ================================= +# ------------------------------------------------------------------ ldconfig && \ apt-get clean && \ apt-get autoremove && \ - rm -rf /var/lib/apt/lists/* /tmp/* ~/* + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.all-py36 b/docker/Dockerfile.all-py36 new file mode 100644 index 0000000..e713586 --- /dev/null +++ b/docker/Dockerfile.all-py36 @@ -0,0 +1,287 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# torch latest (git) +# chainer latest (pip) +# cntk 2.3 (pip) +# mxnet latest (pip) +# pytorch 0.2.0 (pip) +# tensorflow latest (pip) +# theano latest (git) +# keras latest (pip) +# lasagne latest (git) +# opencv latest (git) +# sonnet latest (pip) +# caffe latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# torch +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/torch/distro.git ~/torch --recursive && \ + + cd ~/torch/exe/luajit-rocks && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_LUAJIT21=ON \ + .. && \ + make -j"$(nproc)" install && \ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libreadline-dev \ + && \ + + $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ + cd ~/moses && \ + luarocks install rockspec/moses-1.6.1-1.rockspec && \ + + cd ~/torch && \ + sed -i 's/extra\/cudnn/extra\/cudnn \&\& git checkout R6/' install.sh && \ + sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ + sed -i '/qt/d' install.sh && \ + sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ + sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ + sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ + sed -i '/You might want to/,/^fi$/d' install.sh && \ + yes no | ./install.sh && \ + +# ================================================================== +# boost +# ------------------------------------------------------------------ + + wget -O ~/boost.tar.gz https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz && \ + tar -zxf ~/boost.tar.gz -C ~ && \ + cd ~/boost_* && \ + ./bootstrap.sh --with-python=python3.6 && \ + ./b2 install --prefix=/usr/local && \ + +# ================================================================== +# chainer +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + cupy \ + chainer \ + && \ + +# ================================================================== +# cntk +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + openmpi-bin \ + libpng-dev \ + libtiff-dev \ + libjasper-dev \ + && \ + + $PIP_INSTALL \ + https://cntk.ai/PythonWheel/GPU/cntk-2.3-cp36-cp36m-linux_x86_64.whl \ + && \ + +# ================================================================== +# mxnet +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + graphviz \ + && \ + + $PIP_INSTALL \ + mxnet-cu80 \ + graphviz \ + && \ + +# ================================================================== +# pytorch +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp36-cp36m-manylinux1_x86_64.whl \ + torchvision \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# theano +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# keras +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + h5py \ + keras \ + && \ + +# ================================================================== +# lasagne +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ + cd ~/lasagne && \ + $PIP_INSTALL \ + . && \ + +# ================================================================== +# opencv +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ + + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j"$(nproc)" install && \ + +# ================================================================== +# sonnet +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + dm-sonnet \ + && \ + +# ================================================================== +# caffe +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/NVIDIA/nccl ~/nccl && \ + cd ~/nccl && \ + make -j"$(nproc)" install && \ + + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + cp ~/caffe/Makefile.config.example ~/caffe/Makefile.config && \ + sed -i 's/# USE_CUDNN/USE_CUDNN/g' ~/caffe/Makefile.config && \ + sed -i 's/# PYTHON_LIBRARIES/PYTHON_LIBRARIES/g' ~/caffe/Makefile.config && \ + sed -i 's/# WITH_PYTHON_LAYER/WITH_PYTHON_LAYER/g' ~/caffe/Makefile.config && \ + sed -i 's/# OPENCV_VERSION/OPENCV_VERSION/g' ~/caffe/Makefile.config && \ + sed -i 's/# USE_NCCL/USE_NCCL/g' ~/caffe/Makefile.config && \ + sed -i 's/2\.7/3\.6/g' ~/caffe/Makefile.config && \ + sed -i 's/3\.5/3\.6/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/lib\/python/\/usr\/local\/lib\/python/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/local\/include/\/usr\/local\/include \/usr\/include\/hdf5\/serial/g' ~/caffe/Makefile.config && \ + sed -i 's/hdf5/hdf5_serial/g' ~/caffe/Makefile && \ + cd ~/caffe && \ + make -j"$(nproc)" -Wno-deprecated-gpu-targets distribute && \ + + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ + + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + + cd ~/caffe/distribute/bin && \ + for file in *.bin; do mv "$file" "${file%%.bin}"; done && \ + cd ~/caffe/distribute && \ + cp -r bin include lib proto /usr/local/ && \ + cp -r python/caffe /usr/local/lib/python3.6/dist-packages/ && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.caffe-py27 b/docker/Dockerfile.caffe-py27 new file mode 100644 index 0000000..795d4e6 --- /dev/null +++ b/docker/Dockerfile.caffe-py27 @@ -0,0 +1,129 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# opencv latest (git) +# caffe latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# boost +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libboost-all-dev \ + && \ + +# ================================================================== +# opencv +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ + + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j"$(nproc)" install && \ + +# ================================================================== +# caffe +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/NVIDIA/nccl ~/nccl && \ + cd ~/nccl && \ + make -j"$(nproc)" install && \ + + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + cp ~/caffe/Makefile.config.example ~/caffe/Makefile.config && \ + sed -i 's/# USE_CUDNN/USE_CUDNN/g' ~/caffe/Makefile.config && \ + sed -i 's/# WITH_PYTHON_LAYER/WITH_PYTHON_LAYER/g' ~/caffe/Makefile.config && \ + sed -i 's/# OPENCV_VERSION/OPENCV_VERSION/g' ~/caffe/Makefile.config && \ + sed -i 's/# USE_NCCL/USE_NCCL/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/lib\/python/\/usr\/local\/lib\/python/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/local\/include/\/usr\/local\/include \/usr\/include\/hdf5\/serial/g' ~/caffe/Makefile.config && \ + sed -i 's/hdf5/hdf5_serial/g' ~/caffe/Makefile && \ + cd ~/caffe && \ + make -j"$(nproc)" -Wno-deprecated-gpu-targets distribute && \ + + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ + + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + + cd ~/caffe/distribute/bin && \ + for file in *.bin; do mv "$file" "${file%%.bin}"; done && \ + cd ~/caffe/distribute && \ + cp -r bin include lib proto /usr/local/ && \ + cp -r python/caffe /usr/local/lib/python2.7/dist-packages/ && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.caffe-py36 b/docker/Dockerfile.caffe-py36 new file mode 100644 index 0000000..d0ea070 --- /dev/null +++ b/docker/Dockerfile.caffe-py36 @@ -0,0 +1,143 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# opencv latest (git) +# caffe latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# boost +# ------------------------------------------------------------------ + + wget -O ~/boost.tar.gz https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz && \ + tar -zxf ~/boost.tar.gz -C ~ && \ + cd ~/boost_* && \ + ./bootstrap.sh --with-python=python3.6 && \ + ./b2 install --prefix=/usr/local && \ + +# ================================================================== +# opencv +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ + + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j"$(nproc)" install && \ + +# ================================================================== +# caffe +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/NVIDIA/nccl ~/nccl && \ + cd ~/nccl && \ + make -j"$(nproc)" install && \ + + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + cp ~/caffe/Makefile.config.example ~/caffe/Makefile.config && \ + sed -i 's/# USE_CUDNN/USE_CUDNN/g' ~/caffe/Makefile.config && \ + sed -i 's/# PYTHON_LIBRARIES/PYTHON_LIBRARIES/g' ~/caffe/Makefile.config && \ + sed -i 's/# WITH_PYTHON_LAYER/WITH_PYTHON_LAYER/g' ~/caffe/Makefile.config && \ + sed -i 's/# OPENCV_VERSION/OPENCV_VERSION/g' ~/caffe/Makefile.config && \ + sed -i 's/# USE_NCCL/USE_NCCL/g' ~/caffe/Makefile.config && \ + sed -i 's/2\.7/3\.6/g' ~/caffe/Makefile.config && \ + sed -i 's/3\.5/3\.6/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/lib\/python/\/usr\/local\/lib\/python/g' ~/caffe/Makefile.config && \ + sed -i 's/\/usr\/local\/include/\/usr\/local\/include \/usr\/include\/hdf5\/serial/g' ~/caffe/Makefile.config && \ + sed -i 's/hdf5/hdf5_serial/g' ~/caffe/Makefile && \ + cd ~/caffe && \ + make -j"$(nproc)" -Wno-deprecated-gpu-targets distribute && \ + + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ + + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + + cd ~/caffe/distribute/bin && \ + for file in *.bin; do mv "$file" "${file%%.bin}"; done && \ + cd ~/caffe/distribute && \ + cp -r bin include lib proto /usr/local/ && \ + cp -r python/caffe /usr/local/lib/python3.6/dist-packages/ && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.chainer-py27 b/docker/Dockerfile.chainer-py27 new file mode 100644 index 0000000..2c3bb65 --- /dev/null +++ b/docker/Dockerfile.chainer-py27 @@ -0,0 +1,69 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# chainer latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# chainer +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + cupy \ + chainer \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.chainer-py36 b/docker/Dockerfile.chainer-py36 new file mode 100644 index 0000000..f965135 --- /dev/null +++ b/docker/Dockerfile.chainer-py36 @@ -0,0 +1,78 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# chainer latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# chainer +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + cupy \ + chainer \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.cntk-py27 b/docker/Dockerfile.cntk-py27 new file mode 100644 index 0000000..0e75d8c --- /dev/null +++ b/docker/Dockerfile.cntk-py27 @@ -0,0 +1,75 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# cntk 2.3 (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# cntk +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + openmpi-bin \ + libpng-dev \ + libtiff-dev \ + libjasper-dev \ + && \ + + $PIP_INSTALL \ + https://cntk.ai/PythonWheel/GPU/cntk-2.3-cp27-cp27mu-linux_x86_64.whl \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.cntk-py36 b/docker/Dockerfile.cntk-py36 new file mode 100644 index 0000000..28a20b3 --- /dev/null +++ b/docker/Dockerfile.cntk-py36 @@ -0,0 +1,84 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# cntk 2.3 (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# cntk +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + openmpi-bin \ + libpng-dev \ + libtiff-dev \ + libjasper-dev \ + && \ + + $PIP_INSTALL \ + https://cntk.ai/PythonWheel/GPU/cntk-2.3-cp36-cp36m-linux_x86_64.whl \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.keras-py27 b/docker/Dockerfile.keras-py27 new file mode 100644 index 0000000..aaf717f --- /dev/null +++ b/docker/Dockerfile.keras-py27 @@ -0,0 +1,78 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# tensorflow latest (pip) +# keras latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# keras +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + h5py \ + keras \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.keras-py36 b/docker/Dockerfile.keras-py36 new file mode 100644 index 0000000..3b08c0c --- /dev/null +++ b/docker/Dockerfile.keras-py36 @@ -0,0 +1,87 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# tensorflow latest (pip) +# keras latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# keras +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + h5py \ + keras \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.lasagne-py27 b/docker/Dockerfile.lasagne-py27 new file mode 100644 index 0000000..9a4176b --- /dev/null +++ b/docker/Dockerfile.lasagne-py27 @@ -0,0 +1,91 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# theano latest (git) +# lasagne latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# theano +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# lasagne +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ + cd ~/lasagne && \ + $PIP_INSTALL \ + . && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.lasagne-py36 b/docker/Dockerfile.lasagne-py36 new file mode 100644 index 0000000..ae6953a --- /dev/null +++ b/docker/Dockerfile.lasagne-py36 @@ -0,0 +1,100 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# theano latest (git) +# lasagne latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# theano +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# lasagne +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ + cd ~/lasagne && \ + $PIP_INSTALL \ + . && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.mxnet-py27 b/docker/Dockerfile.mxnet-py27 new file mode 100644 index 0000000..c12fb4d --- /dev/null +++ b/docker/Dockerfile.mxnet-py27 @@ -0,0 +1,74 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# mxnet latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# mxnet +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + graphviz \ + && \ + + $PIP_INSTALL \ + mxnet-cu80 \ + graphviz \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.mxnet-py36 b/docker/Dockerfile.mxnet-py36 new file mode 100644 index 0000000..de6b9fe --- /dev/null +++ b/docker/Dockerfile.mxnet-py36 @@ -0,0 +1,83 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# mxnet latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# mxnet +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + graphviz \ + && \ + + $PIP_INSTALL \ + mxnet-cu80 \ + graphviz \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.pytorch-py27 b/docker/Dockerfile.pytorch-py27 new file mode 100644 index 0000000..632c374 --- /dev/null +++ b/docker/Dockerfile.pytorch-py27 @@ -0,0 +1,69 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# pytorch 0.2.0 (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# pytorch +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp27-cp27mu-manylinux1_x86_64.whl \ + torchvision \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.pytorch-py36 b/docker/Dockerfile.pytorch-py36 new file mode 100644 index 0000000..4840b23 --- /dev/null +++ b/docker/Dockerfile.pytorch-py36 @@ -0,0 +1,78 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# pytorch 0.2.0 (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# pytorch +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + http://download.pytorch.org/whl/cu80/torch-0.2.0.post3-cp36-cp36m-manylinux1_x86_64.whl \ + torchvision \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.sonnet-py27 b/docker/Dockerfile.sonnet-py27 new file mode 100644 index 0000000..f0ba478 --- /dev/null +++ b/docker/Dockerfile.sonnet-py27 @@ -0,0 +1,77 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# tensorflow latest (pip) +# sonnet latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# sonnet +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + dm-sonnet \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.sonnet-py36 b/docker/Dockerfile.sonnet-py36 new file mode 100644 index 0000000..3ef09a6 --- /dev/null +++ b/docker/Dockerfile.sonnet-py36 @@ -0,0 +1,86 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# tensorflow latest (pip) +# sonnet latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# sonnet +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + dm-sonnet \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.tensorflow-py27 b/docker/Dockerfile.tensorflow-py27 new file mode 100644 index 0000000..2dee4ca --- /dev/null +++ b/docker/Dockerfile.tensorflow-py27 @@ -0,0 +1,68 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# tensorflow latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.tensorflow-py36 b/docker/Dockerfile.tensorflow-py36 new file mode 100644 index 0000000..a078a5d --- /dev/null +++ b/docker/Dockerfile.tensorflow-py36 @@ -0,0 +1,77 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# tensorflow latest (pip) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# tensorflow +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.theano-py27 b/docker/Dockerfile.theano-py27 new file mode 100644 index 0000000..6370afd --- /dev/null +++ b/docker/Dockerfile.theano-py27 @@ -0,0 +1,81 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 2.7 (apt) +# theano latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# theano +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.theano-py36 b/docker/Dockerfile.theano-py36 new file mode 100644 index 0000000..cf7b134 --- /dev/null +++ b/docker/Dockerfile.theano-py36 @@ -0,0 +1,90 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# python 3.6 (apt) +# theano latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# python +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + +# ================================================================== +# theano +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\ninclude_path = /usr/local/cuda/targets/x86_64-linux/include\n' > ~/.theanorc && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/docker/Dockerfile.torch b/docker/Dockerfile.torch new file mode 100644 index 0000000..5e2540a --- /dev/null +++ b/docker/Dockerfile.torch @@ -0,0 +1,70 @@ +# ================================================================== +# module list +# ------------------------------------------------------------------ +# torch latest (git) +# ================================================================== + +FROM nvidia/cuda:8.0-cudnn6-devel +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + +# ================================================================== +# torch +# ------------------------------------------------------------------ + + $GIT_CLONE https://github.com/torch/distro.git ~/torch --recursive && \ + + cd ~/torch/exe/luajit-rocks && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_LUAJIT21=ON \ + .. && \ + make -j"$(nproc)" install && \ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libreadline-dev \ + && \ + + $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ + cd ~/moses && \ + luarocks install rockspec/moses-1.6.1-1.rockspec && \ + + cd ~/torch && \ + sed -i 's/extra\/cudnn/extra\/cudnn \&\& git checkout R6/' install.sh && \ + sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ + sed -i '/qt/d' install.sh && \ + sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ + sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ + sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ + sed -i '/You might want to/,/^fi$/d' install.sh && \ + yes no | ./install.sh && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* diff --git a/generator/__init__.py b/generator/__init__.py new file mode 100644 index 0000000..eccfe36 --- /dev/null +++ b/generator/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +"""Top-level package for genv.""" + +__author__ = """Ming""" +__email__ = 'i@ufoym.com' +__version__ = '0.1.1' + +from .core.manager import Manager + +from .modules.tools import Tools +from .modules.python import Python +from .modules.boost import Boost +from .modules.opencv import Opencv + +from .modules.tensorflow import Tensorflow +from .modules.sonnet import Sonnet +from .modules.mxnet import Mxnet +from .modules.cntk import Cntk +from .modules.keras import Keras +from .modules.pytorch import Pytorch +from .modules.chainer import Chainer +from .modules.theano import Theano +from .modules.lasagne import Lasagne +from .modules.caffe import Caffe +from .modules.torch import Torch + +__all__ = [ + 'Manager', 'Tools', 'Python', 'Boost', 'Opencv', + 'Tensorflow', 'Sonnet', 'Mxnet', 'Cntk', 'Keras', + 'Pytorch', 'Chainer', 'Theano', 'Lasagne', 'Caffe', + 'Torch', +] diff --git a/generator/core/__init__.py b/generator/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/generator/core/composer.py b/generator/core/composer.py new file mode 100644 index 0000000..682252d --- /dev/null +++ b/generator/core/composer.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +import textwrap +import functools + + +class Composer(object): + + def __init__(self, modules, versions={}): + if len(modules) == 0: + raise ValueError('Modules should contain at least one module') + pending = self._traverse(modules) + self.modules = [m for m in self._toposort(pending)] + self.instances = self._get_instances(versions) + + def get(self): + return self.modules + + def ver(self, module): + for ins in self.instances: + if ins.__class__ is module: + return ins.version + return None + + def to_dockerfile(self): + + def _indent(n, s): + prefix = ' ' * 4 * n + return ''.join(prefix + l for l in s.splitlines(True)) + + return textwrap.dedent(''.join([ + _indent(3, ''.join([ + self._split('module list'), + ''.join('# %s\n' % repr(m) + for m in self.instances if repr(m)), + self._split(), + ])), + r''' + FROM nvidia/cuda:8.0-cudnn6-devel + RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --depth 10" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + ''', + '\n', + '\n'.join([ + ''.join([ + _indent(3, self._split(m.name())), + _indent(1, m.build()), + ]) for m in self.instances + ]), + '\n', + _indent(3, self._split('config & cleanup')), + r''' + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* ~/* + ''' + ])) + + def _traverse(self, modules): + seen = set(modules) + current_level = modules + while current_level: + next_level = [] + for module in current_level: + yield module + for child in (dep for dep in module.deps if dep not in seen): + next_level.append(child) + seen.add(child) + current_level = next_level + + def _toposort(self, pending): + data = {m: set(m.deps) for m in pending} + for k, v in data.items(): + v.discard(k) + extra_items_in_deps = functools.reduce( + set.union, data.values()) - set(data.keys()) + data.update({item: set() for item in extra_items_in_deps}) + while True: + ordered = set(item for item, dep in data.items() if len(dep) == 0) + if not ordered: + break + for m in sorted(ordered, key=lambda m: m.__name__): + yield m + data = { + item: (dep - ordered) + for item, dep in data.items() + if item not in ordered + } + if len(data) != 0: + raise ValueError( + 'Circular dependencies exist among these items: ' + '{{{}}}'.format(', '.join( + '{!r}:{!r}'.format( + key, value) for key, value in sorted( + data.items())))) + + def _split(self, title=None): + split_l = '# ' + '=' * 66 + '\n' + split_s = '# ' + '-' * 66 + '\n' + s = split_l if title is None else ( + split_l + '# %s\n' % title + split_s) + return s + + def _get_instances(self, versions): + inses = [] + for m in self.modules: + ins = m(self) + if m in versions: + ins.version = versions[m] + inses.append(ins) + return inses diff --git a/generator/generate.py b/generator/generate.py new file mode 100644 index 0000000..f615437 --- /dev/null +++ b/generator/generate.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +"""Console script for generator.""" + +import os +import sys +import click + +from core.composer import Composer + +from modules.tools import Tools +from modules.python import Python +from modules.boost import Boost +from modules.opencv import Opencv + +from modules.tensorflow import Tensorflow +from modules.sonnet import Sonnet +from modules.mxnet import Mxnet +from modules.cntk import Cntk +from modules.keras import Keras +from modules.pytorch import Pytorch +from modules.chainer import Chainer +from modules.theano import Theano +from modules.lasagne import Lasagne +from modules.caffe import Caffe +from modules.torch import Torch + + +@click.command() +@click.argument('path', nargs=1) +@click.argument('modules', nargs=-1) +def main(path, modules): + """ + Generate a dockerfile according to the given modules to be installed. + """ + in_modules = [] + versions = {} + for module in modules: + terms = module.split('==') + m = getattr(sys.modules[__name__], terms[0].title()) + in_modules.append(m) + if len(terms) > 1: + versions[m] = terms[1] + composer = Composer(in_modules, versions) + with open(path, 'w') as f: + f.write(composer.to_dockerfile()) + + +if __name__ == "__main__": + main() diff --git a/generator/modules/__init__.py b/generator/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/generator/modules/__module__.py b/generator/modules/__module__.py new file mode 100644 index 0000000..5056772 --- /dev/null +++ b/generator/modules/__module__.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + + +def parametrized(dec): + def layer(*args, **kwargs): + def repl(f): + return dec(f, *args, **kwargs) + return repl + return layer + + +@parametrized +def dependency(module, *_deps): + module.deps = _deps + return module + + +@parametrized +def source(module, _source): + module.source = _source + return module + + +@parametrized +def version(module, _ver): + module.version = _ver + return module + + +@dependency() +@source('unknown') +@version('latest') +class Module(object): + + def __init__(self, manager): + self.manager = manager + + def __repr__(self): + return '%-13s %-6s (%s)' % ( + self.name(), + self.version, + self.source) + + def build(self): + pass + + def name(self): + return self.__class__.__name__.lower() diff --git a/generator/modules/boost.py b/generator/modules/boost.py new file mode 100644 index 0000000..07c35be --- /dev/null +++ b/generator/modules/boost.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools +from .python import Python + + +@dependency(Tools, Python) +@source('src') +class Boost(Module): + + def __repr__(self): + return '' + + def build(self): + pyver = self.manager.ver(Python) + return r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libboost-all-dev \ + && \ + ''' if pyver == '2.7' else ( + r''' + wget -O ~/boost.tar.gz ''' + + r'''https://dl.bintray.com/boostorg/release/1.65.1''' + + r'''/source/boost_1_65_1.tar.gz && \ + tar -zxf ~/boost.tar.gz -C ~ && \ + cd ~/boost_* && \ + ./bootstrap.sh --with-python=python%s && \ + ./b2 install --prefix=/usr/local && \ + ''' % pyver + ) diff --git a/generator/modules/caffe.py b/generator/modules/caffe.py new file mode 100644 index 0000000..df877d3 --- /dev/null +++ b/generator/modules/caffe.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools +from .boost import Boost +from .python import Python +from .opencv import Opencv + + +@dependency(Tools, Python, Boost, Opencv) +@source('git') +class Caffe(Module): + + def build(self): + pyver = self.manager.ver(Python) + return r''' + $GIT_CLONE https://github.com/NVIDIA/nccl ~/nccl && \ + cd ~/nccl && \ + make -j"$(nproc)" install && \ + + $GIT_CLONE https://github.com/BVLC/caffe ~/caffe && \ + cp ~/caffe/Makefile.config.example ~/caffe/Makefile.config && \ + sed -i 's/# USE_CUDNN/USE_CUDNN/g' ~/caffe/Makefile.config && \ + '''.rstrip() + ( + '' if pyver == '2.7' else r''' + sed -i 's/# PYTHON_LIBRARIES/PYTHON_LIBRARIES/g' ''' + + r'''~/caffe/Makefile.config && \ + '''.rstrip() + ) + r''' + sed -i 's/# WITH_PYTHON_LAYER/WITH_PYTHON_LAYER/g' ''' \ + + r'''~/caffe/Makefile.config && \ + sed -i 's/# OPENCV_VERSION/OPENCV_VERSION/g' ''' \ + + r'''~/caffe/Makefile.config && \ + sed -i 's/# USE_NCCL/USE_NCCL/g' ~/caffe/Makefile.config && \ + '''.rstrip() + (r''' + sed -i 's/2\.7/3\.5/g' ~/caffe/Makefile.config && \ + ''' if pyver == '3.5' else ( + r''' + sed -i 's/2\.7/3\.6/g' ~/caffe/Makefile.config && \ + sed -i 's/3\.5/3\.6/g' ~/caffe/Makefile.config && \ + ''' if pyver == '3.6' else + r''' + ''' + )).rstrip() + r''' + sed -i 's/\/usr\/lib\/python/\/usr\/local\/lib\/python/g' ''' \ + + r'''~/caffe/Makefile.config && \ + sed -i 's/\/usr\/local\/include/\/usr\/local\/include ''' \ + + r'''\/usr\/include\/hdf5\/serial/g' ~/caffe/Makefile.config && \ + sed -i 's/hdf5/hdf5_serial/g' ~/caffe/Makefile && \ + cd ~/caffe && \ + make -j"$(nproc)" -Wno-deprecated-gpu-targets distribute && \ + + # fix ValueError caused by python-dateutil 1.x + sed -i 's/,<2//g' ~/caffe/python/requirements.txt && \ + + $PIP_INSTALL \ + -r ~/caffe/python/requirements.txt && \ + + cd ~/caffe/distribute/bin && \ + for file in *.bin; do mv "$file" "${file%%%%.bin}"; done && \ + cd ~/caffe/distribute && \ + cp -r bin include lib proto /usr/local/ && \ + cp -r python/caffe /usr/local/lib/python%s/dist-packages/ && \ + ''' % pyver diff --git a/generator/modules/chainer.py b/generator/modules/chainer.py new file mode 100644 index 0000000..4c3b6f2 --- /dev/null +++ b/generator/modules/chainer.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .python import Python + + +@dependency(Python) +@source('pip') +class Chainer(Module): + + def build(self): + return r''' + $PIP_INSTALL \ + cupy \ + chainer \ + && \ + ''' diff --git a/generator/modules/cntk.py b/generator/modules/cntk.py new file mode 100644 index 0000000..56d5f7d --- /dev/null +++ b/generator/modules/cntk.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source, version +from .python import Python + + +@dependency(Python) +@source('pip') +@version('2.3') +class Cntk(Module): + + def build(self): + pyver = self.manager.ver(Python) + platform = 'cp35-cp35m' if pyver == '3.5' else ( + 'cp36-cp36m' if pyver == '3.6' else 'cp27-cp27mu') + return r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + openmpi-bin \ + libpng-dev \ + libtiff-dev \ + libjasper-dev \ + && \ + + $PIP_INSTALL \ + https://cntk.ai/PythonWheel/GPU/cntk-%s-%s-linux_x86_64.whl \ + && \ + ''' % (self.version, platform) diff --git a/generator/modules/keras.py b/generator/modules/keras.py new file mode 100644 index 0000000..2c4e53c --- /dev/null +++ b/generator/modules/keras.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .python import Python +from .tensorflow import Tensorflow + + +@dependency(Python, Tensorflow) +@source('pip') +class Keras(Module): + + def build(self): + return r''' + $PIP_INSTALL \ + h5py \ + keras \ + && \ + ''' diff --git a/generator/modules/lasagne.py b/generator/modules/lasagne.py new file mode 100644 index 0000000..4d3ee7e --- /dev/null +++ b/generator/modules/lasagne.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools +from .python import Python +from .theano import Theano + + +@dependency(Tools, Python, Theano) +@source('git') +class Lasagne(Module): + + def build(self): + return r''' + $GIT_CLONE https://github.com/Lasagne/Lasagne ~/lasagne && \ + cd ~/lasagne && \ + $PIP_INSTALL \ + . && \ + ''' diff --git a/generator/modules/mxnet.py b/generator/modules/mxnet.py new file mode 100644 index 0000000..aefedc3 --- /dev/null +++ b/generator/modules/mxnet.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .python import Python + + +@dependency(Python) +@source('pip') +class Mxnet(Module): + + def build(self): + return r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + graphviz \ + && \ + + $PIP_INSTALL \ + mxnet-cu80 \ + graphviz \ + && \ + ''' diff --git a/generator/modules/opencv.py b/generator/modules/opencv.py new file mode 100644 index 0000000..797577d --- /dev/null +++ b/generator/modules/opencv.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools +from .boost import Boost +from .python import Python + + +@dependency(Tools, Python, Boost) +@source('git') +class Opencv(Module): + + def build(self): + return r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libatlas-base-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + && \ + + $GIT_CLONE https://github.com/opencv/opencv ~/opencv && \ + mkdir -p ~/opencv/build && cd ~/opencv/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_IPP=OFF \ + -D WITH_CUDA=OFF \ + -D WITH_OPENCL=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + .. && \ + make -j"$(nproc)" install && \ + ''' diff --git a/generator/modules/python.py b/generator/modules/python.py new file mode 100644 index 0000000..c012512 --- /dev/null +++ b/generator/modules/python.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source, version +from .tools import Tools + + +@dependency(Tools) +@version('3.5') +@source('apt') +class Python(Module): + + def __init__(self, manager, **args): + super(self.__class__, self).__init__(manager, **args) + if self.version not in ('2.7', '3.5', '3.6',): + raise NotImplementedError('unsupported python version') + + def build(self): + return (r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3-pip \ + python3-dev \ + && \ + ln -s /usr/bin/python3 /usr/local/bin/python && \ + pip3 --no-cache-dir install --upgrade pip && \ + $PIP_INSTALL \ + setuptools \ + && \ + ''' if self.version == '3.5' else ( + r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python3.6 \ + python3.6-dev \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python3.6 ~/get-pip.py && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ + ln -s /usr/bin/python3.6 /usr/local/bin/python && \ + $PIP_INSTALL \ + setuptools \ + && \ + ''' if self.version == '3.6' else + r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python-pip \ + python-dev \ + && \ + $PIP_INSTALL \ + setuptools \ + pip \ + && \ + ''' + )).rstrip() + r''' + $PIP_INSTALL \ + numpy \ + scipy \ + pandas \ + scikit-learn \ + matplotlib \ + Cython \ + && \ + ''' diff --git a/generator/modules/pytorch.py b/generator/modules/pytorch.py new file mode 100644 index 0000000..ff12214 --- /dev/null +++ b/generator/modules/pytorch.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source, version +from .python import Python + + +@dependency(Python) +@source('pip') +@version('0.2.0') +class Pytorch(Module): + + def build(self): + pyver = self.manager.ver(Python) + platform = 'cp35-cp35m' if pyver == '3.5' else ( + 'cp36-cp36m' if pyver == '3.6' else 'cp27-cp27mu') + return r''' + $PIP_INSTALL \ + http://download.pytorch.org/whl/cu80/''' \ + + r'''torch-%s.post3-%s-manylinux1_x86_64.whl \ + torchvision \ + && \ + ''' % (self.version, platform) diff --git a/generator/modules/sonnet.py b/generator/modules/sonnet.py new file mode 100644 index 0000000..e01bdfc --- /dev/null +++ b/generator/modules/sonnet.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .python import Python +from .tensorflow import Tensorflow + + +@dependency(Python, Tensorflow) +@source('pip') +class Sonnet(Module): + + def build(self): + return r''' + $PIP_INSTALL \ + dm-sonnet \ + && \ + ''' diff --git a/generator/modules/tensorflow.py b/generator/modules/tensorflow.py new file mode 100644 index 0000000..eb8f100 --- /dev/null +++ b/generator/modules/tensorflow.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .python import Python + + +@dependency(Python) +@source('pip') +class Tensorflow(Module): + + def build(self): + return r''' + $PIP_INSTALL \ + tensorflow_gpu \ + && \ + ''' diff --git a/generator/modules/theano.py b/generator/modules/theano.py new file mode 100644 index 0000000..e045c33 --- /dev/null +++ b/generator/modules/theano.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools +from .python import Python + + +@dependency(Tools, Python) +@source('git') +class Theano(Module): + + def build(self): + return r''' + $GIT_CLONE https://github.com/Theano/Theano ~/theano && \ + cd ~/theano && \ + $PIP_INSTALL \ + . && \ + + $GIT_CLONE https://github.com/Theano/libgpuarray ~/gpuarray && \ + mkdir -p ~/gpuarray/build && cd ~/gpuarray/build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + .. && \ + make -j"$(nproc)" install && \ + cd ~/gpuarray && \ + python setup.py build && \ + python setup.py install && \ + + printf '[global]\nfloatX = float32\ndevice = cuda0\n\n[dnn]\n''' \ + + r'''include_path = /usr/local/cuda/targets''' \ + + r'''/x86_64-linux/include\n' > ~/.theanorc && \ + ''' diff --git a/generator/modules/tools.py b/generator/modules/tools.py new file mode 100644 index 0000000..d4725c9 --- /dev/null +++ b/generator/modules/tools.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, source + + +@source('apt') +class Tools(Module): + + def __repr__(self): + return '' + + def build(self): + return r''' + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + ca-certificates \ + cmake \ + wget \ + git \ + vim \ + && \ + ''' diff --git a/generator/modules/torch.py b/generator/modules/torch.py new file mode 100644 index 0000000..ed0cb40 --- /dev/null +++ b/generator/modules/torch.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +from .__module__ import Module, dependency, source +from .tools import Tools + + +@dependency(Tools) +@source('git') +class Torch(Module): + + def build(self): + return r''' + $GIT_CLONE https://github.com/torch/distro.git ~/torch''' \ + + r''' --recursive && \ + + cd ~/torch/exe/luajit-rocks && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_LUAJIT21=ON \ + .. && \ + make -j"$(nproc)" install && \ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libreadline-dev \ + && \ + + $GIT_CLONE https://github.com/Yonaba/Moses ~/moses && \ + cd ~/moses && \ + luarocks install rockspec/moses-1.6.1-1.rockspec && \ + + cd ~/torch && \ + sed -i 's/extra\/cudnn/extra\/cudnn ''' \ + + r'''\&\& git checkout R6/' install.sh && \ + sed -i 's/$PREFIX\/bin\/luarocks/luarocks/' install.sh && \ + sed -i '/qt/d' install.sh && \ + sed -i '/Installing Lua/,/^cd \.\.$/d' install.sh && \ + sed -i '/path_to_nvidiasmi/,/^fi$/d' install.sh && \ + sed -i '/Restore anaconda/,/^Not updating$/d' install.sh && \ + sed -i '/You might want to/,/^fi$/d' install.sh && \ + yes no | ./install.sh && \ + ''' diff --git a/generator/requirements.txt b/generator/requirements.txt new file mode 100644 index 0000000..dca9a90 --- /dev/null +++ b/generator/requirements.txt @@ -0,0 +1 @@ +click diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..3c91c8c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,22 @@ +# How to generate `docker/Dockerfile.*` and `circle.yml` + +#### Step 1. generate scripts for generating dockerfiles + +```bash +python make-gen-docker.py +``` + +This should generate `gen-docker.sh`. + +#### Step 2. generate dockerfiles + +Run `gen-docker.sh`, which should generate `docker/Dockerfile.*`. + + +#### Step 3. generate the configuration file for CircleCI + +```bash +python make-circleci.py +``` + +This should generate `circle.yml`. diff --git a/scripts/clean.sh b/scripts/clean.sh new file mode 100644 index 0000000..5b6754e --- /dev/null +++ b/scripts/clean.sh @@ -0,0 +1,3 @@ +rm gen-docker.sh +rm ../docker/* +rm ../circle.yml \ No newline at end of file diff --git a/scripts/gen-docker.sh b/scripts/gen-docker.sh new file mode 100644 index 0000000..c216d17 --- /dev/null +++ b/scripts/gen-docker.sh @@ -0,0 +1,23 @@ +python ../generator/generate.py ../docker/Dockerfile.tensorflow-py27 tensorflow python==2.7 +python ../generator/generate.py ../docker/Dockerfile.tensorflow-py36 tensorflow python==3.6 +python ../generator/generate.py ../docker/Dockerfile.sonnet-py27 sonnet python==2.7 +python ../generator/generate.py ../docker/Dockerfile.sonnet-py36 sonnet python==3.6 +python ../generator/generate.py ../docker/Dockerfile.mxnet-py27 mxnet python==2.7 +python ../generator/generate.py ../docker/Dockerfile.mxnet-py36 mxnet python==3.6 +python ../generator/generate.py ../docker/Dockerfile.cntk-py27 cntk python==2.7 +python ../generator/generate.py ../docker/Dockerfile.cntk-py36 cntk python==3.6 +python ../generator/generate.py ../docker/Dockerfile.keras-py27 keras python==2.7 +python ../generator/generate.py ../docker/Dockerfile.keras-py36 keras python==3.6 +python ../generator/generate.py ../docker/Dockerfile.pytorch-py27 pytorch python==2.7 +python ../generator/generate.py ../docker/Dockerfile.pytorch-py36 pytorch python==3.6 +python ../generator/generate.py ../docker/Dockerfile.chainer-py27 chainer python==2.7 +python ../generator/generate.py ../docker/Dockerfile.chainer-py36 chainer python==3.6 +python ../generator/generate.py ../docker/Dockerfile.theano-py27 theano python==2.7 +python ../generator/generate.py ../docker/Dockerfile.theano-py36 theano python==3.6 +python ../generator/generate.py ../docker/Dockerfile.lasagne-py27 lasagne python==2.7 +python ../generator/generate.py ../docker/Dockerfile.lasagne-py36 lasagne python==3.6 +python ../generator/generate.py ../docker/Dockerfile.caffe-py27 caffe python==2.7 +python ../generator/generate.py ../docker/Dockerfile.caffe-py36 caffe python==3.6 +python ../generator/generate.py ../docker/Dockerfile.torch torch +python ../generator/generate.py ../docker/Dockerfile.all-py27 tensorflow sonnet mxnet cntk keras pytorch chainer theano lasagne caffe torch python==2.7 +python ../generator/generate.py ../docker/Dockerfile.all-py36 tensorflow sonnet mxnet cntk keras pytorch chainer theano lasagne caffe torch python==3.6 diff --git a/scripts/make-circleci.py b/scripts/make-circleci.py new file mode 100644 index 0000000..2e96f6f --- /dev/null +++ b/scripts/make-circleci.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +"""Generate dockerfiles & CI configuration.""" + +import os +import textwrap + + +def indent(n, s): + prefix = ' ' * 4 * n + return ''.join(prefix + l for l in s.splitlines(True)) + + +def get_tags(postfix, py_split='-py'): + tags = [postfix] + if py_split in postfix: + name, pyver = postfix.split(py_split) + if pyver == '36': + tags.append(name) + if name == 'all': + tags.append('py%s' % pyver) + if pyver == '36': + tags.append('latest') + print(tags) + return tags + + +def get_job(tags): + job_name = '_'.join(tags) + build_scripts = indent(1, textwrap.dedent(''' + "%s": + machine: true + steps: + - checkout + - run: docker build %s -f docker/Dockerfile.%s . + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS''' % ( + job_name, + ' '.join('-t $DOCKER_REPO:%s' % tag for tag in tags), + tags[0]))) + for tag in tags: + build_scripts += indent(3, textwrap.dedent(''' + - run: docker push $DOCKER_REPO:%s''' % tag)) + build_scripts += '\n' + return job_name, build_scripts + + +def generate(ci_fname): + with open(ci_fname, 'w') as f: + f.write(textwrap.dedent(''' + version: 2.0 + jobs: + ''')[1:]) + + job_names = [] + for fn in os.listdir(os.path.join('..', 'docker')): + postfix = fn.split('.')[-1] + tags = get_tags(postfix) + job_name, build_scripts = get_job(tags) + job_names.append(job_name) + + with open(ci_fname, 'a') as f: + f.write(build_scripts) + + + workflow_scripts = textwrap.dedent(''' + + workflows: + version: 2 + build: + jobs: + ''')[1:] + workflow_scripts += indent( + 3, '\n'.join('- "%s"' % job_name for job_name in job_names)) + with open(ci_fname, 'a') as f: + f.write(workflow_scripts) + + +if __name__ == '__main__': + generate('../circle.yml') diff --git a/scripts/make-gen-docker.py b/scripts/make-gen-docker.py new file mode 100644 index 0000000..8ef03c1 --- /dev/null +++ b/scripts/make-gen-docker.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +"""Generate scripts for generating dockerfiles.""" + +candidate_modules = [ + 'tensorflow', + 'sonnet', + 'mxnet', + 'cntk', + 'keras', + 'pytorch', + 'chainer', + 'theano', + 'lasagne', + 'caffe', + 'torch', +] + +non_python_modules = [ + 'torch', +] + +pyvers = [ + '2.7', + # '3.5', + '3.6', +] + + +def get_command(modules, postfix): + return 'python ../generator/generate.py ../docker/Dockerfile.%s %s\n' % ( + postfix, ' '.join(m for m in modules)) + + +def generate(script_path): + + with open(script_path, 'w') as f: + + # single module + for module in candidate_modules: + if module in non_python_modules: + modules = [module] + f.write(get_command(modules, module)) + else: + for pyver in pyvers: + modules = [module, 'python==%s' % pyver] + postfix = '%s-py%s' % ( + module, pyver.replace('.', '')) + f.write(get_command(modules, postfix)) + + # all modules + for pyver in pyvers: + modules = candidate_modules + ['python==%s' % pyver] + postfix = 'all-py%s' % pyver.replace('.', '') + f.write(get_command(modules, postfix)) + + +if __name__ == '__main__': + generate('gen-docker.sh')