Use venv for Python development

.gitignore

@@ -15,3 +15,144 @@ python/replicate/bin
 # scratch notebooks
 Untitled.ipynb
 .ipynb_checkpoints
+
+#Taken from https://github.com/github/gitignore/blob/master/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

.vscode/settings.json

@@ -1,5 +1,13 @@
 {
   "files.exclude": {
     "go": true
-  }
+  },
+  "python.pythonPath": "${workspaceFolder}/venv/bin/python3",
+  "python.testing.pytestArgs": [
+    "python/tests",
+    "end-to-end-test/end_to_end_test",
+  ],
+  "python.testing.unittestEnabled": false,
+  "python.testing.nosetestsEnabled": false,
+  "python.testing.pytestEnabled": true
 }

Makefile

@@ -1,31 +1,40 @@
 ENVIRONMENT := development
+ROOTDIR=./
+VENV=$(ROOTDIR)venv/bin/
+
+.PHONY: venv
+venv:
+	test -d venv || python -m venv $(ROOTDIR)venv && $(VENV)pip install --upgrade pip
 
 OS := $(shell uname -s)
 
 .PHONY: build
-build: verify-dev-env
+build: verify-dev-env venv
 	cd go && $(MAKE) build-all ENVIRONMENT=$(ENVIRONMENT)
 	cd python && $(MAKE) build
 
 .PHONY: install
 install: build
 ifeq ($(OS),Linux)
-	pip install python/dist/replicate-*-py3-none-manylinux1_x86_64.whl
+	$(VENV)pip install python/dist/replicate-*-py3-none-manylinux1_x86_64.whl
 else ifeq ($(OS),Darwin)
-	pip install python/dist/replicate-*-py3-none-macosx_*.whl
+	$(VENV)pip install python/dist/replicate-*-py3-none-macosx_*.whl
 else
 	@echo Unknown OS: $(OS)
 endif
+	cd go && $(MAKE) copy-binary
 
 .PHONY: develop
-develop: verify-dev-env
+develop: verify-dev-env venv
 	cd go && $(MAKE) build
 	cd go && $(MAKE) install
-	cd python && python setup.py develop
+	cd python && $(MAKE) develop
+	@printf "\nPython venv created at $(ROOTDIR)venv. Run `tput bold`source $(ROOTDIR)venv/bin/activate`tput sgr0` to activate the venv in your shell.\n"
+	@printf "If you're using vscode, select this venv as the Python interpretor in the bottom left.\n"
 
 .PHONY: install-test-dependencies
-install-test-dependencies:
-	pip install -r requirements-test.txt
+install-test-dependencies: venv
+	$(VENV)pip install -r requirements-test.txt
 
 .PHONY: test
 test: install-test-dependencies develop

end-to-end-test/Makefile

@@ -1,7 +1,10 @@
+ROOTDIR=../
+VENV=$(ROOTDIR)venv/bin/
+
 .PHONY: test
 test:
-	pytest -m "not external"
+	$(VENV)pytest -m "not external"
 
 .PHONY: test-external
 test-external:
-	pytest
+	$(VENV)pytest

end-to-end-test/end_to_end_test/test_checkout.py

@@ -5,7 +5,8 @@
 import os
 import subprocess
 import pytest  # type: ignore
-from .utils import get_env
+
+from .utils import get_env, PYTHON_PATH
 
 
 @pytest.mark.parametrize(
@@ -69,7 +70,7 @@ def main():
         )
 
     env = get_env()
-    cmd = ["python", "train.py"]
+    cmd = [PYTHON_PATH, "train.py"]
     subprocess.run(cmd, cwd=tmpdir, env=env, check=True)
 
     experiments = json.loads(

end-to-end-test/end_to_end_test/test_delete.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 import pytest  # type: ignore
 
-from .utils import path_exists, get_env
+from .utils import path_exists, get_env, PYTHON_PATH
 
 
 @pytest.mark.parametrize(
@@ -57,7 +57,7 @@ def main():
         )
 
     env = get_env()
-    cmd = ["python", "train.py", "--foo"]
+    cmd = [PYTHON_PATH, "train.py", "--foo"]
     subprocess.run(cmd, cwd=tmpdir, env=env, check=True)
 
     experiments = json.loads(

end-to-end-test/end_to_end_test/test_list.py

@@ -4,7 +4,7 @@
 import pytest  # type: ignore
 from dateutil.parser import parse as parse_date
 
-from .utils import get_env
+from .utils import get_env, PYTHON_PATH
 
 
 @pytest.mark.parametrize(
@@ -58,7 +58,7 @@ def main():
 
     env = get_env()
 
-    subprocess.run(["python", "train.py", "--foo"], cwd=tmpdir, env=env, check=True)
+    subprocess.run([PYTHON_PATH, "train.py", "--foo"], cwd=tmpdir, env=env, check=True)
 
     experiments = json.loads(
         subprocess.run(
@@ -106,7 +106,9 @@ def main():
 experiment.checkpoint(path=".", step=3)
 """
         )
-    subprocess.run(["python", "train2.py", exp["id"]], cwd=tmpdir, env=env, check=True)
+    subprocess.run(
+        [PYTHON_PATH, "train2.py", exp["id"]], cwd=tmpdir, env=env, check=True
+    )
     experiments = json.loads(
         subprocess.run(
             ["replicate", "--verbose", "list", "--json"],

end-to-end-test/end_to_end_test/utils.py

@@ -5,6 +5,7 @@
 from google.cloud import storage as google_storage
 
 ROOT_DIRECTORY = Path(__file__).parent.parent.parent
+PYTHON_PATH = os.path.abspath(os.path.join(ROOT_DIRECTORY, "venv/bin/python"))
 
 
 def get_env():

go/Makefile

@@ -51,14 +51,17 @@ build-all:
 
 # install without sudo if the install path exists and is writeable,
 # or if it doesn't exist and its directory is writeable
-.PHONY: install
-install: build
+.PHONY: copy-binary
+copy-binary:
 	if [[ (-f "$(INSTALL_PATH)" && -w "$(INSTALL_PATH)") || (! -f "$(INSTALL_PATH)" && -w $$(dirname "$(INSTALL_PATH)")) ]]; then \
 	    cp $(BINARY) $(INSTALL_PATH); \
 	else \
 	    sudo cp $(BINARY) $(INSTALL_PATH); \
 	fi
 
+.PHONY: install
+install: build copy-binary
+
 .PHONY: clean
 clean:
 	rm -rf $(RELEASE_DIR)

python/Makefile

@@ -1,25 +1,32 @@
+ROOTDIR=../
+VENV=$(ROOTDIR)venv/bin/
+
 .PHONY: pre-commit
 pre-commit: lint
 
 .PHONY: fmt
 fmt:
-	black .
+	$(VENV)black .
+
+.PHONY: develop
+develop:
+	$(VENV)python setup.py develop
 
 # --import-mode makes pytest import from globally installed package by default instead of local replicate/ module.
 # This assumes you've run `make develop` before running tests
 # https://docs.pytest.org/en/stable/pythonpath.html#pythonpath
 .PHONY: test
 test: lint
-	pytest --import-mode=importlib -m "not external"
+	$(VENV)pytest --import-mode=importlib -m "not external"
 
 .PHONY: test-external
 test-external: lint
-	pytest --import-mode=importlib
+	$(VENV)pytest --import-mode=importlib
 
 .PHONY: lint
 lint:
-	mypy .
-	black --check .
+	$(VENV)mypy .
+	$(VENV)black --check .
 
 .PHONY: clean
 clean:
@@ -42,15 +49,15 @@ clean:
 # https://docs.python.org/3/distutils/apiref.html#distutils.util.get_platform
 .PHONY: build
 build: clean
-	pip install wheel
-	python setup.py bdist_wheel --plat-name manylinux1_x86_64
-	python setup.py bdist_wheel --plat-name macosx_10_9_x86_64
+	$(VENV)pip install wheel
+	$(VENV)python setup.py bdist_wheel --plat-name manylinux1_x86_64
+	$(VENV)python setup.py bdist_wheel --plat-name macosx_10_9_x86_64
 
 .PHONY: targets
 targets:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$'
 
 .PHONY: vendor
 vendor:
-	pip install vendoring
-	vendoring sync -v .
+	$(VENV)pip install vendoring
+	$(VENV)vendoring sync -v .