diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..b1790f6 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,62 @@ +ARG PY_IMG_VARIANT=3.9-slim +ARG POETRY_VARIANT=1.3.1 + +FROM python:${PY_IMG_VARIANT} +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + + +ENV PYTHONUNBUFFERED=1 \ + # prevents python creating .pyc files + PYTHONDONTWRITEBYTECODE=1 \ + # pip + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_NO_CACHE_DIR=off \ + PIP_DEFAULT_TIMEOUT=100 \ + \ + # poetry + # https://python-poetry.org/docs/configuration/#using-environment-variables + POETRY_VERSION=${POETRY_VARIANT} \ + # do not ask any interactive question + POETRY_NO_INTERACTION=1 \ + # make poetry use gloabal pakages + # https://github.com/python-poetry/poetry/pull/3209#issuecomment-710678083 + POETRY_VIRTUALENVS_CREATE=false \ + # make poetry install to this location + POETRY_HOME="/opt/poetry" \ + \ + # paths + # this is where our requirements + virtual environment will live + PYSETUP_PATH="/opt/pysetup" + +ENV PATH="$POETRY_HOME/bin:$PATH" + +# install systep packages +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 + && apt-get purge -y imagemagick imagemagick-6-common \ + && apt-get install --no-install-recommends -y \ + sudo \ + bash \ + curl \ + build-essential \ + python3-dev + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && touch /etc/sudoers.d/$USERNAME \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +# install python package manager and builder +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - + +# copy project requirement files here to ensure they will be cached +# https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-346113 +# becouse of docker isolation there is no venv, vscode will mount project catalog to image by himself +WORKDIR $PYSETUP_PATH +COPY poetry.lock pyproject.toml ./ +RUN poetry install --no-root --no-ansi +EXPOSE 8888 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec7fdda..e2a9fde 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,27 +1,68 @@ +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.202.3/containers/python-3 { "name": "Python 3", - "image": "mcr.microsoft.com/devcontainers/python:3.10-bullseye", - "features": { - "ghcr.io/devcontainers/features/node:1": { - "version": "none" - }, - "ghcr.io/devcontainers/features/docker-in-docker:1": {}, - "ghcr.io/devcontainers/features/git:1": {} + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + "PY_IMG_VARIANT": "3.8-slim", + "POETRY_VARIANT": "1.3.1" + } }, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "apt-get update && apt-get install -y binutils upx-ucl && pip install pyinstaller && pip install .", - - // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", + // Configure tool-specific properties. "customizations": { + // Configure properties specific to VS Code. "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.languageServer": "Default", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. "extensions": [ - "ms-python.python" + "ms-python.python", + "ms-python.vscode-pylance" ] } - } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [8888], + + // Use 'portsAttributes' to set default properties for specific forwarded ports. More info: https://code.visualstudio.com/docs/remote/devcontainerjson-reference. + "portsAttributes": { + "8888": { + "label": "local proxy in servre mode", + "onAutoForward": "notify" + } + }, + + // Use 'otherPortsAttributes' to configure any ports that aren't configured using 'portsAttributes'. + // "otherPortsAttributes": { + // "onAutoForward": "silent" + // }, + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "poetry run proxybroker --version", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" } diff --git a/Dockerfile b/Dockerfile index 6df2f48..3e08424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,70 @@ -FROM python:3.9-slim as base +ARG PY_VARIANT=3.8 +ARG PY_IMG_VARIANT=slim +ARG POETRY_VARIANT=1.3.1 +# `builder-base-python-base` stage is used to build deps + create our virtual environment +FROM python:${PY_VARIANT}-${PY_IMG_VARIANT} as python-base + +ARG IMG_VER +ARG BUILD_DATE +ARG APP_NAME="" +ARG OWNER_NAME="" +# https://github.com/opencontainers/image-spec +LABEL org.opencontainers.image.authors="Matthew Lien, Ronnie McGrog" \ + org.opencontainers.image.url="https://github.com/${OWNER_NAME}/${APP_NAME}" \ + org.opencontainers.image.documentation="https://github.com/${OWNER_NAME}/${APP_NAME}/blob/master/README.md" \ + org.opencontainers.image.source="https://github.com/${OWNER_NAME}/${APP_NAME}/blob/master/Dockerfile" \ + org.opencontainers.image.title="${APP_NAME}" \ + org.opencontainers.image.description="The New (auto rotate) Proxy [Finder | Checker | Server]" \ + org.opencontainers.image.version="${IMG_VER}" \ + org.opencontainers.image.created="${BUILD_DATE}" ENV \ # Keeps Python from generating .pyc files in the container PYTHONDONTWRITEBYTECODE=1 \ # Turns off buffering for easier container logging PYTHONUNBUFFERED=1 \ + # pip PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=100 \ + # poetry + POETRY_VERSION=${POETRY_VARIANT} \ + # do not ask any interactive question + POETRY_NO_INTERACTION=1 \ + # make poetry create the virtual environment + POETRY_VIRTUALENVS_CREATE=true \ + POETRY_VIRTUALENVS_IN_PROJECT=true \ + # make poetry install to this location + POETRY_HOME="/opt/poetry" \ + # paths + PYSETUP_PATH="/opt/pysetup" \ + POETRY_VIRTUALENVS_PATH="/opt/pysetup/.venv" -#RUN apt-get update \ -# && apt-get install -y --no-install-recommends gcc libc-dev libffi-dev \ -# && apt-get clean +# this is activate venv, no need `. $POETRY_VIRTUALENVS_PATH/bin/activate` +ENV PATH="$POETRY_HOME/bin:$POETRY_VIRTUALENVS_PATH/bin:$PATH" -RUN \ - pip install -U poetry +FROM python-base as builder-base -FROM base as builder +RUN export DEBIAN_FRONTEND=noninteractive\ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + # deps for installing poetry + curl \ + # deps for building python deps + build-essential -WORKDIR /app -COPY poetry.lock pyproject.toml ./ +# install poetry - respects $POETRY_VERSION & $POETRY_HOME +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - -RUN poetry config virtualenvs.create false && \ - poetry install --no-interaction --no-ansi --no-dev +WORKDIR $PYSETUP_PATH +COPY poetry.lock pyproject.toml ./ +RUN poetry install --without dev -COPY proxybroker proxybroker +FROM python-base as production +WORKDIR $PYSETUP_PATH +# copy venv too +COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH +COPY ./proxybroker proxybroker EXPOSE 8888 - -ENTRYPOINT ["python", "-m", "proxybroker" ] +# venv is activated, so all requirements are available +ENTRYPOINT ["python", "-m", "proxybroker"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89df9c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +# VERSIONS --------------------------------------------------------------------- +IMG_VER=2.0.0-alpha5 +PY_VARIANT=3.8 +PY_IMG_VARIANT=slim-buster +POETRY_VARIANT=1.3.1 +APP_NAME=proxybroker2 +OWNER_NAME=mcgr0g +IMG_NAME=$(OWNER_NAME)/$(APP_NAME) +BUILD_DATE:=$(shell date '+%Y-%m-%d') + +BFLAGS=docker buildx build \ + --build-arg IMG_VER=$(IMG_VER) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --build-arg PY_VARIANT=$(PY_VARIANT) \ + --build-arg PY_IMG_VARIANT=$(PY_IMG_VARIANT) \ + --build-arg POETRY_VARIANT=$(POETRY_VARIANT) \ + --build-arg APP_NAME=$(APP_NAME) \ + --build-arg OWNER_NAME=$(OWNER_NAME) \ + --tag $(IMG_NAME):$(IMG_VER) \ + --tag $(IMG_NAME):latest + +BUILD_FAST=$(BFLAGS) . +BUILD_FULL=$(BFLAGS) --progress=plain --no-cache . + +# IMAGE ----------------------------------------------------------------------- + +img-build: kit-install + $(BUILD_FAST) + +img-build-full: + $(BUILD_FULL) + +img-get-size-final: + docker images \ + -f "label=org.opencontainers.image.title=$(APP_NAME)" \ + -f "label=org.opencontainers.image.version=$(IMG_VER)" \ + --format "table {{.ID}}\t{{.Repository}}\t{{.Tag}}\t{{.CreatedAt}}\t{{.Size}}" \ + $(IMG_NAME):$(IMG_VER) + +img-get-size-layered: + docker history -H $(IMG_NAME):$(IMG_VER) + +img-push: + docker push $(IMG_NAME) --all-tags +# CONTAINER ------------------------------------------------------------------- + +run-find: + docker run --rm --name $(APP_NAME) \ + $(IMG_NAME):$(IMG_VER) \ + --log INFO find --types HTTPS --lvl High --countries US UK DE FR NL --strict -l 10 + +run-serve: + docker run --rm --name $(APP_NAME) \ + $(IMG_NAME):$(IMG_VER) \ + --log INFO serve --host 127.0.0.1 --port 8888 --types HTTPS --lvl High --countries UK DE FR NL --min-queue 5 + +# BuildKit ------------------------------------------------------------------- +AFLAGS=--platform linux/amd64,linux/arm64/v8 --push #--progress=plain +BUILD_MULIARCH=$(BFLAGS) $(AFLAGS) . +BUILDER_NAME=kofee_shop + +kit-install: + docker buildx install + +kit-context: kit-install + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker buildx create --name $(BUILDER_NAME)--use --bootstrap + +multi-arch: kit-context + $(BUILD_MULIARCH) + docker buildx rm $(BUILDER_NAME) \ No newline at end of file diff --git a/README.md b/README.md index 3f33fde..49e1a80 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,27 @@ Contributing - Submit a pull request! - [Contributor workflow](https://github.com/bluet/proxybroker2/issues/93) +### Dev tool on Windows +In one step: with VS Code just clone project into WSL2 and agree to reopen it in [Dev Containder](https://code.visualstudio.com/learn/develop-cloud/containers). +Now try application `poetry run proxybroker --log INFO find --types SOCKS5 --lvl High --countries US UK DE FR NL --strict -l 10` + +NB: to build your own image you have to switch to usual WSL. Then use Build tool on linux + + +### Build tools on linux +prerequisites: [buildx](https://docs.docker.com/build/install-buildx/) +just try `make img-build`, `make run-find` and `make img-push` for linux/amd64 + +Makefile has instruction for multi-arch build. But you need some prerequisites: + +qemu [support](https://stackoverflow.com/questions/60080264/docker-cannot-build-multi-platform-images-with-docker-buildx): +``` +sudo apt purge --auto-remove qemu-user qemu-user-binfmt binfmt-support +sudo apt install qemu-user +``` + +After that use `make multi-arch` and wait about 400 seconds + License ------- diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index 296fc05..0000000 --- a/docker-build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -VERSION=v2.0.0-alpha4 - -docker build --pull -t bluet/proxybroker2 . -docker scan bluet/proxybroker2:latest - -docker tag bluet/proxybroker2:latest bluet/proxybroker2:${VERSION} -git tag "${VERSION}" -a -m "proxybroker2 ${VERSION}" -git push -git push --tags - -# Fixes busybox trigger error https://github.com/tonistiigi/xx/issues/36#issuecomment-926876468 -docker run --privileged -it --rm tonistiigi/binfmt --install all - -docker buildx create --use - -while true; do - read -p "Have I Updated VERSION Info? (Is current VERSION=${VERSION} ?) [y/N]" yn - case $yn in - [Yy]* ) docker buildx build -t bluet/proxybroker2:latest -t bluet/proxybroker2:${VERSION} --platform linux/amd64,linux/arm64/v8 --pull --push .; break;; - [Nn]* ) exit;; - * ) echo "";; - esac -done - -