Add s3_endpoint_url option to support S3-like service (bentoml#656)

* adding s3_endpoint_url config to YataiService * random formatting * nits * fix s3 file path
dogancanbakir · May 14, 2020 · e33406d · e33406d
1 parent cc70ba2
commit e33406d
Show file tree

Hide file tree

Showing 7 changed files with 97 additions and 64 deletions.
diff --git a/README.md b/README.md
@@ -160,6 +160,7 @@ BentoML provides a convenient way to containerize the model API server with Dock
 
 
 ```bash
+# If jq command not found, install jq (the command-line JSON processor) here: https://stedolan.github.io/jq/download/
 saved_path=$(bentoml get IrisClassifier:latest -q | jq -r ".uri.uri")
 
 docker build -t {docker_username}/iris-classifier $saved_path
@@ -178,17 +179,17 @@ scale-to-zero, canary rollout and multi-armed bandit.
 BentoML can also deploy SavedBundle directly to cloud services such as AWS Lambda or 
 AWS SageMaker, with the bentoml CLI command:
 
-```
-> bentoml get IrisClassifier
+```bash
+$ bentoml get IrisClassifier
 BENTO_SERVICE                         CREATED_AT        APIS                       ARTIFACTS
 IrisClassifier:20200121114004_360ECB  2020-01-21 19:40  predict<DataframeHandler>  model<SklearnModelArtifact>
 IrisClassifier:20200120082658_4169CF  2020-01-20 16:27  predict<DataframeHandler>  clf<PickleArtifact>
 ...
 
-> bentoml lambda deploy test-deploy -b IrisClassifier:20200121114004_360ECB
+$ bentoml lambda deploy test-deploy -b IrisClassifier:20200121114004_360ECB
 ...
 
-> bentoml deployment list
+$ bentoml deployment list
 NAME           NAMESPACE    PLATFORM    BENTO_SERVICE                         STATUS    AGE
 test-deploy    dev          aws-lambda  IrisClassifier:20200121114004_360ECB  running   2 days and 11 hours
 ...

diff --git a/bentoml/cli/yatai_service.py b/bentoml/cli/yatai_service.py
@@ -40,27 +40,45 @@ def add_yatai_service_sub_command(cli):
     @click.option(
         '--repo-base-url',
         type=click.STRING,
-        help='Base URL for storing saved BentoService bundle files, this can be a '
-        'filesystem path(POSIX/Windows), or a S3 URL, usually starts with "s3://"',
+        help='Base URL for storing BentoML saved bundle files, this can be a filesystem'
+        'path(POSIX/Windows), or a S3 URL, usually starting with "s3://"',
         envvar='BENTOML_REPO_BASE_URL',
     )
     @click.option(
-        '--grpc-port', type=click.INT, default=50051, help='Port for Yatai server'
+        '--grpc-port',
+        type=click.INT,
+        default=50051,
+        help='Port to run YataiService gRPC server',
+        envvar='BENTOML_GRPC_PORT',
     )
     @click.option(
-        '--ui-port', type=click.INT, default=3000, help='Port for Yatai web UI'
+        '--ui-port',
+        type=click.INT,
+        default=3000,
+        help='Port to run YataiService Web UI server',
+        envvar='BENTOML_WEB_UI_PORT',
     )
     @click.option(
         '--ui/--no-ui',
         default=True,
-        help='Start BentoML YataiService without Web UI',
+        help='Run YataiService with or without Web UI, when running with --no-ui, it '
+        'will only run the gRPC server',
         envvar='BENTOML_ENABLE_WEB_UI',
     )
-    def yatai_service_start(db_url, repo_base_url, grpc_port, ui_port, ui):
+    @click.option(
+        '--s3-endpoint-url',
+        type=click.STRING,
+        help='S3 Endpoint URL is used for deploying with storage services that are '
+        'compatible with Amazon S3, such as MinIO',
+        envvar='BENTOML_S3_ENDPOINT_URL',
+    )
+    def yatai_service_start(
+        db_url, repo_base_url, grpc_port, ui_port, ui, s3_endpoint_url
+    ):
         track_cli('yatai-service-start')
         try:
             start_yatai_service_grpc_server(
-                db_url, repo_base_url, grpc_port, ui_port, ui
+                db_url, repo_base_url, grpc_port, ui_port, ui, s3_endpoint_url
             )
         except BentoMLException as e:
             _echo(f'Yatai gRPC server failed: {str(e)}', CLI_COLOR_ERROR)
diff --git a/bentoml/configuration/default_bentoml.cfg b/bentoml/configuration/default_bentoml.cfg
@@ -19,15 +19,8 @@
 [core]
 debug = false
 usage_tracking = true
-default_repository_base_url = {BENTOML_HOME}/repository/
 bentoml_deploy_version = {BENTOML_VERSION}
 
-[deployment]
-default_namespace = dev
-
-[db]
-url = sqlite:///{BENTOML_HOME}/storage.db
-
 [instrument]
 default_namespace = BENTOML
 prometheus_multiproc_dir = {BENTOML_HOME}/prometheus_multiproc_dir
@@ -56,10 +49,17 @@ yatai_web_server_log_filename = yatai_web_server.log
 zipkin_api_url =
 
 [yatai_service]
-# example for connecting local YataiService grpc server: http://127.0.0.1:50051
+# URL of remote YataiService gRPC server endpoint to use
 url =
+
+# YataiService server configs
+repository_base_url = {BENTOML_HOME}/repository/
+db_url = sqlite:///{BENTOML_HOME}/storage.db
+s3_endpoint_url =
+default_namespace = dev
 client_certificate_file =
 
+
 [apiserver]
 default_port = 5000
 enable_metrics = true

diff --git a/bentoml/repository/__init__.py b/bentoml/repository/__init__.py
@@ -129,21 +129,27 @@ def dangerously_delete(self, bento_name, bento_version):
 
 
 class _S3BentoRepository(BentoRepositoryBase):
-    def __init__(self, base_url):
+    def __init__(self, base_url, s3_endpoint_url=None):
         self.uri_type = BentoUri.S3
 
         parse_result = urlparse(base_url)
         self.bucket = parse_result.netloc
         self.base_path = parse_result.path.lstrip('/')
 
-        self.s3_client = boto3.client("s3")
+        s3_client_args = {}
+        if s3_endpoint_url is not None:
+            s3_client_args['endpoint_url'] = s3_endpoint_url
+        self.s3_client = boto3.client("s3", **s3_client_args)
 
     @property
     def _expiration(self):
         return config('yatai').getint('bento_uri_default_expiration')
 
     def _get_object_name(self, bento_name, bento_version):
-        return "/".join([self.base_path, bento_name, bento_version]) + '.tar.gz'
+        if self.base_path:
+            return "/".join([self.base_path, bento_name, bento_version]) + '.tar.gz'
+        else:
+            return "/".join([bento_name, bento_version]) + '.tar.gz'
 
     def add(self, bento_name, bento_version):
         # Generate pre-signed s3 path for upload
@@ -206,12 +212,12 @@ def dangerously_delete(self, bento_name, bento_version):
 
 
 class BentoRepository(BentoRepositoryBase):
-    def __init__(self, base_url=None):
+    def __init__(self, base_url=None, s3_endpoint_url=None):
         if base_url is None:
             base_url = config().get('default_repository_base_url')
 
         if is_s3_url(base_url):
-            self._repo = _S3BentoRepository(base_url)
+            self._repo = _S3BentoRepository(base_url, s3_endpoint_url)
         else:
             self._repo = _LocalBentoRepository(base_url)
 

diff --git a/bentoml/yatai/__init__.py b/bentoml/yatai/__init__.py
@@ -38,41 +38,33 @@
 
 
 def get_yatai_service(
-    channel_address=None, db_url=None, repo_base_url=None, default_namespace=None
+    channel_address=None,
+    db_url=None,
+    repo_base_url=None,
+    s3_endpoint_url=None,
+    default_namespace=None,
 ):
-    channel_address = channel_address or config().get('yatai_service', 'url')
-    client_cacert_path = (
-        config().get('yatai_service', 'client_certificate_file')
-        or certifi.where()  # default: Mozilla ca cert
-    )
+    channel_address = channel_address or config('yatai_service').get('url')
     channel_address = channel_address.strip()
     if channel_address:
         from bentoml.proto.yatai_service_pb2_grpc import YataiStub
 
-        if db_url is not None:
-            logger.warning(
-                "Config 'db_url' is ignored in favor of remote YataiService at `%s`",
-                channel_address,
-            )
-        if repo_base_url is not None:
-            logger.warning(
-                "Config 'repo_base_url:%s' is ignored in favor of remote YataiService "
-                "at `%s`",
-                repo_base_url,
-                channel_address,
-            )
-        if default_namespace is not None:
+        if any([db_url, repo_base_url, s3_endpoint_url, default_namespace]):
             logger.warning(
-                "Config 'default_namespace:%s' is ignored in favor of remote "
-                "YataiService at `%s`",
-                default_namespace,
+                "Using remote YataiService at `%s`, local YataiService configs "
+                "including db_url, repo_base_url, s3_endpoint_url and default_namespace"
+                "will all be ignored.",
                 channel_address,
             )
-        logger.debug("Using BentoML with remote Yatai server: %s", channel_address)
 
+        logger.debug("Connecting YataiService gRPC server at: %s", channel_address)
         scheme, addr = parse_grpc_url(channel_address)
 
         if scheme in ('grpcs', 'https'):
+            client_cacert_path = (
+                config().get('yatai_service', 'client_certificate_file')
+                or certifi.where()  # default: Mozilla ca cert
+            )
             with open(client_cacert_path, 'rb') as ca_cert_file:
                 ca_cert = ca_cert_file.read()
             credentials = grpc.ssl_channel_credentials(ca_cert, None, None)
@@ -83,24 +75,24 @@ def get_yatai_service(
     else:
         from bentoml.yatai.yatai_service_impl import YataiService
 
-        logger.debug("Using BentoML with local Yatai server")
-
-        default_namespace = default_namespace or config().get(
-            'deployment', 'default_namespace'
-        )
-        repo_base_url = repo_base_url or config().get('default_repository_base_url')
-        db_url = db_url or config().get('db', 'url')
-
+        logger.debug("Creating local YataiService instance")
         return YataiService(
             db_url=db_url,
             repo_base_url=repo_base_url,
+            s3_endpoint_url=s3_endpoint_url,
             default_namespace=default_namespace,
         )
 
 
-def start_yatai_service_grpc_server(db_url, repo_base_url, grpc_port, ui_port, with_ui):
+def start_yatai_service_grpc_server(
+    db_url, repo_base_url, grpc_port, ui_port, with_ui, s3_endpoint_url
+):
     track_server('yatai-service-grpc-server')
-    yatai_service = get_yatai_service(db_url=db_url, repo_base_url=repo_base_url)
+    from bentoml.yatai.yatai_service_impl import YataiService
+
+    yatai_service = YataiService(
+        db_url=db_url, repo_base_url=repo_base_url, s3_endpoint_url=s3_endpoint_url,
+    )
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
     add_YataiServicer_to_server(yatai_service, server)
     debug_mode = config().getboolean('core', 'debug')
@@ -140,7 +132,9 @@ def start_yatai_service_grpc_server(db_url, repo_base_url, grpc_port, ui_port, w
         f'* Debug mode: { "on" if debug_mode else "off"}\n'
         f'* Web UI: {f"running on http://127.0.0.1:{ui_port}" if with_ui else "off"}\n'
         f'* Running on 127.0.0.1:{grpc_port} (Press CTRL+C to quit)\n'
-        f'* Usage: `bentoml config set yatai_service.url=127.0.0.1:{grpc_port}`\n'
+        f'* Usage:\n'
+        f'*  Set config: `bentoml config set yatai_service.url=127.0.0.1:{grpc_port}`\n'
+        f'*  Set env var: `export BENTOML__YATAI_SERVICE__URL=127.0.0.1:{grpc_port}`\n'
         f'* Help and instructions: '
         f'https://docs.bentoml.org/en/latest/guides/yatai_service.html\n'
         f'{f"* Web server log can be found here: {web_ui_log_path}" if with_ui else ""}'

diff --git a/bentoml/yatai/yatai_service_impl.py b/bentoml/yatai/yatai_service_impl.py
@@ -18,6 +18,7 @@
 
 import logging
 
+from bentoml import config
 from bentoml.proto.deployment_pb2 import (
     GetDeploymentResponse,
     DescribeDeploymentResponse,
@@ -57,9 +58,21 @@ class YataiService(YataiServicer):
 
     # pylint: disable=unused-argument
 
-    def __init__(self, db_url, repo_base_url, default_namespace):
+    def __init__(
+        self,
+        db_url=None,
+        repo_base_url=None,
+        s3_endpoint_url=None,
+        default_namespace=None,
+    ):
+        cfg = config('yatai_service')
+        repo_base_url = repo_base_url or cfg.get('repository_base_url')
+        db_url = db_url or cfg.get('db_url')
+        s3_endpoint_url = s3_endpoint_url or cfg.get('s3_endpoint_url') or None
+        default_namespace = default_namespace or cfg.get('default_namespace')
+
         self.default_namespace = default_namespace
-        self.repo = BentoRepository(repo_base_url)
+        self.repo = BentoRepository(repo_base_url, s3_endpoint_url)
         self.sess_maker = init_db(db_url)
         self.deployment_store = DeploymentStore(self.sess_maker)
         self.bento_metadata_store = BentoMetadataStore(self.sess_maker)

diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py
@@ -143,10 +143,11 @@ def temporary_yatai_service_url():
     with TempDirectory() as temp_dir:
         temp_docker_file_path = os.path.join(temp_dir, 'Dockerfile')
         with open(temp_docker_file_path, 'w') as f:
-            f.write(f"""\
-        FROM bentoml/yatai-service:{PREV_PYPI_RELEASE_VERSION}
-        ADD . /bentoml-local-repo
-        RUN pip install /bentoml-local-repo
+            f.write(
+                f"""\
+FROM bentoml/yatai-service:{PREV_PYPI_RELEASE_VERSION}
+ADD . /bentoml-local-repo
+RUN pip install /bentoml-local-repo
             """
             )
         logger.info('building docker image')