allow publishing via /api/bookstore/published

Author: rgbkrk

.gitignore

@@ -112,4 +112,5 @@ binder/*run*.ipynb
 
 # Local jupyter notebook config for working on bookstore
 jupyter_notebook_config.py
+jupyter_config.py
 

bookstore/archive.py

@@ -11,6 +11,8 @@
 
 from .bookstore_config import BookstoreSettings
 
+from .s3_paths import s3_path, s3_display_path
+
 
 class ArchiveRecord(NamedTuple):
     filepath: str
@@ -28,7 +30,8 @@ def __init__(self, *args, **kwargs):
         # opt ourselves into being part of the Jupyter App that should have Bookstore Settings applied
         self.settings = BookstoreSettings(parent=self)
 
-        self.log.info("Archiving notebooks to {}".format(self.full_prefix))
+        self.log.info("Archiving notebooks to {}".format(s3_display_path(
+            self.settings.s3_bucket, self.settings.workspace_prefix)))
 
         self.fs = s3fs.S3FileSystem(
             key=self.settings.s3_access_key_id,
@@ -82,22 +85,9 @@ def write_to_s3(self, full_path: str, content: str):
         with self.fs.open(full_path, mode="wb") as f:
             f.write(content.encode("utf-8"))
 
-    @property
-    def delimiter(self):
-        """It's a slash! Normally this could be configurable. This leaves room for that later,
-        keeping it centralized for now"""
-        return "/"
-
-    @property
-    def full_prefix(self):
-        """Full prefix: bucket + workspace prefix"""
-        return self.delimiter.join(
-            [self.settings.s3_bucket, self.settings.workspace_prefix]
-        )
-
     def s3_path(self, path):
         """compute the s3 path based on the bucket, prefix, and the path to the notebook"""
-        return self.delimiter.join([self.full_prefix, path])
+        return s3_path(self.settings.s3_bucket, self.settings.workspace_prefix, path)
 
     def run_pre_save_hook(self, model, path, **kwargs):
         """Store notebook to S3 when saves happen

bookstore/handlers.py

@@ -1,13 +1,19 @@
 import json
+import s3fs
 
 from notebook.base.handlers import APIHandler
 from notebook.utils import url_path_join
-from tornado import web
+from tornado import web, gen
 
 from ._version import get_versions
+from .bookstore_config import BookstoreSettings
+
+from notebook.base.handlers import IPythonHandler, APIHandler, path_regex
 
 version = get_versions()['version']
 
+from .s3_paths import s3_path, s3_display_path
+
 
 class BookstoreVersionHandler(APIHandler):
     """Returns the version of bookstore currently running. Used mostly to lay foundations
@@ -19,13 +25,104 @@ def get(self):
         self.finish(json.dumps({"bookstore": True, "version": version}))
 
 
+# NOTE: We need to ensure that publishing is not configured if bookstore settings are not
+# set. Because of how the APIHandlers cannot be configurable, all we can do is reach into settings
+
+
+class BookstorePublishHandler(APIHandler):
+    """Publish a notebook to the publish path"""
+
+    def __init__(self, *args, **kwargs):
+        super(APIHandler, self).__init__(*args, **kwargs)
+        # create an easy helper to get at our bookstore settings quickly
+        self.bookstore_settings = BookstoreSettings(
+            config=self.settings['config']['BookstoreSettings']
+        )
+
+        self.fs = s3fs.S3FileSystem(
+            key=self.bookstore_settings.s3_access_key_id,
+            secret=self.bookstore_settings.s3_secret_access_key,
+            client_kwargs={
+                "endpoint_url": self.bookstore_settings.s3_endpoint_url,
+                "region_name": self.bookstore_settings.s3_region_name,
+            },
+            config_kwargs={},
+            s3_additional_kwargs={},
+        )
+
+    @property
+    def bucket(self):
+        return self.bookstore_settings.s3_bucket
+
+    @property
+    def prefix(self):
+        return self.bookstore_settings.published_prefix
+
+    def s3_path(self, path):
+        """compute the s3 path based on the bucket, prefix, and the path to the notebook"""
+        return s3_path(self.bucket, self.prefix, path)
+
+    def _publish(self, model, path):
+        if model['type'] != 'notebook':
+            raise web.HTTPError(400, "bookstore only publishes notebooks")
+        content = model['content']
+
+        full_s3_path = self.s3_path(path)
+
+        self.log.info("Publishing to %s", s3_display_path(self.bucket, self.prefix, path))
+
+        # Likely implementation:
+        #
+        # with self.fs.open(full_s3_path, mode="wb") as f:
+        #     f.write(content.encode("utf-8"))
+        #
+        # However, we need to get back other information for our response
+        # Ideally, we'd return back the version id
+        #
+        # self.status(201)
+        # self.finish(json.dumps({"s3path": full_s3_path, "versionID": vID}))
+        #
+
+        # Return 501 - Not Implemented
+        # Until we're ready
+        self.set_status(501)
+
+    @web.authenticated
+    @gen.coroutine
+    def put(self, path=''):
+        '''Publish a notebook on a given path. The payload for this directly matches that of the contents API for PUT.
+        '''
+        if path == '' or path == '/':
+            raise web.HTTPError(400, "Must have a path to publish to")
+
+        model = self.get_json_body()
+
+        if model:
+            self._publish(model, path.lstrip('/'))
+        else:
+            raise web.HTTPError(400, "Cannot publish empty model")
+
+
 def load_jupyter_server_extension(nb_app):
     web_app = nb_app.web_app
     host_pattern = '.*$'
-    base_bookstore_pattern = url_path_join(
-        web_app.settings['base_url'], '/api/bookstore'
-    )
+    base_bookstore_pattern = url_path_join(web_app.settings['base_url'], '/api/bookstore')
+
+    # Always enable the version handler
+    web_app.add_handlers(host_pattern, [(base_bookstore_pattern, BookstoreVersionHandler)])
+
+    config = web_app.settings['config']
+    bookstore_settings = config.get("BookstoreSettings")
 
-    web_app.add_handlers(
-        host_pattern, [(base_bookstore_pattern, BookstoreVersionHandler)]
-    )
+    if not bookstore_settings:
+        nb_app.log.info("Not enabling bookstore publishing since bookstore endpoint not configured")
+    else:
+        web_app.add_handlers(
+            host_pattern,
+            [
+                (
+                    url_path_join(base_bookstore_pattern, r"/published%s" % path_regex),
+                    BookstorePublishHandler,
+                )
+            ],
+        )

bookstore/s3_paths.py

@@ -0,0 +1,22 @@
+# Our S3 path delimiter will remain fixed as '/' in all uses
+delimiter = "/"
+
+
+def _join(*args):
+    '''Join S3 bucket args together, removing empty entries and stripping left-leading
+    '''
+    return delimiter.join(filter(lambda s: s != '', map(lambda s: s.lstrip(delimiter), args)))
+
+
+def s3_path(bucket, prefix, path=''):
+    """compute the s3 path based on the bucket, prefix, and the path to the notebook"""
+    return _join(bucket, prefix, path)
+
+
+def s3_display_path(bucket, prefix, path=''):
+    """create a display name for use in logs"""
+    return 's3://' + s3_path(bucket, prefix, path)
+
+
+print(s3_path('notebooks', 'workspace', 'test/what.ipynb'))
+print(s3_display_path('notebooks', 'workspace', 'test/what.ipynb'))

ci/jupyter_notebook_config.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+print("using CI config")
+
 from bookstore import BookstoreContentsArchiver, BookstoreSettings
 
 # jupyter config
@@ -11,9 +13,10 @@
 c.NotebookApp.contents_manager_class = BookstoreContentsArchiver
 
 c.BookstoreSettings.workspace_prefix = "ci-workspace"
+c.BookstoreSettings.published_prefix = "ci-published"
 
 # If using minio for development
-c.BookstoreSettings.s3_endpoint_url = "http://127.0.0.1:9000"
+c.BookstoreSettings.s3_endpoint_url = "http://localhost:9000"
 c.BookstoreSettings.s3_bucket = "bookstore"
 
 # Straight out of `circleci/config.yml`

ci/s3.js

@@ -45,11 +45,17 @@ class Client {
     this.minioClient = new Minio.Client(s3Config);
   }
 
-  makeBucket(bucketName) {
-    return makeBucket(this.minioClient, bucketName);
+  async makeBucket(bucketName) {
+    const bucketExists = await this.minioClient.bucketExists(bucketName);
+
+    if (bucketExists) {
+      return;
+    }
+
+    return await makeBucket(this.minioClient, bucketName);
   }
 
-  getObject(bucketName, objectName) {
+  async getObject(bucketName, objectName) {
     return getObject(this.minioClient, bucketName, objectName);
   }
 }

jupyter_notebook_config.py.example jupyter_config.py.example

@@ -8,7 +8,7 @@
   },
   "private": "true",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node ci/integration.js"
   },
   "repository": {
     "type": "git",