diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..180e9a37 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.pyc +/build/ +/dist/ +/.conda/ +/.jupyter/ +/.local/ +/node_modules/ +/notebooks*/ +/*.egg-info +/*.egg +/*.eggs +.DS_Store +.vagrant \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2f4030ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +RUN python -m pip install six click nbformat diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..50e720ec --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,361 @@ +### GNU GENERAL PUBLIC LICENSE + +Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +### Preamble + +The licenses for most software are designed to take away your freedom +to share and change it. By contrast, the GNU General Public License is +intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + +To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if +you distribute copies of the software, or if you modify it. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + +We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + +Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, +we want its recipients to know that what they have is not the +original, so that any problems introduced by others will not reflect +on the original authors' reputations. + +Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at +all. + +The precise terms and conditions for copying, distribution and +modification follow. + +### TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +**0.** This License applies to any program or other work which +contains a notice placed by the copyright holder saying it may be +distributed under the terms of this General Public License. The +"Program", below, refers to any such program or work, and a "work +based on the Program" means either the Program or any derivative work +under copyright law: that is to say, a work containing the Program or +a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is +included without limitation in the term "modification".) Each licensee +is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the Program +(independent of having been made by running the Program). Whether that +is true depends on what the Program does. + +**1.** You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a +fee. + +**2.** You may modify your copy or copies of the Program or any +portion of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + +**a)** You must cause the modified files to carry prominent notices +stating that you changed the files and the date of any change. + + +**b)** You must cause any work that you distribute or publish, that in +whole or in part contains or is derived from the Program or any part +thereof, to be licensed as a whole at no charge to all third parties +under the terms of this License. + + +**c)** If the modified program normally reads commands interactively +when run, you must cause it, when started running for such interactive +use in the most ordinary way, to print or display an announcement +including an appropriate copyright notice and a notice that there is +no warranty (or else, saying that you provide a warranty) and that +users may redistribute the program under these conditions, and telling +the user how to view a copy of this License. (Exception: if the +Program itself is interactive but does not normally print such an +announcement, your work based on the Program is not required to print +an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + +**3.** You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + +**a)** Accompany it with the complete corresponding machine-readable +source code, which must be distributed under the terms of Sections 1 +and 2 above on a medium customarily used for software interchange; or, + + +**b)** Accompany it with a written offer, valid for at least three +years, to give any third party, for a charge no more than your cost of +physically performing source distribution, a complete machine-readable +copy of the corresponding source code, to be distributed under the +terms of Sections 1 and 2 above on a medium customarily used for +software interchange; or, + + +**c)** Accompany it with the information you received as to the offer +to distribute corresponding source code. (This alternative is allowed +only for noncommercial distribution and only if you received the +program in object code or executable form with such an offer, in +accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + +**4.** You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt otherwise +to copy, modify, sublicense or distribute the Program is void, and +will automatically terminate your rights under this License. However, +parties who have received copies, or rights, from you under this +License will not have their licenses terminated so long as such +parties remain in full compliance. + +**5.** You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +**6.** Each time you redistribute the Program (or any work based on +the Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + +**7.** If, as a consequence of a court judgment or allegation of +patent infringement or for any other reason (not limited to patent +issues), conditions are imposed on you (whether by court order, +agreement or otherwise) that contradict the conditions of this +License, they do not excuse you from the conditions of this License. +If you cannot distribute so as to satisfy simultaneously your +obligations under this License and any other pertinent obligations, +then as a consequence you may not distribute the Program at all. For +example, if a patent license would not permit royalty-free +redistribution of the Program by all those who receive copies directly +or indirectly through you, then the only way you could satisfy both it +and this License would be to refrain entirely from distribution of the +Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + +**8.** If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + +**9.** The Free Software Foundation may publish revised and/or new +versions of the General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Program does not specify a +version number of this License, you may choose any version ever +published by the Free Software Foundation. + +**10.** If you wish to incorporate parts of the Program into other +free programs whose distribution conditions are different, write to +the author to ask for permission. For software which is copyrighted by +the Free Software Foundation, write to the Free Software Foundation; +we sometimes make exceptions for this. Our decision will be guided by +the two goals of preserving the free status of all derivatives of our +free software and of promoting the sharing and reuse of software +generally. + +**NO WARRANTY** + +**11.** BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +**12.** IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + +### END OF TERMS AND CONDITIONS + +### How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these +terms. + +To do so, attach the following notices to the program. It is safest to +attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + one line to give the program's name and an idea of what it does. + Copyright (C) yyyy name of author + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +Also add information on how to contact you by electronic and paper +mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details + type `show w'. This is free software, and you are welcome + to redistribute it under certain conditions; type `show c' + for details. + +The hypothetical commands \`show w' and \`show c' should show the +appropriate parts of the General Public License. Of course, the +commands you use may be called something other than \`show w' and +\`show c'; they could even be mouse-clicks or menu items--whatever +suits your program. + +You should also get your employer (if you work as a programmer) or +your school, if any, to sign a "copyright disclaimer" for the program, +if necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright + interest in the program `Gnomovision' + (which makes passes at compilers) written + by James Hacker. + + signature of Ty Coon, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, +you may consider it more useful to permit linking proprietary +applications with the library. If this is what you want to do, use the +[GNU Lesser General Public +License](https://www.gnu.org/licenses/lgpl.html) instead of this +License. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..35c74867 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ + +all-tests: all-images test-2.7 test-3.5 test-3.6 test-3.7 test-3.8 + +all-images: image-2.7 image-3.5 image-3.6 image-3.7 image-3.8 + +image-%: + docker build -t rsconnect-python:$* --build-arg BASE_IMAGE=python:$* . + +test-%: + docker run -it --rm \ + -v $(PWD):/rsconnect \ + -w /rsconnect \ + rsconnect-python:$* \ + bash -c 'python setup.py install && python -m unittest discover' diff --git a/README.md b/README.md new file mode 100644 index 00000000..cc45136c --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# rsconnect-python + +This package is a library used by the rsconnect-jupyter package to deploy Jupyter notebooks to RStudio Connect. It can also be used by other Python-based deployment tools. + +There is also a CLI deployment tool which can be used directly to deploy notebooks. + +``` +rsconnect deploy \ + --api-key my-api-key \ + --server https://my.connect.server:3939 \ + ./my-notebook.ipynb +``` diff --git a/rsconnect/__init__.py b/rsconnect/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rsconnect/api.py b/rsconnect/api.py new file mode 100644 index 00000000..33cb10c0 --- /dev/null +++ b/rsconnect/api.py @@ -0,0 +1,352 @@ +import json +import logging +import re +import socket +import time +import ssl + +from six.moves import http_client as http +from six.moves.urllib_parse import urlparse, urlencode, urljoin +from six.moves.http_cookies import SimpleCookie + + +class RSConnectException(Exception): + def __init__(self, message): + super(RSConnectException, self).__init__(message) + self.message = message + + +logger = logging.getLogger('rsconnect') +logger.setLevel(logging.INFO) + + +def url_path_join(*parts): + joined = '/'.join(parts) + return re.sub('/+', '/', joined) + + +def wait_until(predicate, timeout, period=0.1): + """ + Run every seconds until it returns True or until + seconds have passed. + + Returns True if returns True before elapses, False + otherwise. + """ + ending = time.time() + timeout + while time.time() < ending: + if predicate(): + return True + time.sleep(period) + return False + + +settings_path = '__api__/server_settings' +max_redirects = 5 + +def https_helper(hostname, port, disable_tls_check, cadata): + if cadata is not None and disable_tls_check: + raise Exception("Cannot both disable TLS checking and provide a custom certificate") + if cadata is not None: + return http.HTTPSConnection(hostname, port=(port or http.HTTPS_PORT), timeout=10, + context=ssl.create_default_context(cadata=cadata)) + elif disable_tls_check: + return http.HTTPSConnection(hostname, port=(port or http.HTTPS_PORT), timeout=10, + context=ssl._create_unverified_context()) + else: + return http.HTTPSConnection(hostname, port=(port or http.HTTPS_PORT), timeout=10) + +def verify_server(server_address, disable_tls_check, cadata): + server_url = urljoin(server_address, settings_path) + return _verify_server(server_url, max_redirects, disable_tls_check, cadata) + +def _verify_server(server_address, max_redirects, disable_tls_check, cadata): + """ + Verifies that a server is present at the given address. + Assumes that `__api__/server_settings` is accessible from the jupyter server. + :returns address + :raises Base Exception with string error, or errors from HTTP(S)Connection + """ + r = urlparse(server_address) + conn = None + try: + if r.scheme == 'http': + conn = http.HTTPConnection(r.hostname, port=(r.port or http.HTTP_PORT), timeout=10) + else: + conn = https_helper(r.hostname, r.port, disable_tls_check, cadata) + + conn.request('GET', server_address) + response = conn.getresponse() + + if response.status >= 400: + err = 'Response from Connect server: %s %s' % (response.status, response.reason) + logger.error(err) + raise Exception(err) + elif response.status >= 300: + # process redirects now so we don't have to later + target = response.getheader('Location') + logger.warning('Redirected to: %s' % target) + + if max_redirects > 0: + return _verify_server(urljoin(server_address, target), max_redirects - 1) + else: + err = 'Too many redirects' + logger.error(err) + raise Exception(err) + else: + content_type = response.getheader('Content-Type') + if not content_type.startswith('application/json'): + err = 'Unexpected Content-Type %s from %s' % (content_type, server_address) + logger.error(err) + raise Exception(err) + + except (http.HTTPException, OSError, socket.error) as exc: + logger.error('Error connecting to Connect: %s' % str(exc)) + raise exc + finally: + if conn is not None: + conn.close() + + if server_address.endswith(settings_path): + return server_address[:-len(settings_path)] + else: + return server_address + + +class RSConnect: + def __init__(self, uri, api_key, cookies=[], disable_tls_check=False, cadata=None): + if disable_tls_check and (cadata is not None): + raise Exception("Cannot both disable TLS checking and provide custom certificate data") + self.path_prefix = uri.path or '/' + self.api_key = api_key + self.conn = None + self.mk_conn = lambda: http.HTTPConnection(uri.hostname, port=uri.port, timeout=10) + if uri.scheme == 'https': + self.mk_conn = lambda: https_helper(uri.hostname, uri.port, disable_tls_check, cadata) + self.http_headers = { + 'Authorization': 'Key %s' % self.api_key, + } + self.cookies = cookies + self._inject_cookies(cookies) + + def __enter__(self): + self.conn = self.mk_conn() + return self + + def __exit__(self, *args): + self.conn.close() + self.conn = None + + def request(self, method, path, *args, **kwargs): + request_path = url_path_join(self.path_prefix, path) + logger.debug('Performing: %s %s' % (method, request_path)) + try: + self.conn.request(method, request_path, *args, **kwargs) + return self.json_response() + except http.HTTPException as e: + logger.error('HTTPException: %s' % e) + raise RSConnectException(str(e)) + except (IOError, OSError) as e: + logger.error('IO/OS Error: %s' % e) + raise RSConnectException(str(e)) + except (socket.error, socket.herror, socket.gaierror) as e: + logger.error('Socket Error: %s' % e) + raise RSConnectException(str(e)) + except socket.timeout: + logger.error('Socket Timeout') + raise RSConnectException('Connection timed out') + + def _handle_set_cookie(self, response): + headers = filter(lambda h: h[0].lower() == 'set-cookie', response.getheaders()) + values = [] + + for header in headers: + cookie = SimpleCookie(header[1]) + for morsel in cookie.values(): + values.append((dict(key=morsel.key, value=morsel.value))) + + self.cookies = values + self._inject_cookies(values) + + def _inject_cookies(self, cookies): + if cookies: + self.http_headers['Cookie'] = '; '.join(['%s="%s"' % (kv['key'], kv['value']) for kv in cookies]) + elif 'Cookie' in self.http_headers: + del self.http_headers['Cookie'] + + def json_response(self): + response = self.conn.getresponse() + + self._handle_set_cookie(response) + raw = response.read().decode('utf-8') + + if response.status >= 500: + logger.error('Received HTTP 500: %s', raw) + try: + message = json.loads(raw)['error'] + except: + message = 'Unexpected response code: %d' % (response.status) + raise RSConnectException(message) + elif response.status >= 400: + data = json.loads(raw) + raise RSConnectException(data['error']) + else: + data = json.loads(raw) + return data + + def me(self): + return self.request('GET', '__api__/me', None, self.http_headers) + + def app_find(self, filters): + params = urlencode(filters) + data = self.request('GET', '__api__/applications?' + params, None, self.http_headers) + if data['count'] > 0: + return data['applications'] + + def app_create(self, name): + params = json.dumps({'name': name}) + return self.request('POST', '__api__/applications', params, self.http_headers) + + def app_get(self, app_id): + return self.request('GET', '__api__/applications/%d' % app_id, None, self.http_headers) + + def app_upload(self, app_id, tarball): + return self.request('POST', '__api__/applications/%d/upload' % app_id, tarball, self.http_headers) + + def app_update(self, app_id, updates): + params = json.dumps(updates) + return self.request('POST', '__api__/applications/%d' % app_id, params, self.http_headers) + + def app_deploy(self, app_id, bundle_id = None): + params = json.dumps({'bundle': bundle_id}) + return self.request('POST', '__api__/applications/%d/deploy' % app_id, params, self.http_headers) + + def app_publish(self, app_id, access): + params = json.dumps({ + 'access_type': access, + 'id': app_id, + 'needs_config': False + }) + return self.request('POST', '__api__/applications/%d' % app_id, params, self.http_headers) + + def app_config(self, app_id): + return self.request('GET', '__api__/applications/%d/config' % app_id, None, self.http_headers) + + def task_get(self, task_id, first_status=None): + url = '__api__/tasks/%s' % task_id + if first_status is not None: + url += '?first_status=%d' % first_status + return self.request('GET', url, None, self.http_headers) + + +def wait_for_task(api, task_id, timeout, period=1.0): + last_status = None + ending = time.time() + timeout + + while time.time() < ending: + task_status = api.task_get(task_id, first_status=last_status) + + if task_status['last_status'] != last_status: + # we've gotten an updated status, reset timer + logger.info('Deployment status: %s', task_status['status']) + ending = time.time() + timeout + last_status = task_status['last_status'] + + if task_status['finished']: + return task_status + + time.sleep(period) + return None + + +def deploy(uri, api_key, app_id, app_name, app_title, tarball, disable_tls_check, cadata): + with RSConnect(uri, api_key, disable_tls_check=disable_tls_check, cadata=cadata) as api: + if app_id is None: + # create an app if id is not provided + app = api.app_create(app_name) + else: + # assume app exists. if it was deleted then Connect will + # raise an error + app = api.app_get(app_id) + + if app['title'] != app_title: + api.app_update(app['id'], {'title': app_title}) + + app_bundle = api.app_upload(app['id'], tarball) + task_id = api.app_deploy(app['id'], app_bundle['id'])['id'] + + return { + 'task_id': task_id, + 'app_id': app['id'], + 'cookies': api.cookies, + } + + +def task_get(uri, api_key, task_id, last_status, cookies, disable_tls_check, cadata): + with RSConnect(uri, api_key, cookies, disable_tls_check=disable_tls_check, cadata=cadata) as api: + return api.task_get(task_id, first_status=last_status) + + +def app_config(uri, api_key, app_id, disable_tls_check, cadata): + with RSConnect(uri, api_key, disable_tls_check=disable_tls_check, cadata=cadata) as api: + return api.app_config(app_id) + + +def verify_api_key(uri, api_key, disable_tls_check, cadata): + with RSConnect(uri, api_key, disable_tls_check=disable_tls_check, cadata=cadata) as api: + try: + api.me() + return True + except RSConnectException: + return False + +APP_MODE_STATIC = 4 +APP_MODE_JUPYTER_STATIC = 7 + +app_modes = { + APP_MODE_STATIC: 'static', + APP_MODE_JUPYTER_STATIC: 'jupyter-static', +} + + +def app_search(uri, api_key, app_title, app_id, disable_tls_check, cadata): + with RSConnect(uri, api_key, disable_tls_check=disable_tls_check, cadata=cadata) as api: + data = [] + + filters = [('count', 5), + ('filter', 'min_role:editor'), + ('search', app_title)] + + apps = api.app_find(filters) + found = False + + def app_data(app): + return { + 'id': app['id'], + 'name': app['name'], + 'title': app['title'], + 'app_mode': app_modes.get(app['app_mode']), + 'config_url': api.app_config(app['id'])['config_url'], + } + + for app in apps or []: + if app['app_mode'] in (APP_MODE_STATIC, APP_MODE_JUPYTER_STATIC): + data.append(app_data(app)) + if app['id'] == app_id: + found = True + + if app_id and not found: + try: + # offer the current location as an option + app = api.app_get(app_id) + if app['app_mode'] in (APP_MODE_STATIC, APP_MODE_JUPYTER_STATIC): + data.append(app_data(app)) + except RSConnectException: + logger.exception('Error getting info for previous app_id "%s", skipping', app_id) + + return data + + +def app_get(uri, api_key, app_id, disable_tls_check, cadata): + with RSConnect(uri, api_key, disable_tls_check=disable_tls_check, cadata=cadata) as api: + return api.app_get(app_id) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py new file mode 100644 index 00000000..e95d0ab4 --- /dev/null +++ b/rsconnect/bundle.py @@ -0,0 +1,205 @@ + +import hashlib +import io +import json +import logging +import os +import posixpath +import tarfile +import tempfile + +from os.path import dirname, exists, join, relpath, split, splitext + +import nbformat +from ipython_genutils import text + +log = logging.getLogger('rsconnect') +log.setLevel(logging.DEBUG) + + +def make_source_manifest(entrypoint, environment, appmode): + package_manager = environment['package_manager'] + + manifest = { + "version": 1, + "metadata": { + "appmode": appmode, + "entrypoint": entrypoint + }, + "locale": environment['locale'], + "python": { + "version": environment['python'], + "package_manager": { + "name": package_manager, + "version": environment[package_manager], + "package_file": environment['filename'] + } + }, + "files": {} + } + return manifest + + +def manifest_add_file(manifest, rel_path, base_dir): + """Add the specified file to the manifest files section + + The file must be specified as a pathname relative to the notebook directory. + """ + path = join(base_dir, rel_path) + + manifest['files'][rel_path] = { + 'checksum': file_checksum(path) + } + + +def manifest_add_buffer(manifest, filename, buf): + """Add the specified in-memory buffer to the manifest files section""" + manifest['files'][filename] = { + 'checksum': buffer_checksum(buf) + } + + +def file_checksum(path): + """Calculate the md5 hex digest of the specified file""" + with open(path, 'rb') as f: + m = hashlib.md5() + chunk_size = 64 * 1024 + + chunk = f.read(chunk_size) + while chunk: + m.update(chunk) + chunk = f.read(chunk_size) + return m.hexdigest() + + +def buffer_checksum(buf): + """Calculate the md5 hex digest of a buffer (str or bytes)""" + m = hashlib.md5() + m.update(to_bytes(buf)) + return m.hexdigest() + + +def to_bytes(s): + if hasattr(s, 'encode'): + return s.encode('utf-8') + return s + + +def bundle_add_file(bundle, rel_path, base_dir): + """Add the specified file to the tarball. + + The file path is relative to the notebook directory. + """ + path = join(base_dir, rel_path) + bundle.add(path, arcname=rel_path) + log.debug('added file: %s', path) + + +def bundle_add_buffer(bundle, filename, contents): + """Add an in-memory buffer to the tarball. + + `contents` may be a string or bytes object + """ + buf = io.BytesIO(to_bytes(contents)) + fileinfo = tarfile.TarInfo(filename) + fileinfo.size = len(buf.getvalue()) + bundle.addfile(fileinfo, buf) + log.debug('added buffer: %s', filename) + + +def write_manifest(relative_dir, nb_name, environment, output_dir): + """Create a manifest for source publishing the specified notebook. + + The manifest will be written to `manifest.json` in the output directory.. + A requirements.txt file will be created if one does not exist. + + Returns the list of filenames written. + """ + manifest_filename = 'manifest.json' + manifest = make_source_manifest(nb_name, environment, 'jupyter-static') + manifest_file = join(output_dir, manifest_filename) + created = [] + skipped = [] + + manifest_relpath = join(relative_dir, manifest_filename) + if exists(manifest_file): + skipped.append(manifest_relpath) + else: + with open(manifest_file, 'w') as f: + f.write(json.dumps(manifest, indent=2)) + created.append(manifest_relpath) + log.debug('wrote manifest file: %s', manifest_file) + + environment_filename = environment['filename'] + environment_file = join(output_dir, environment_filename) + environment_relpath = join(relative_dir, environment_filename) + if exists(environment_file): + skipped.append(environment_relpath) + else: + with open(environment_file, 'w') as f: + f.write(environment['contents']) + created.append(environment_relpath) + log.debug('wrote environment file: %s', environment_file) + + return created, skipped + + +def list_files(base_dir, include_subdirs, walk=os.walk): + """List the files in the directory at path. + + If include_subdirs is True, recursively list + files in subdirectories. + + Returns an iterable of file paths relative to base_dir. + """ + skip_dirs = ['.ipynb_checkpoints', '.git'] + + def iter_files(): + for root, subdirs, files in walk(base_dir): + if include_subdirs: + for skip in skip_dirs: + if skip in subdirs: + subdirs.remove(skip) + else: + # tell walk not to traverse any subdirs + subdirs[:] = [] + + for filename in files: + yield relpath(join(root, filename), base_dir) + return list(iter_files()) + + +def make_source_bundle(model, environment, ext_resources_dir, extra_files=[]): + """Create a bundle containing the specified notebook and python environment. + + Returns a file-like object containing the bundle tarball. + """ + nb_name = model['name'] + nb_content = nbformat.writes(model['content'], nbformat.NO_CONVERT) + '\n' + + manifest = make_source_manifest(nb_name, environment, 'jupyter-static') + manifest_add_buffer(manifest, nb_name, nb_content) + manifest_add_buffer(manifest, environment['filename'], environment['contents']) + + if extra_files: + skip = [nb_name, environment['filename'], 'manifest.json'] + extra_files = sorted(list(set(extra_files) - set(skip))) + + for rel_path in extra_files: + manifest_add_file(manifest, rel_path, ext_resources_dir) + + log.debug('manifest: %r', manifest) + + bundle_file = tempfile.TemporaryFile(prefix='rsc_bundle') + with tarfile.open(mode='w:gz', fileobj=bundle_file) as bundle: + + # add the manifest first in case we want to partially untar the bundle for inspection + bundle_add_buffer(bundle, 'manifest.json', json.dumps(manifest, indent=2)) + bundle_add_buffer(bundle, nb_name, nb_content) + bundle_add_buffer(bundle, environment['filename'], environment['contents']) + + for rel_path in extra_files: + bundle_add_file(bundle, rel_path, ext_resources_dir) + + bundle_file.seek(0) + return bundle_file diff --git a/rsconnect/environment.py b/rsconnect/environment.py new file mode 100644 index 00000000..24a3a759 --- /dev/null +++ b/rsconnect/environment.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +import json +import locale +import os +import re +import subprocess +import sys + + +version_re = re.compile(r'\d+\.\d+(\.\d+)?') +exec_dir = os.path.dirname(sys.executable) + + +class EnvironmentException(Exception): + pass + + +def detect_environment(dirname): + """Determine the python dependencies in the environment. + + `pip freeze` will be used to introspect the environment. + + Returns a dictionary containing the package spec filename + and contents if successful, or a dictionary containing 'error' + on failure. + """ + result = (output_file(dirname, 'requirements.txt', 'pip') or + pip_freeze(dirname)) + + if result is not None: + result['python'] = get_python_version() + result['pip'] = get_version('pip') + result['locale'] = get_default_locale() + + return result + + +def get_python_version(): + v = sys.version_info + return "%d.%d.%d" % (v[0], v[1], v[2]) + + +def get_default_locale(): + return '.'.join(locale.getdefaultlocale()) + + +def get_version(module): + try: + args = [sys.executable, '-m', module, '--version'] + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = proc.communicate() + match = version_re.search(stdout) + if match: + return match.group() + + msg = "Failed to get version of '%s' from the output of: %s" % (module, ' '.join(args)) + raise EnvironmentException(msg) + except Exception as exc: + raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exc))) + + +def output_file(dirname, filename, package_manager): + """Read an existing package spec file. + + Returns a dictionary containing the filename and contents + if successful, None if the file does not exist, + or a dictionary containing 'error' on failure. + """ + try: + path = os.path.join(dirname, filename) + if not os.path.exists(path): + return None + + with open(path, 'r') as f: + data = f.read() + + data = '\n'.join([line for line in data.split('\n') + if 'rsconnect' not in line]) + + return { + 'filename': filename, + 'contents': data, + 'source': 'file', + 'package_manager': package_manager, + } + except Exception as exc: + raise EnvironmentException('Error reading %s: %s' % (filename, str(exc))) + + +def pip_freeze(dirname): + """Inspect the environment using `pip freeze`. + + Returns a dictionary containing the filename + (always 'requirements.txt') and contents if successful, + or a dictionary containing 'error' on failure. + """ + try: + proc = subprocess.Popen( + [sys.executable, '-m', 'pip', 'freeze'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + pip_stdout, pip_stderr = proc.communicate() + pip_status = proc.returncode + except Exception as exc: + raise EnvironmentException('Error during pip freeze: %s' % str(exc)) + + if pip_status != 0: + msg = pip_stderr or ('exited with code %d' % pip_status) + raise EnvironmentException('Error during pip freeze: %s' % msg) + + pip_stdout = '\n'.join([line for line in pip_stdout.split('\n') + if 'rsconnect' not in line]) + + return { + 'filename': 'requirements.txt', + 'contents': pip_stdout, + 'source': 'pip_freeze', + 'package_manager': 'pip', + } + + +if __name__ == '__main__': + try: + if len(sys.argv) < 2: + raise EnvironmentException('Usage: %s DIRECTORY' % sys.argv[0]) + + result = detect_environment(sys.argv[1]) + except EnvironmentException as exc: + result = dict(error=str(exc)) + + json.dump(result, sys.stdout, indent=4) diff --git a/rsconnect/tests/__init__.py b/rsconnect/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rsconnect/tests/data/pip1/dummy.ipynb b/rsconnect/tests/data/pip1/dummy.ipynb new file mode 100644 index 00000000..76fe3342 --- /dev/null +++ b/rsconnect/tests/data/pip1/dummy.ipynb @@ -0,0 +1,52 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'this is a notebook'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"this is a notebook\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/rsconnect/tests/data/pip1/requirements.txt b/rsconnect/tests/data/pip1/requirements.txt new file mode 100644 index 00000000..40218701 --- /dev/null +++ b/rsconnect/tests/data/pip1/requirements.txt @@ -0,0 +1,3 @@ +numpy +pandas +matplotlib diff --git a/rsconnect/tests/data/pip2/data.csv b/rsconnect/tests/data/pip2/data.csv new file mode 100644 index 00000000..2763b7f8 --- /dev/null +++ b/rsconnect/tests/data/pip2/data.csv @@ -0,0 +1,9 @@ +Label,Value +black,0 +blue,1 +green,2 +cyan,3 +red,4 +magenta,5 +yellow,6 +white,7 diff --git a/rsconnect/tests/data/pip2/dummy.ipynb b/rsconnect/tests/data/pip2/dummy.ipynb new file mode 100644 index 00000000..76fe3342 --- /dev/null +++ b/rsconnect/tests/data/pip2/dummy.ipynb @@ -0,0 +1,52 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'this is a notebook'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"this is a notebook\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/rsconnect/tests/test_bundle.py b/rsconnect/tests/test_bundle.py new file mode 100644 index 00000000..1532afa7 --- /dev/null +++ b/rsconnect/tests/test_bundle.py @@ -0,0 +1,179 @@ + +import json +import logging +import os +import sys +import tarfile + +from datetime import datetime +from unittest import TestCase +from os.path import basename, dirname, exists, join + +import nbformat + +from rsconnect.environment import detect_environment +from rsconnect.bundle import list_files, make_source_bundle + + +class TestBundle(TestCase): + def get_dir(self, name): + path = join(dirname(__file__), 'data', name) + self.assertTrue(exists(path)) + return path + + def python_version(self): + return u'.'.join(map(str, sys.version_info[:3])) + + def read_notebook(self, nb_path): + return { + 'name': basename(nb_path), + 'last_modified': datetime.fromtimestamp(os.stat(nb_path).st_mtime), + 'content': nbformat.read(nb_path, nbformat.NO_CONVERT), + } + + def test_source_bundle1(self): + self.maxDiff = 5000 + dir = self.get_dir('pip1') + nb_path = join(dir, 'dummy.ipynb') + + # Note that here we are introspecting the environment from within + # the test environment. Don't do this in the production code, which + # runs in the notebook server. We need the introspection to run in + # the kernel environment and not the notebook server environment. + environment = detect_environment(dir) + notebook = self.read_notebook(nb_path) + with make_source_bundle(notebook, environment, dir) as bundle, \ + tarfile.open(mode='r:gz', fileobj=bundle) as tar: + + names = sorted(tar.getnames()) + self.assertEqual(names, [ + 'dummy.ipynb', + 'manifest.json', + 'requirements.txt', + ]) + + reqs = tar.extractfile('requirements.txt').read() + self.assertEqual(reqs, b'numpy\npandas\nmatplotlib\n') + + manifest = json.loads(tar.extractfile('manifest.json').read().decode('utf-8')) + + # don't check locale value, just require it be present + del manifest['locale'] + del manifest['python']['package_manager']['version'] + + self.assertEqual(manifest, { + u"version": 1, + u"metadata": { + u"appmode": u"jupyter-static", + u"entrypoint": u"dummy.ipynb" + }, + u"python": { + u"version": self.python_version(), + u"package_manager": { + u"name": u"pip", + u"package_file": u"requirements.txt" + } + }, + u"files": { + u"dummy.ipynb": { + u"checksum": u"36873800b48ca5ab54760d60ba06703a" + }, + u"requirements.txt": { + u"checksum": u"5f2a5e862fe7afe3def4a57bb5cfb214" + } + } + }) + + def test_source_bundle2(self): + self.maxDiff = 5000 + dir = self.get_dir('pip2') + nb_path = join(dir, 'dummy.ipynb') + + # Note that here we are introspecting the environment from within + # the test environment. Don't do this in the production code, which + # runs in the notebook server. We need the introspection to run in + # the kernel environment and not the notebook server environment. + environment = detect_environment(dir) + notebook = self.read_notebook(nb_path) + + with make_source_bundle(notebook, environment, dir, extra_files=['data.csv']) as bundle, \ + tarfile.open(mode='r:gz', fileobj=bundle) as tar: + + names = sorted(tar.getnames()) + self.assertEqual(names, [ + 'data.csv', + 'dummy.ipynb', + 'manifest.json', + 'requirements.txt', + ]) + + reqs = tar.extractfile('requirements.txt').read() + + # these are the dependencies declared in our setup.py + self.assertIn(b'six', reqs) + + manifest = json.loads(tar.extractfile('manifest.json').read().decode('utf-8')) + + # don't check requirements.txt since we don't know the checksum + del manifest['files']['requirements.txt'] + + # also don't check locale value, just require it be present + del manifest['locale'] + del manifest['python']['package_manager']['version'] + + self.assertEqual(manifest, { + u"version": 1, + u"metadata": { + u"appmode": u"jupyter-static", + u"entrypoint": u"dummy.ipynb" + }, + u"python": { + u"version": self.python_version(), + u"package_manager": { + u"name": u"pip", + u"package_file": u"requirements.txt" + } + }, + u"files": { + u"dummy.ipynb": { + u"checksum": u"36873800b48ca5ab54760d60ba06703a" + }, + u"data.csv": { + u"checksum": u"f2bd77cc2752b3efbb732b761d2aa3c3" + } + } + }) + + def test_list_files(self): + paths = [ + 'notebook.ipynb', + 'somedata.csv', + 'subdir/subfile', + 'subdir2/subfile2', + '.ipynb_checkpoints/notebook.ipynb', + '.git/config', + ] + + def walk(base_dir): + dirnames = [] + filenames = [] + + for path in paths: + if '/' in path: + dirname, filename = path.split('/', 1) + dirnames.append(dirname) + else: + filenames.append(path) + + yield (base_dir, dirnames, filenames) + + for subdir in dirnames: + for path in paths: + if path.startswith(subdir + '/'): + yield (base_dir + '/' + subdir, [], [path.split('/', 1)[1]]) + + files = list_files('/', True, walk=walk) + self.assertEqual(files, paths[:4]) + + files = list_files('/', False, walk=walk) + self.assertEqual(files, paths[:2]) diff --git a/rsconnect/tests/test_environment.py b/rsconnect/tests/test_environment.py new file mode 100644 index 00000000..758003fb --- /dev/null +++ b/rsconnect/tests/test_environment.py @@ -0,0 +1,58 @@ +import re +import sys + +from unittest import TestCase +from os.path import dirname, exists, join + +from rsconnect.environment import detect_environment + +version_re = re.compile(r'\d+\.\d+(\.\d+)?') + +class TestEnvironment(TestCase): + def get_dir(self, name): + path = join(dirname(__file__), 'data', name) + self.assertTrue(exists(path)) + return path + + def python_version(self): + return '.'.join(map(str, sys.version_info[:3])) + + def test_file(self): + result = detect_environment(self.get_dir('pip1')) + + pip_version = result.pop('pip') + self.assertTrue(version_re.match(pip_version)) + + locale = result.pop('locale') + self.assertIsInstance(locale, str) + self.assertIn('.', locale) + + self.assertEqual(result, { + 'package_manager': 'pip', + 'source': 'file', + 'filename': 'requirements.txt', + 'contents': 'numpy\npandas\nmatplotlib\n', + 'python': self.python_version(), + }) + + def test_pip_freeze(self): + result = detect_environment(self.get_dir('pip2')) + contents = result.pop('contents') + + # these are the dependencies declared in our setup.py + self.assertIn('six', contents) + self.assertIn('nbformat', contents) + + pip_version = result.pop('pip') + self.assertTrue(version_re.match(pip_version)) + + locale = result.pop('locale') + self.assertIsInstance(locale, str) + self.assertIn('.', locale) + + self.assertEqual(result, { + 'package_manager': 'pip', + 'source': 'pip_freeze', + 'filename': 'requirements.txt', + 'python': self.python_version(), + }) diff --git a/rsconnect/version.txt b/rsconnect/version.txt new file mode 100644 index 00000000..6e8bf73a --- /dev/null +++ b/rsconnect/version.txt @@ -0,0 +1 @@ +0.1.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..a7bdf864 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +universal = 1 + +[metadata] +license_file = LICENSE.md diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..a1c0f00b --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +from setuptools import setup +import os +import sys + + +def readme(): + with open('README.md') as f: + return f.read() + +print('setup.py using python', sys.version_info[0]) + +with open('rsconnect/version.txt', 'r') as f: + VERSION = f.read().strip() + +BUILD = os.environ.get('BUILD_NUMBER', '9999') + +setup(name='rsconnect_python', + version='{version}.{build}'.format(version=VERSION, build=BUILD), + description='Python integration with RStudio Connect', + long_description=readme(), + long_description_content_type='text/markdown', + url='http://github.com/rstudio/rsconnect-python', + project_urls={ + "Documentation": "https://docs.rstudio.com/rsconnect-python", + }, + author='Michael Marchetti', + author_email='mike@rstudio.com', + license='GPL-2.0', + packages=['rsconnect'], + install_requires=[ + 'six', + 'click', + 'nbformat', + ], + python_requires = '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*' +)