diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..ee6453f --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,7 @@ +[bumpversion] +current_version = 1.0.0 +commit = True +tag = True + +[bumpversion:file:setup.py] + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2be9c5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.pyc +.idea/ +build/ +dist/ +*.swp +venv/ +target/ +.coverage +*egg-info/ +.tox/ +.cache/ +.DS_Store diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e012529 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +language: python +python: 3.6 + +install: pip install tox wheel + +env: + - TOX_ENV=py36 + - TOX_ENV=pep8 + +script: + - tox -e $TOX_ENV + +before_deploy: + - bin/ihrpi-build + +deploy: + provider: releases + api_key: + secure: "UCRWPTHQfgDKIaYfOUrxPOOXqa0Qs7UKYOYWj4RhrU+0wn70AHL9oT01wEj25es+QN1u/q6K95vgyLLEEG/C+RDduY0nzDhRrwjfPP4ofBxCzJl19fiZ2AZVz21OWaScVZViPSOV0SR0F5qewe1WbOlwd6GGjblJuZtJiOZz0aPsoa4XP3LPsQ7D1OCU9Z0sKpp+V+1sq3vxP42SL1/jxc78ouXhCMP7GmlfeE2U8f+gbGvGr0soLvhQi2lFfPpdA1S5M6hx4AnjjfTp1zkrQ90+EuamQFeQ9SZbwTgzfT1qv88AUh9gddzLoX8pFIK7vbiOmUpllU1Ao5+ldSCz/4JfQ4UvZ56kDxrk560p7WJDT2YZL3CrJWNlUaFjXkiAmk3kl6f8m6XhFn3ZuHBH0ygXeXebpUdCAisalpawyweWE7t+hIK6zN60oqt96BOaYruTJpZP1Tmgi//Lp9Zd6QBdnjhlxthuUYvTKm/d03r8KdPo8dF2iVocjypV4D9c38REc7K7NffOEIT+tpsDUlt/aCEar9eF0EKzHIiTFEgenrqZPuxwIcKdBCmpSIuJV9x/OqCSUFXjSVaOSPBSAaGSBNHp2fIDWy1RUAH6zLIYBXfZ1C63cmF0S2mc5Mrp5aDzijHHlh/uCpQOu3P27oP3zTzzZvaBGiePn4TNWpk=" + skip_cleanup: true + file_glob: true + file: + - dist/ihrpi-*.whl + on: + tags: true + branch: master + repo: iheartradio/ihrpi diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..cc544cc --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include bin +include README.md + +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d48f19e --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# ihrpi 🍒🍏 +> iHeart private packaging tools & index + +[![Build Status](https://travis-ci.com/iheartradio/ihrpi.svg?token=DRZy5ZWDwtAusJJ7MRfF&branch=master)](https://travis-ci.com/iheartradio/ihrpi) + +## Setup for building and installing private packages + +1. `git clone git@github.com:iheartradio/ihrpi.git` +2. `cd ihrpi/ && python setup.py install` this will install ihrpi and setup cli + tools under [bin](https://github.com/iheartradio/ihrpi/tree/master/bin) +3. Assemble values for `IHRPI_HOST`, `IHRPI_USER`, and `IHRPI_PASS` +4. `ihrpi-setup-env` to setup pip.conf with `IHRPI_*` values from previous step +5. `pip install ihrpi[tools]` if you plan to use the cli tools + +## Configuration + +Basic application file is provided. See [`create_app()`](https://github.com/iheartradio/ihrpi/blob/master/ihrpi/app.py) for setting s3 bucket +and prefix. + +## Tools + +### Releasing & Pushing packages to your ihrpi from your local machine + + # In your python project of choice: + + cd my_project/ + ihrpi-release + ihrpi-publish + +### Installing packages from ihrpi + + pip install 'mypkg>=0.17.0' \ + --extra-index-url ${IHRPI_URL} \ + --trusted-host ${IHRPI_HOST} + +### Access ihrpi from travis build + +Travis can be configured to access the private package index. Be careful to not +leak the credentials for ihrpi in travis build logs. + +#### Setup + + cd my_project/ + ihrpi-configure-travis + + # tox.ini sections should have: + install_command = + ihrpi-tox-install {opts} {packages} + + # .travis.yml install section should include: + install: + - git clone git@github.com:iheartradio/ihrpi.git && cd ihrpi/ && python setup.py install && cd .. + + # .travis.yml script section should be: + script: + - ihrpi-tox-run + +## FAQ + +Should I be careful when using tox and the `IHRPI_*` env variables on travis? + + Yes!! tox spews configuration parameters to stdout including the entire + environment when a run doesn't succeed. This is why `ihrpi-tox-run` and + `ihrpi-tox-install` scripts are used. + +## API + +ihrpi API is a flask app serving an s3 bucket formatted to the pypi +[simple specification](https://www.python.org/dev/peps/pep-0503/). We use +`pip2pi` to handle formatting when we run `ihrpi-publish`. + +### Debug + +This curl should return "pong" from each environment: + + curl "http://${IHRPI_USER}:${IHRPI_PASS}@${IHRPI_HOST}:8089/ping" diff --git a/bin/ihrpi-build b/bin/ihrpi-build new file mode 100755 index 0000000..912fa47 --- /dev/null +++ b/bin/ihrpi-build @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +python setup.py bdist_wheel +python setup.py sdist + +# files end up in dist/ dir diff --git a/bin/ihrpi-configure-travis b/bin/ihrpi-configure-travis new file mode 100755 index 0000000..d704317 --- /dev/null +++ b/bin/ihrpi-configure-travis @@ -0,0 +1,23 @@ +#!/bin/bash + +if ! [ -x "$(command -v travis)" ]; then + echo >&2 "Install travis before proceeding." + echo >&2 "https://github.com/travis-ci/travis.rb#installation" + exit 1 +fi + +echo "# IHRPI_USER" +u=$(travis encrypt IHRPI_USER=${IHRPI_USER}) +echo "secure: ${u}" + +echo "# IHRPI_PASS" +p=$(travis encrypt IHRPI_PASS=${IHRPI_PASS}) +echo "secure: ${p}" + +echo "# IHRPI_HOST" +h=$(travis encrypt IHRPI_HOST=${IHRPI_HOST}) +echo "secure: ${h}" + +echo "# IHRPI_URL" +url=$(travis encrypt IHRPI_URL="http://\${IHRPI_USER}:\${IHRPI_PASS}@\${IHRPI_HOST}:8089/simple") +echo "secure: ${url}" diff --git a/bin/ihrpi-gen-requirements b/bin/ihrpi-gen-requirements new file mode 100644 index 0000000..cc8deb4 --- /dev/null +++ b/bin/ihrpi-gen-requirements @@ -0,0 +1,6 @@ +#!/bin/bash + +# NOTE: we do not want ihrpi host or credentials to be saved in github so we're +# using no-index and no-emit-trusted-host options +pip-compile --no-index --no-emit-trusted-host \ + --output-file requirements.txt requirements.in diff --git a/bin/ihrpi-publish b/bin/ihrpi-publish new file mode 100755 index 0000000..dc00370 --- /dev/null +++ b/bin/ihrpi-publish @@ -0,0 +1,31 @@ +#!/bin/bash + +d="/tmp/packages" +rm -rf "${d}" +mkdir "${d}" + +rm -rf dist/* +python setup.py sdist --format=zip +python setup.py bdist_wheel + +# pull down included packages as well +if [ -d ".tox/py27" ]; then + source .tox/py27/bin/activate + pip freeze > reqs +elif [ -d ".tox/py3" ]; then + source .tox/py3/bin/activate + pip freeze > reqs +fi + +if [ -f "reqs" ]; then + deactivate + pip2tgz "${d}" -r reqs --extra-index-url ${IHRPI_URL} + rm -rf reqs +fi + +cp dist/*.zip /tmp/packages/ +cp dist/*.whl /tmp/packages/ + +dir2pi "${d}" +echo "Syncing ${d} with s3://${IHRPI_S3_BUCKET}/packages..." +aws s3 sync "${d}" s3://${IHRPI_S3_BUCKET}/packages diff --git a/bin/ihrpi-release b/bin/ihrpi-release new file mode 100755 index 0000000..76f7dfa --- /dev/null +++ b/bin/ihrpi-release @@ -0,0 +1,26 @@ +#!/bin/bash + +echo "Running tests..." +tox + +echo "Bump version - type [major, minor, or patch] and press ENTER." +read part +bumpversion ${part} +bump_status=$? +if [ "${bump_status}" -ne "0" ] ; then + echo "Bump version failed." + exit 1 +fi + +echo "Press ENTER to push tags to origin and changes to origin/master." +read + +current_version=$(ihrpi-gcv) +cv_status=$? +if [ "${cv_status}" -ne "0" ] ; then + echo "Get current version failed." + exit 2 +fi + +git push origin "${current_version}" +git push origin master diff --git a/bin/ihrpi-setup-env b/bin/ihrpi-setup-env new file mode 100755 index 0000000..d3c088b --- /dev/null +++ b/bin/ihrpi-setup-env @@ -0,0 +1,46 @@ +#!/bin/bash + +# TODO: support zsh and others + +echo "Consult your admin for the IHRPI_HOST, IHRPI_USER, IHRPI_PASS value." +echo "Input the IHRPI_HOST here and press ENTER." +read ihrpi_host +echo "Input the IHRPI_USER here and press ENTER." +read ihrpi_user +echo "Input the IHRPI_PASS here and press ENTER." +read ihrpi_pass +echo "Input the IHRPI_S3_BUCKET here and press ENTER." +read ihrpi_s3_bucket + +ihrpi_port=8089 +for pair in "IHRPI_USER=\"${ihrpi_user}\"" \ + "IHRPI_HOST=\"${ihrpi_host}\"" \ + "IHRPI_PASS=\"${ihrpi_pass}\"" \ + "IHRPI_URL=\"http://$\{IHRPI_USER\}:$\{IHRPI_PASS\}@$\{IHRPI_HOST\}:${ihrpi_port}/simple\"" \ + "IHRPI_S3_BUCKET=\"${ihrpi_s3_bucket}\"" + +do + echo "export ${pair}" >> ~/.bash_profile +done + +source ~/.bash_profile + +echo "Configuring pip.conf..." +echo + +pip_conf="~/.pip/pip.conf" + +if [ -f ${pip_conf} ]; then + echo "${pip_conf} already exists. Please make sure it contains the following:" + echo + echo "[global]" + echo "trusted-host = ${ihrpi_host}" + echo "extra-index-url = http://${IHRPI_USER}:${IHRPI_PASS}@${IHRPI_HOST}:${ihrpi_port}/simple" +else + echo "Updating ~/.pip/pip.conf" + mkdir -p ~/.pip + + echo "[global]" >> ${pip_conf} + echo "trusted-host = ${ihrpi_host}" >> ${pip_conf} + echo "extra-index-url = http://${IHRPI_USER}:${IHRPI_PASS}@${IHRPI_HOST}:${ihrpi_port}/simple" >> ${pip_conf} +fi diff --git a/bin/ihrpi-tox-install b/bin/ihrpi-tox-install new file mode 100644 index 0000000..60a2859 --- /dev/null +++ b/bin/ihrpi-tox-install @@ -0,0 +1,13 @@ +#!/bin/bash + +opts_and_packages="$@" + +install=$(pip install -q --retries 1 --trusted-host ${IHRPI_HOST} --extra-index-url ${IHRPI_URL} ${opts_and_packages}) +inst_status=$? + +echo "${install}" | \ + sed "s|${IHRPI_URL}|IHRPI_URL|g" | \ + sed "s|${IHRPI_HOST}|IHRPI_HOST|g" | \ + sed "s|${IHRPI_PASS}|IHRPI_PASS|g" + +exit ${inst_status} diff --git a/bin/ihrpi-tox-run b/bin/ihrpi-tox-run new file mode 100644 index 0000000..2e8179a --- /dev/null +++ b/bin/ihrpi-tox-run @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ -z "$TOX_ENV" ]; then + a_run=$(tox) +else + a_run=$(tox -e ${TOX_ENV}) +fi + +run_status=$? + +echo "${a_run}" | \ + sed "s|${IHRPI_URL}|IHRPI_URL|g" | \ + sed "s|${IHRPI_HOST}|IHRPI_HOST|g" | \ + sed "s|${IHRPI_PASS}|IHRPI_PASS|g" + +exit ${run_status} diff --git a/ci/requirements.txt b/ci/requirements.txt new file mode 100644 index 0000000..9e0047f --- /dev/null +++ b/ci/requirements.txt @@ -0,0 +1,4 @@ +boto3 +flask>= 0.10.1 +uwsgi +Flask-HTTPAuth diff --git a/ihrpi/__init__.py b/ihrpi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ihrpi/app.py b/ihrpi/app.py new file mode 100644 index 0000000..0a47689 --- /dev/null +++ b/ihrpi/app.py @@ -0,0 +1,13 @@ +#!/bin/python +from ihrpi.factory import create_app + +application = create_app() +print(application.url_map) + + +def main(): + application.run(host='0.0.0.0') + + +if __name__ == "__main__": + main() diff --git a/ihrpi/blueprints/__init__.py b/ihrpi/blueprints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ihrpi/blueprints/ihrpi.py b/ihrpi/blueprints/ihrpi.py new file mode 100644 index 0000000..49b1ff0 --- /dev/null +++ b/ihrpi/blueprints/ihrpi.py @@ -0,0 +1,149 @@ +import boto3 +import itertools +import logging +import re + +from flask_httpauth import HTTPBasicAuth +from werkzeug.security import check_password_hash +from botocore.client import Config +from botocore.exceptions import ClientError + +from os.path import join, basename +from flask import Blueprint, current_app, Response + +auth = HTTPBasicAuth() +bp = Blueprint('ihrpi', __name__) + +CHUNK_SIZE = 1024 + +HTML_LINK_PREFIX = '{key}
' +HTML_LINK_FILE = '{1}
' + + +@auth.get_password +def get_pw(username): + users = current_app.config['USERS'] + if username in users: + return users.get(username) + return None + + +@auth.verify_password +def verify_password(username, password): + users = current_app.config['USERS'] + if username in users: + return check_password_hash(users.get(username), password) + return False + + +def _get_client(): + return boto3.client('s3', config=Config(signature_version='s3v4')) + + +def _handle_client_error(e, path): + logging.error('Error calling s3_proxy', path, e) + if e.response['Error']['Code'] == 'NoSuchKey': + return Response(e.response['Error']['Message'], status=404) + else: + return Response(e.response['Error']['Message'], status=500) + + +def _generate(result): + """Support generator for s3 response. + + See: + https://github.com/boto/boto3/issues/426#issuecomment-184889230 + """ + for chunk in iter(lambda: result['Body'].read(CHUNK_SIZE), b''): + yield chunk + + +def get_bucket(): + return current_app.config['BUCKET'] + + +def get_prefix(): + return current_app.config['PREFIX'] + + +@bp.route('/ping') +@auth.login_required +def ping(): + return "pong" + + +def _match_suffix(s): + p = '^.*\.tar\.gz$|^.*\.zip$|^.*\.egg|^.*\.whl$' + return re.match(p, s) + + +def paginate(client, b, p, fmt): + paginator = client.get_paginator('list_objects') + p = p+"/" + s = set() + for result in paginator.paginate(Bucket=b, Prefix=p): + for key in result.get('Contents', []): + k = key.get('Key')[len(p):].split('/')[0] + if k not in s: + s.add(k) + yield fmt.format(key=k) + + +def paginate_files(client, b, prefix, p, fmt): + offset = len(p)+1 + link_offset = len(prefix)+1 + paginator = client.get_paginator('list_objects') + for result in paginator.paginate(Bucket=b, Prefix=p): + for key in result.get('Contents', []): + yield fmt.format(key.get('Key')[link_offset:], + key.get('Key')[offset:]) + + +@bp.route('/simple', strict_slashes=False) +@auth.login_required +def s3_list_bucket(): + b = get_bucket() + prefix = get_prefix() + s3_client = _get_client() + try: + logging.info("List bucket: %s", b) + return Response(paginate(s3_client, b, prefix, HTML_LINK_PREFIX), + mimetype='text/html') + except ClientError as e: + _handle_client_error(e, prefix) + + +def _peek(iterable): + try: + first = next(iterable) + except StopIteration: + return None + return itertools.chain([first], iterable) + + +@bp.route('/simple/') +@auth.login_required +def s3_proxy(path=None): + try: + b = get_bucket() + prefix = get_prefix() + p = join(prefix, path) + + logging.info("Getting bucket: %s - key: %s", b, p) + s3_client = _get_client() + is_match = _match_suffix(p) + if is_match: + s3_res = s3_client.get_object(Bucket=b, Key=p) + headers = {'Content-Disposition': + 'attachment;filename=' + basename(p)} + return Response(_generate(s3_res), mimetype='application/zip', + headers=headers) + else: + iter = _peek( + paginate_files(s3_client, b, prefix, p, HTML_LINK_FILE)) + if iter: + return Response(iter, mimetype='text/html') + else: + return Response('No path matches: %s' % path, status=404) + except ClientError as e: + _handle_client_error(e, path) diff --git a/ihrpi/factory.py b/ihrpi/factory.py new file mode 100644 index 0000000..97b0dbb --- /dev/null +++ b/ihrpi/factory.py @@ -0,0 +1,42 @@ +import configparser +import logging +import os + +from flask import Flask +from werkzeug.utils import find_modules, import_string + + +def _get_users(): + config = configparser.ConfigParser() + config.read(os.getenv('IHRPI_CONF')) + return { + config[s]['user']: config[s]['pass'] + for s in config.sections() + } + + +def create_app(config=None): + logging.basicConfig(level=logging.INFO) + + app = Flask('ihrpi') + + app.config.update(dict( + DEBUG=False, + BUCKET='some-s3-bucket', # override me + PREFIX='packages/simple', # override me + USERS=_get_users() + )) + app.config.update(config or {}) + app.config.from_envvar('IHRPI_SETTINGS', silent=True) + + register_blueprints(app) + return app + + +def register_blueprints(app): + """Register all blueprint modules.""" + for name in find_modules('ihrpi.blueprints'): + mod = import_string(name) + if hasattr(mod, 'bp'): + app.register_blueprint(mod.bp) + return None diff --git a/ihrpi/tools.py b/ihrpi/tools.py new file mode 100644 index 0000000..527f892 --- /dev/null +++ b/ihrpi/tools.py @@ -0,0 +1,21 @@ +try: + import configparser +except ImportError: + import ConfigParser as configparser + + +def get_current_version(): + config = configparser.ConfigParser() + config.read('.bumpversion.cfg') + cv = config.get('bumpversion', 'current_version') + if config.has_option('bumpversion', 'tag_name'): + return config.get('bumpversion', 'tag_name').format(new_version=cv) + else: + return "v"+cv + + +def gcv_main(): + """doing this because fn gets parsed like so: + https://packaging.python.org/specifications/entry-points/#use-for-scripts""" # noqa + print(get_current_version()) + return 0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..87ae22e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +--index-url https://pypi.python.org/simple + +-e . diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..df6bbdb --- /dev/null +++ b/setup.py @@ -0,0 +1,42 @@ +from setuptools import setup, find_packages + +api = [ + 'boto3' + , 'flask >= 0.10.1' + , 'uwsgi' + , 'Flask-HTTPAuth' + , 'dscommons' +] +tools = [ + 'awscli' + , 'bumpversion' + , 'pip2pi' + , 'tox' + , 'pip-tools' +] + +setup( + name='ihrpi', + version='1.0.0', + author='Sam Garrett', + author_email='samgarrett@iheartmedia.com', + description='iHeart private packaging tools & index.', + scripts=[ + 'bin/ihrpi-build', + 'bin/ihrpi-configure-travis', + 'bin/ihrpi-gen-requirements', + 'bin/ihrpi-publish', + 'bin/ihrpi-release', + 'bin/ihrpi-setup-env', + 'bin/ihrpi-tox-install', + 'bin/ihrpi-tox-run', + ], + entry_points={ + 'console_scripts': ['ihrpi-gcv=ihrpi.tools:gcv_main'], + }, + extras_require={ + 'api': api + , 'tools': tools + }, + packages=find_packages(exclude=['tests']) +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..828212e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,40 @@ +import boto3 +import pytest + +from werkzeug.security import generate_password_hash +from ihrpi import factory +from mock import patch +from moto import mock_s3 + +@pytest.fixture(scope="session") +def s3_resource(request): + mock = mock_s3() + mock.start() + s3r = boto3.resource('s3', region_name='us-east-1') + request.addfinalizer(lambda: mock.stop()) + return s3r + + +@pytest.fixture(autouse=True) +def app(request, monkeypatch): + p = 'plain_pass' + monkeypatch.setattr(factory, '_get_users', lambda: { + 'basic_user': generate_password_hash(p) + }) + config = { + 'TESTING': True, + 'BUCKET': 'ihr-local', + 'PREFIX': 'packages/simple', + 'PLAIN_PASS': p + } + app = factory.create_app(config=config) + with app.app_context(): + yield app + + +@pytest.fixture +def client(request, app): + + client = app.test_client() + + return client diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..8fea524 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,77 @@ +import boto3 +import pytest +import six + +from base64 import b64encode +from os.path import join +from requests.auth import HTTPBasicAuth + +class TestBase: + + @classmethod + def get_auth_headers(cls, u, p): + s = "{0}:{1}".format(u, p) + return { + 'Authorization': 'Basic %s' % b64encode(bytes(s, "utf-8")).decode("ascii") + } + + + @pytest.fixture(autouse=True) + def setup_method(self, tmpdir, app, s3_resource): + bucket = app.config['BUCKET'] + prefix = app.config['PREFIX'] + (user, _) = list(app.config['USERS'].items())[0] + p = app.config['PLAIN_PASS'] + self.auth_headers = self.get_auth_headers(user, p) + self.pkg_names = ["bye", "hi"] + self.base = "simple" + s3_resource.create_bucket(Bucket=bucket) + + self.n_versions = 3 + self.pkg_val = [] + self.key = [] + s3_client = boto3.client('s3') + for pkg in self.pkg_names: + for i in range(self.n_versions): + self.pkg_val.append("{}_{}".format(pkg, i)) + self.key.append("{}-0.{}.0.tar.gz".format(pkg, i)) + s3_client.put_object(Bucket=bucket, + Key=join(prefix, pkg, self.key[i]), + Body=self.pkg_val[i]) + + + def test_ping(self, client): + r = client.get('/ping', + headers=self.auth_headers) + assert six.b('pong') in r.data + + + def test_basic(self, app, client): + r = client.get(join(self.base, self.pkg_names[0], self.key[0]), + headers=self.auth_headers) + assert six.b(self.pkg_val[0]) == r.data + + + def test_top_level_list(self, app, client): + r = client.get("/"+self.base+"/", + headers=self.auth_headers) + for n in self.pkg_names: + expect = six.b("a href=\"/{base}/{n}/\">{n}<".format( + base=self.base, n=n)) + assert expect in r.data + + + def test_basic_list(self, app, client): + pkg = self.pkg_names[0] + r = client.get(join(self.base, pkg), + headers=self.auth_headers) + for idx, fname in enumerate(self.key[:self.n_versions]): + expect = six.b("a href=\"/{base}/{pkg}/{fname}\">{fname}<".format( + base=self.base, pkg=pkg, fname=fname)) + assert expect in r.data + + + def test_not_found(self, app, client): + r = client.get(join(self.base, "blah"), + headers=self.auth_headers) + assert 404 == r.status_code diff --git a/tests/test_tools.py b/tests/test_tools.py new file mode 100644 index 0000000..6451a91 --- /dev/null +++ b/tests/test_tools.py @@ -0,0 +1,31 @@ +import os + +from posixpath import join +from ihrpi import tools + +class TestTools: + + def get_test_file(self, v, tag_name): + return """ +[bumpversion] +current_version = {} +commit = True +tag = True +{} + +[bumpversion:file:setup.py] + +""".format(v, tag_name if tag_name is not None else '') + + def test_get_current_version(self, tmpdir): + p = join(tmpdir, '.bumpversion.cfg') + os.chdir(tmpdir) + + v = '0.1.0' + prefix = 'my_version' + for expected, t in [('v'+v, ''), + (prefix+v, 'tag_name = %s{new_version}'%prefix)]: + with open(p, 'w') as f: + f.write(self.get_test_file(v, t)) + actual = tools.get_current_version() + assert expected == actual diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..db29d2a --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = pep8,py3 + +[testenv] +deps = + pytest + coverage + mock + moto + +commands = + pip install -r ci/requirements.txt + python -m coverage run -m pytest {posargs: tests} + python -m coverage report -m --include="ihrpi/*" + +[testenv:pep8] +basepython = python3 +deps = + flake8-docstrings==0.2.8 + pep8-naming +commands = + flake8 ihrpi