From b4a83eaf68bcd23030ac576292b2713403f3cc48 Mon Sep 17 00:00:00 2001 From: Derrick Chambers Date: Fri, 10 Apr 2020 12:27:29 -0700 Subject: [PATCH] add cli, blacken code --- README.md | 16 ++ obspy_github_api/__init__.py | 2 +- obspy_github_api/cli.py | 52 ++++++ obspy_github_api/obspy_github_api.py | 148 +++++++++++------- obspy_github_api/tests/test_cli.py | 49 ++++++ .../tests/test_obspy_github_api.py | 44 ++++-- setup.py | 31 ++-- 7 files changed, 256 insertions(+), 86 deletions(-) create mode 100644 obspy_github_api/cli.py create mode 100644 obspy_github_api/tests/test_cli.py diff --git a/README.md b/README.md index 89f4168..a965583 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,22 @@ # obspy_github_api Helper routines to interact with obspy/obspy via GitHub API +## Quick start + +The easiest way to use obspy_github_api is via its command line interface. + +```shell script +# Use the magic strings found in issue 101's comments to create a config file +obshub make_config 101 --path obspy_config.json + +# Read a specified option. +obshub read_config_value module_list --path obspy_config.json + +# Use a value in the config in another command line utility. +export BUILDDOCS=`bshub read_config_value module_list --path obspy_config.json` +some-other-command --docs $BUILDDOCS +``` + ## Release Versions Release versions are done from separate branches, see https://github.com/obspy/obspy_github_api/branches. diff --git a/obspy_github_api/__init__.py b/obspy_github_api/__init__.py index 6438f1e..7d41f8e 100644 --- a/obspy_github_api/__init__.py +++ b/obspy_github_api/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- from .obspy_github_api import * -__version__ = '0.0.0.dev' +__version__ = "0.0.0.dev" diff --git a/obspy_github_api/cli.py b/obspy_github_api/cli.py new file mode 100644 index 0000000..990cd58 --- /dev/null +++ b/obspy_github_api/cli.py @@ -0,0 +1,52 @@ +""" +Command line Interface for obspy_github_api +""" +import json +from typing import Optional + +import typer + +from obspy_github_api.obspy_github_api import make_ci_json_config + +app = typer.Typer() + +DEFAULT_CONFIG_PATH = "obspy_config/conf.json" + + +@app.command() +def make_config( + issue_number: int, path: str = DEFAULT_CONFIG_PATH, token: Optional[str] = None +): + """ + Create ObsPy's configuration json file for a particular issue. + + This command parses the comments in an issue's text looking for any magic + strings (defined in ObsPy's issue template) and stores the values assigned + to them to a json file for later use. + + The following names are stored in the config file: + module_list - A string of requested modules separated by commas. + module_list_spaces - A string of requested modules separated by spaces. + docs - True if a doc build is requested. + """ + make_ci_json_config(issue_number, path=path, token=token) + + +@app.command() +def read_config_value(name: str, path: str = DEFAULT_CONFIG_PATH): + """ + Read a value from the configuration file. + """ + with open(path, "r") as fi: + params = json.load(fi) + value = params[name] + print(value) + return value + + +def main(): + app() + + +if __name__ == "__main__": + main() diff --git a/obspy_github_api/obspy_github_api.py b/obspy_github_api/obspy_github_api.py index 810dceb..dae042f 100644 --- a/obspy_github_api/obspy_github_api.py +++ b/obspy_github_api/obspy_github_api.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import ast import datetime -import importlib.util import json import os import re @@ -14,9 +13,9 @@ # regex pattern in comments for requesting a docs build -PATTERN_DOCS_BUILD = r'\+DOCS' +PATTERN_DOCS_BUILD = r"\+DOCS" # regex pattern in comments for requesting tests of specific submodules -PATTERN_TEST_MODULES = r'\+TESTS:([a-zA-Z0-9_\.,]*)' +PATTERN_TEST_MODULES = r"\+TESTS:([a-zA-Z0-9_\.,]*)" @lru_cache() @@ -29,8 +28,10 @@ def get_github_client(token=None): """ token = token or os.environ.get("GITHUB_TOKEN", None) if token is None: - msg = ("Could not get authorization token for ObsPy github API " - "(env variable OBSPY_COMMIT_STATUS_TOKEN)") + msg = ( + "Could not get authorization token for ObsPy github API " + "(env variable OBSPY_COMMIT_STATUS_TOKEN)" + ) warnings.warn(msg) gh = github3.GitHub() else: @@ -80,9 +81,7 @@ def check_specific_module_tests_requested(issue_number, token=None): def get_module_test_list( - issue_number, - token=None, - module_path='./obspy/core/util/base.py', + issue_number, token=None, module_path="./obspy/core/util/base.py" ): """ Gets the list of modules that should be tested for the given issue number. @@ -132,7 +131,7 @@ def get_values_from_module(node, names): node = ast.parse(open(node).read()) # Parse nodes, any assignments to any of requested_names is saved. - if hasattr(node, 'body'): + if hasattr(node, "body"): for subnode in node.body: out.update(get_values_from_module(subnode, names=requested_names)) elif isinstance(node, ast.Assign): @@ -142,7 +141,7 @@ def get_values_from_module(node, names): return out -def check_docs_build_requested(issue_number, token): +def check_docs_build_requested(issue_number, token=None): """ Check if a docs build was requested for given issue number (by magic string '+DOCS' anywhere in issue comments). @@ -170,7 +169,7 @@ def get_pull_requests(state="open", sort="updated", direction="desc", token=None return prs -def get_commit_status(commit, context=None, fork='obspy', token=None): +def get_commit_status(commit, context=None, fork="obspy", token=None): """ Return current commit status. Either for a specific context, or overall. @@ -194,8 +193,10 @@ def get_commit_status(commit, context=None, fork='obspy', token=None): commit = repo.commit(commit) statuses = {} for status in commit.statuses(): - if (status.context not in statuses or - status.updated_at > statuses[status.context].updated_at): + if ( + status.context not in statuses + or status.updated_at > statuses[status.context].updated_at + ): statuses[status.context] = status # just return current status for given context @@ -221,8 +222,9 @@ def get_commit_time(commit, fork="obspy", token=None): gh = get_github_client(token) repo = gh.repository(fork, "obspy") commit = repo.commit(commit) - dt = datetime.datetime.strptime(commit.commit.committer["date"], - '%Y-%m-%dT%H:%M:%SZ') + dt = datetime.datetime.strptime( + commit.commit.committer["date"], "%Y-%m-%dT%H:%M:%SZ" + ) return time.mktime(dt.timetuple()) @@ -230,11 +232,13 @@ def get_issue_numbers_that_request_docs_build(verbose=False, token=None): """ :rtype: list of int """ - open_prs = get_pull_requests(state="open", token=None) + open_prs = get_pull_requests(state="open", token=token) if verbose: - print("Checking the following open PRs if a docs build is requested " - "and needed: {}".format(str(num for num, _ in open_prs))) + print( + "Checking the following open PRs if a docs build is requested " + "and needed: {}".format(str(num for num, _ in open_prs)) + ) todo = [] for pr in open_prs: @@ -245,7 +249,8 @@ def get_issue_numbers_that_request_docs_build(verbose=False, token=None): def set_pr_docs_that_need_docs_build( - pr_docs_info_dir="/home/obspy/pull_request_docs", verbose=False, token=None): + pr_docs_info_dir="/home/obspy/pull_request_docs", verbose=False, token=None +): """ Relies on a local directory with some files to mark when PR docs have been built etc. @@ -261,9 +266,10 @@ def set_pr_docs_that_need_docs_build( # need to figure out time of last push from commit details.. -_- time = get_commit_time(commit, fork) if verbose: - print("PR #{} requests a docs build, latest commit {} at " - "{}.".format(number, commit, - str(datetime.fromtimestamp(time)))) + print( + "PR #{} requests a docs build, latest commit {} at " + "{}.".format(number, commit, str(datetime.fromtimestamp(time))) + ) filename = os.path.join(pr_docs_info_dir, str(number)) filename_todo = filename + ".todo" @@ -282,9 +288,12 @@ def set_pr_docs_that_need_docs_build( time_done = os.stat(filename_done).st_atime if time_done > time: if verbose: - print("PR #{} was last built at {} and does not need a " - "new build.".format( - number, str(datetime.fromtimestamp(time_done)))) + print( + "PR #{} was last built at {} and does not need a " + "new build.".format( + number, str(datetime.fromtimestamp(time_done)) + ) + ) continue # ..otherwise touch the .todo file with open(filename_todo, "wb"): @@ -295,9 +304,18 @@ def set_pr_docs_that_need_docs_build( print("Done checking which PRs require a docs build.") -def set_commit_status(commit, status, context, description, - target_url=None, fork="obspy", only_when_changed=True, - only_when_no_status_yet=False, verbose=False, token=None): +def set_commit_status( + commit, + status, + context, + description, + target_url=None, + fork="obspy", + only_when_changed=True, + only_when_no_status_yet=False, + verbose=False, + token=None, +): """ :param only_when_changed: Whether to only set a status if the commit status would change (commit statuses can not be updated or deleted and there @@ -320,23 +338,35 @@ def set_commit_status(commit, status, context, description, if only_when_no_status_yet: if current_status is not None: if verbose: - print("Commit {} already has a commit status ({}), " - "skipping.".format(commit, current_status)) + print( + "Commit {} already has a commit status ({}), " + "skipping.".format(commit, current_status) + ) return if only_when_changed: if current_status == status: if verbose: - print("Commit {} status would not change ({}), " - "skipping.".format(commit, current_status)) + print( + "Commit {} status would not change ({}), " + "skipping.".format(commit, current_status) + ) return repo = gh.repository(fork, "obspy") commit = repo.commit(commit) - repo.create_status(sha=commit.sha, state=status, context=context, - description=description, target_url=target_url) + repo.create_status( + sha=commit.sha, + state=status, + context=context, + description=description, + target_url=target_url, + ) if verbose: - print("Set commit {} status (context '{}') to '{}'.".format( - commit.sha, context, status)) + print( + "Set commit {} status (context '{}') to '{}'.".format( + commit.sha, context, status + ) + ) def set_all_updated_pull_requests_docker_testbot_pending(verbose=False, token=None): @@ -347,19 +377,24 @@ def set_all_updated_pull_requests_docker_testbot_pending(verbose=False, token=No open_prs = get_pull_requests(state="open", token=token) if verbose: - print("Working on PRs: " + ", ".join( - [str(pr.number) for pr in open_prs])) + print("Working on PRs: " + ", ".join([str(pr.number) for pr in open_prs])) for pr in open_prs: set_commit_status( - commit=pr.head.sha, status="pending", context="docker-testbot", + commit=pr.head.sha, + status="pending", + context="docker-testbot", description="docker testbot results not available yet", only_when_no_status_yet=True, - verbose=verbose) + verbose=verbose, + ) def get_docker_build_targets( - context="docker-testbot", branches=["master", "maintenance_1.0.x"], - prs=True, token=None): + context="docker-testbot", + branches=["master", "maintenance_1.0.x"], + prs=True, + token=None, +): """ Returns a list of build targets that need a build of a given context. @@ -384,12 +419,12 @@ def get_docker_build_targets( :rtype: string """ if not branches and not prs: - return '' + return "" gh = get_github_client(token) - status_needs_build = (None, 'pending') + status_needs_build = (None, "pending") targets = [] - repo = gh.repository('obspy', 'obspy') + repo = gh.repository("obspy", "obspy") if branches: for name in branches: @@ -400,22 +435,22 @@ def get_docker_build_targets( continue # branches don't have a PR number, use dummy placeholder 'XXX' so # that variable splitting in bash still works - targets.append('XXX_obspy:{}'.format(sha)) + targets.append("XXX_obspy:{}".format(sha)) if prs: - open_prs = get_pull_requests(state='open') + open_prs = get_pull_requests(state="open") for pr in open_prs: fork = pr.head.user sha = pr.head.sha status = get_commit_status(sha, context=context) if status not in status_needs_build: continue - targets.append('{}_{}:{}'.format(str(pr.number), fork, sha)) + targets.append("{}_{}:{}".format(str(pr.number), fork, sha)) - return ' '.join(targets) + return " ".join(targets) -def make_ci_json_config(issue_number, path='obspy_ci_conf.json', token=None): +def make_ci_json_config(issue_number, path="obspy_ci_conf.json", token=None): """ Make a json file for configuring additional actions in CI. @@ -427,10 +462,15 @@ def make_ci_json_config(issue_number, path='obspy_ci_conf.json', token=None): docs = check_docs_build_requested(issue_number, token=token) out = dict( - module_list=('obspy.' + ',obspy.').join(module_list), - module_list_spaces=' '.join(module_list), + module_list=("obspy." + ",obspy.").join(module_list), + module_list_spaces=" ".join(module_list), docs=docs, ) - with open(path, 'w') as fi: - json.dump(out, fi) + # make sure path exists + path = Path(path) + path_dir = path if path.is_dir() else path.parent + path_dir.mkdir(exist_ok=True, parents=True) + + with path.open("w") as fi: + json.dump(out, fi, indent=4) diff --git a/obspy_github_api/tests/test_cli.py b/obspy_github_api/tests/test_cli.py new file mode 100644 index 0000000..82224c2 --- /dev/null +++ b/obspy_github_api/tests/test_cli.py @@ -0,0 +1,49 @@ +""" +Tests for command line interface. +""" +import contextlib +import json +import unittest +import tempfile +from pathlib import Path +from subprocess import run + +import pytest + + +class TestCli: + """" + Test case for command line interface. + """ + + config_dir = tempfile.mkdtemp() + config_path = Path(config_dir) / "conf.json" + pr_number = 100 + + @pytest.fixture(scope="class") + def config_path(self, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obspy_config") + return Path(tmpdir) / "conf.json" + + @pytest.fixture(scope="class") + def populated_config(self, config_path): + """ Get the config for the test PR. """ + run_str = f"obshub make-config {self.pr_number} --path {config_path}" + run(run_str, shell=True, check=True) + return config_path + + def test_path_exists(self, populated_config): + """The config file should now exist.""" + assert Path(populated_config).exists() + + def test_is_json(self, populated_config): + """Ensue the file created can be read by json module. """ + with Path(populated_config).open("r") as fi: + out = json.load(fi) + assert isinstance(out, dict) + + def test_read_config_value(self, populated_config): + """Ensure the config value is printed to screen""" + run_str = f"obshub read-config-value docs --path {populated_config}" + out = run(run_str, shell=True, capture_output=True) + assert out.stdout.decode("utf8").rstrip() == "False" diff --git a/obspy_github_api/tests/test_obspy_github_api.py b/obspy_github_api/tests/test_obspy_github_api.py index c1169e5..853055c 100644 --- a/obspy_github_api/tests/test_obspy_github_api.py +++ b/obspy_github_api/tests/test_obspy_github_api.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- import mock + from obspy_github_api import ( - check_docs_build_requested, check_specific_module_tests_requested, - get_commit_status, get_commit_time, - get_issue_numbers_that_request_docs_build, get_module_test_list, - ) + check_docs_build_requested, + check_specific_module_tests_requested, + get_commit_status, + get_commit_time, + get_issue_numbers_that_request_docs_build, + get_module_test_list, +) MOCK_DEFAULT_MODULES = ["core", "clients.arclink"] @@ -20,18 +24,20 @@ def test_check_docs_build_requested(): def test_check_specific_module_tests_requested(): assert check_specific_module_tests_requested(100) is False assert check_specific_module_tests_requested(101) is True - assert check_specific_module_tests_requested(102) == ["clients.arclink", - "clients.fdsn"] + assert check_specific_module_tests_requested(102) == [ + "clients.arclink", + "clients.fdsn", + ] -@mock.patch('obspy.core.util.base.DEFAULT_MODULES', MOCK_DEFAULT_MODULES) -@mock.patch('obspy.core.util.base.ALL_MODULES', MOCK_ALL_MODULES) +@mock.patch("obspy.core.util.base.DEFAULT_MODULES", MOCK_DEFAULT_MODULES) +@mock.patch("obspy.core.util.base.ALL_MODULES", MOCK_ALL_MODULES) def test_get_module_test_list(): assert get_module_test_list(100) == MOCK_DEFAULT_MODULES assert get_module_test_list(101) == MOCK_ALL_MODULES assert get_module_test_list(102) == sorted( - set.union(set(MOCK_DEFAULT_MODULES), - ["clients.arclink", "clients.fdsn"])) + set.union(set(MOCK_DEFAULT_MODULES), ["clients.arclink", "clients.fdsn"]) + ) def test_get_commit_status(): @@ -39,12 +45,18 @@ def test_get_commit_status(): sha = "f74e0f5bcf26a47df6138c1ce026d9d14d68c4d7" assert get_commit_status(sha) == "pending" assert get_commit_status(sha, context="docker-testbot") == "pending" - assert get_commit_status( - sha, context="continuous-integration/appveyor/branch") == "success" - assert get_commit_status( - sha, context="continuous-integration/appveyor/pr") == "success" - assert get_commit_status( - sha, context="continuous-integration/travis-ci/pr") == "success" + assert ( + get_commit_status(sha, context="continuous-integration/appveyor/branch") + == "success" + ) + assert ( + get_commit_status(sha, context="continuous-integration/appveyor/pr") + == "success" + ) + assert ( + get_commit_status(sha, context="continuous-integration/travis-ci/pr") + == "success" + ) assert get_commit_status(sha, context="coverage/coveralls") == "failure" diff --git a/setup.py b/setup.py index b988ea6..fc8c03c 100644 --- a/setup.py +++ b/setup.py @@ -10,24 +10,25 @@ from setuptools import setup if sys.version_info < (2, 7): - sys.exit('Python < 2.7 is not supported') + sys.exit("Python < 2.7 is not supported") INSTALL_REQUIRES = [ - 'github3.py>=1.0.0a1', # works with 1.0.0a4 + "github3.py>=1.0.0a1", # works with 1.0.0a4 + "typer", # soft dependency on ObsPy itself, for function `get_module_test_list` # or the path to obspy.core.utils.base.py can be provided to avoid # needing to have ObsPy installed. - ] +] -SETUP_DIRECTORY = os.path.dirname(os.path.abspath(inspect.getfile( - inspect.currentframe()))) +SETUP_DIRECTORY = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe())) +) # get the package version from from the main __init__ file. version_regex_pattern = r"__version__ += +(['\"])([^\1]+)\1" -for line in open(os.path.join(SETUP_DIRECTORY, 'obspy_github_api', - '__init__.py')): - if '__version__' in line: +for line in open(os.path.join(SETUP_DIRECTORY, "obspy_github_api", "__init__.py")): + if "__version__" in line: version = re.match(version_regex_pattern, line).group(2) @@ -37,7 +38,8 @@ def find_packages(): """ modules = [] for dirpath, _, filenames in os.walk( - os.path.join(SETUP_DIRECTORY, "obspy_github_api")): + os.path.join(SETUP_DIRECTORY, "obspy_github_api") + ): if "__init__.py" in filenames: modules.append(os.path.relpath(dirpath, SETUP_DIRECTORY)) return [_i.replace(os.sep, ".") for _i in modules] @@ -52,10 +54,10 @@ def find_packages(): url="https://github.com/obspy/obspy_github_api", download_url="https://github.com/obspy/obspy_github_api.git", install_requires=INSTALL_REQUIRES, - python_requires='>3.5', + python_requires=">3.5", keywords=["obspy", "github"], packages=find_packages(), - entry_points={}, + entry_points={"console_scripts": ["obshub=obspy_github_api.cli:main"]}, classifiers=[ "Programming Language :: Python", "Development Status :: 4 - Beta", @@ -63,7 +65,6 @@ def find_packages(): "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", - ], - long_description="Helper routines to interact with obspy/obspy via GitHub " - "API", - ) + ], + long_description="Helper routines to interact with obspy/obspy via GitHub " "API", +)