From 8a7ec839e815f4d20a8c3999173c2c01582b8645 Mon Sep 17 00:00:00 2001 From: "josiah.johnston" Date: Fri, 16 Aug 2019 12:30:56 -0700 Subject: [PATCH 1/3] Store precise version number, and append suffixes as-needed to ensure exact version of code can be known: release[+gitsha]+localmod. If git is available, the release will be based on the last tag in the git history that starts with "2". [+gitsha] will be ignored if this is exactly a release. +localmod will be dropped if there are no uncommitted modifications to the code. If the git is unavailable or the attempt to get more precise information fails for whatever reason, the base version as recorded in version.py will be used. Also, save the installed switch version in the output directory for improved record keeping and reproducibility, especially during active software development. --- .gitignore | 1 + get_and_record_version.py | 121 ++++++++++++++++++++++++++++++++++ setup.py | 14 ++-- switch_model/data/__init__.py | 3 + switch_model/utilities.py | 8 +++ switch_model/version.py | 11 +++- 6 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 get_and_record_version.py create mode 100644 switch_model/data/__init__.py diff --git a/.gitignore b/.gitignore index b4ecdf8c0..4d5e2aa3b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ venv build*/ dist*/ autodoc* +switch_model/data/installed_version.txt diff --git a/get_and_record_version.py b/get_and_record_version.py new file mode 100644 index 000000000..c7869638f --- /dev/null +++ b/get_and_record_version.py @@ -0,0 +1,121 @@ +from __future__ import print_function +import argparse +import logging +import os +import subprocess + +""" +Define a precise package version that includes any git digests for any commits +made subsequently to a package release. + +Example: +1) Some commits have been made subsequent to an official release (possibly on +a branch), plus some uncommitted modifications. The version would be: +v1.0.4+{gitsha}+localmod +2) Same scenario, but no uncommitted modifications: v1.0.4+{gitsha} +3) No commits since the last official release: v1.0.4 + +These functions are encoded into a separate file from setup.py to support +including precise versions in docker tags. +""" + +def get_git_version(): + """ + Try to get git version like '{tag}+{gitsha}', with the added suffix + "+localmod" if the git repo has had any uncommitted modifications. + The "+{gitsha}" suffix will be dropped if this is the tagged version. + Code adapted from setuptools_git_version which has an MIT license. + https://pypi.org/project/setuptools-git-version/ + Note: Only look for tags that start with "2." to avoid tags like "demo-v1.0.1". + """ + git_command = "git describe --tags --long --match '2.*' --dirty --always" + fmt = '{tag}+{gitsha}{dirty}' + + git_version = subprocess.check_output(git_command, shell=True).decode('utf-8').strip() + parts = git_version.split('-') + # FYI, if it can't find a tag for whatever reason, len may be 1 or 2 + assert len(parts) in (3, 4), ( + "Trouble parsing git version output. Got {}, expected 3 or 4 things " + "separated by dashes. This has been caused by the repository having no " + "available tags, which was solved by fetching from the main repo:\n" + "`git remote add main https://github.com/switch-model/switch.git && " + "git fetch --all`".format(git_version) + ) + if len(parts) == 4: + dirty = '+localmod' + else: + dirty = '' + tag, count, sha = parts[:3] + if count == '0' and not dirty: + return tag + return fmt.format(tag=tag, gitsha=sha.lstrip('g'), dirty=dirty) + +def get_and_record_version(repo_path): + """ + Attempt to get an absolute version number that includes commits made since + the last release. If that succeeds, record the absolute version and use it + for the pip catalog. If that fails, fall back to something reasonable and + vague for the pip catalog, using the data from base_version.py. + """ + pkg_dir = os.path.join(repo_path , 'switch_model' ) + data_dir = os.path.join(pkg_dir, 'data' ) + __version__ = None + try: + __version__ = get_git_version() + with open(os.path.join(data_dir, 'installed_version.txt'), 'w+') as f: + f.write(__version__) + except subprocess.CalledProcessError as e: + logging.warning( + "Could not call git as a subprocess to determine precise version." + "Falling back to using the static version from version.py") + logging.exception(e) + except AssertionError as e: + logging.warning("Trouble parsing git output.") + logging.exception(e) + except Exception as e: + logging.warning( + "Trouble getting precise version from git repository; " + "using base version from switch_model/version.py. " + "Error was: {}".format(e) + ) + if __version__ is None: + module_dat = {} + with open(os.path.join(pkg_dir, 'version.py')) as fp: + exec(fp.read(), module_dat) + __version__ = module_dat['__version__'] + return __version__ + +def get_args(): + parser = argparse.ArgumentParser( + description='Get a precise local version of this git repository', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--verbose', '-v', dest='verbose', default=False, + action='store_const', const=logging.WARNING, + help='Show information about model preparation and solution') + parser.add_argument( + '--very-verbose', '-vv', dest='verbose', default=False, + action='store_const', const=logging.INFO, + help='Show more information about model preparation and solution') + parser.add_argument( + '--very-very-verbose', '-vvv', dest='verbose', default=False, + action='store_const', const=logging.DEBUG, + help='Show debugging-level information about model preparation and solution') + parser.add_argument( + '--quiet', '-q', dest='verbose', action='store_false', + help="Don't show information about model preparation and solution " + "(cancels --verbose setting)") + + args = parser.parse_args() + return args + +def main(): + args = get_args() + if args.verbose: + logging.basicConfig(format='%(levelname)s:%(message)s', level=args.verbose) + repo_path = os.path.dirname(os.path.realpath(__file__)) + __version__ = get_and_record_version(repo_path) + print(__version__) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup.py b/setup.py index 30deab938..109647e2b 100644 --- a/setup.py +++ b/setup.py @@ -15,12 +15,10 @@ import os from setuptools import setup, find_packages -# Get the version number. Strategy #3 from https://packaging.python.org/single_source_version/ -version_path = os.path.join(os.path.dirname(__file__), 'switch_model', 'version.py') -version = {} -with open(version_path) as f: - exec(f.read(), version) -__version__ = version['__version__'] +from get_and_record_version import get_and_record_version + +repo_path = os.path.dirname(os.path.realpath(__file__)) +__version__ = get_and_record_version(repo_path) def read(*rnames): return open(os.path.join(os.path.dirname(__file__), *rnames)).read() @@ -55,6 +53,9 @@ def read(*rnames): 'Topic :: Software Development :: Libraries :: Python Modules' ], packages=find_packages(include=['switch_model', 'switch_model.*']), + package_data = { + 'switch_model': ['data/*'] + }, keywords=[ 'renewable', 'power', 'energy', 'electricity', 'production cost', 'capacity expansion', @@ -65,6 +66,7 @@ def read(*rnames): 'pint', # needed by Pyomo when we run our tests, but not included 'testfixtures', # used for standard tests 'pandas', # used for input upgrades and testing that functionality + 'setuptools', # For parsing version numbers; it is part of almost all python distributions, but not guaranteed. ], extras_require={ # packages used for advanced demand response, progressive hedging diff --git a/switch_model/data/__init__.py b/switch_model/data/__init__.py new file mode 100644 index 000000000..2ae5248df --- /dev/null +++ b/switch_model/data/__init__.py @@ -0,0 +1,3 @@ +"""This directory contains any necessary package data or default configuration +files. +""" \ No newline at end of file diff --git a/switch_model/utilities.py b/switch_model/utilities.py index f97ce27c4..d61d8ab6d 100644 --- a/switch_model/utilities.py +++ b/switch_model/utilities.py @@ -12,6 +12,8 @@ from pyomo.environ import * import pyomo.opt +import switch_model + # Define string_types (same as six.string_types). This is useful for # distinguishing between strings and other iterables. try: @@ -263,6 +265,12 @@ def post_solve(instance, outputs_dir=None): if hasattr(module, 'post_solve'): module.post_solve(instance, outputs_dir) + # Save the precise version used to solve this problem. + version_path = os.path.join(outputs_dir, 'software_version.txt') + with open(version_path, 'w') as f: + f.write("This problem was solved with switch version {}.{}".format( + switch_model.__version__, os.linesep)) + def min_data_check(model, *mandatory_model_components): """ diff --git a/switch_model/version.py b/switch_model/version.py index 813233d64..b98fbca11 100644 --- a/switch_model/version.py +++ b/switch_model/version.py @@ -5,4 +5,13 @@ distribution because it needs to be executed before Switch (and its dependencies) are installed. """ -__version__='2.0.6-dev' +import os + +base_version = '2.0.6-dev' + +try: + DATA_ROOT = os.path.join(os.path.dirname(__file__), 'data') + with open(os.path.join(DATA_ROOT, 'installed_version.txt'), 'r') as f: + __version__ = f.read().strip() +except (IOError, NameError): + __version__ = base_version From 45585fd39493a513280c5d06045f7c726f15871b Mon Sep 17 00:00:00 2001 From: "josiah.johnston" Date: Wed, 11 Sep 2019 13:59:58 -0700 Subject: [PATCH 2/3] Update precise version parser to handle "tags/" prefix that appear in git descriptions in some contexts. --- get_and_record_version.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/get_and_record_version.py b/get_and_record_version.py index c7869638f..2df65a4af 100644 --- a/get_and_record_version.py +++ b/get_and_record_version.py @@ -32,16 +32,16 @@ def get_git_version(): fmt = '{tag}+{gitsha}{dirty}' git_version = subprocess.check_output(git_command, shell=True).decode('utf-8').strip() - parts = git_version.split('-') - # FYI, if it can't find a tag for whatever reason, len may be 1 or 2 - assert len(parts) in (3, 4), ( + match = re.match("(tags/)?(.*)-([\d]+)-g([0-9a-f]+)(-dirty)?", git_version) + assert match, ( "Trouble parsing git version output. Got {}, expected 3 or 4 things " "separated by dashes. This has been caused by the repository having no " "available tags, which was solved by fetching from the main repo:\n" "`git remote add main https://github.com/switch-model/switch.git && " "git fetch --all`".format(git_version) ) - if len(parts) == 4: + parts = match.groups()[1:] + if parts[-1] == '-dirty': dirty = '+localmod' else: dirty = '' From bcfd9a7dcaef32d9fbc68ba6beb6babf7b89eddc Mon Sep 17 00:00:00 2001 From: "josiah.johnston" Date: Wed, 11 Sep 2019 15:36:42 -0700 Subject: [PATCH 3/3] Update precise version to include the number of commits since the tagged version. Also use the git-standard "dirty" suffix instead of "localmod" for installations from code that hasn't been committed. --- get_and_record_version.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/get_and_record_version.py b/get_and_record_version.py index 2df65a4af..9f407f51c 100644 --- a/get_and_record_version.py +++ b/get_and_record_version.py @@ -6,14 +6,17 @@ """ Define a precise package version that includes any git digests for any commits -made subsequently to a package release. +made subsequently to a package release. The base version (2.0.4 in this +example) is obtained from the last tag that starts with "2". version. Also use +the git-standard "dirty" suffix instead of "localmod" for installations from +code that hasn't been committed. Example: -1) Some commits have been made subsequent to an official release (possibly on +1) 112 commits were made subsequent to an official release (possibly on a branch), plus some uncommitted modifications. The version would be: -v1.0.4+{gitsha}+localmod -2) Same scenario, but no uncommitted modifications: v1.0.4+{gitsha} -3) No commits since the last official release: v1.0.4 +2.0.4+112+{gitsha}+dirty +2) Same scenario, but no uncommitted modifications: 2.0.4+112+{gitsha} +3) No commits since the last tagged release: 2.0.4 These functions are encoded into a separate file from setup.py to support including precise versions in docker tags. @@ -22,33 +25,35 @@ def get_git_version(): """ Try to get git version like '{tag}+{gitsha}', with the added suffix - "+localmod" if the git repo has had any uncommitted modifications. + "+dirty" if the git repo has had any uncommitted modifications. The "+{gitsha}" suffix will be dropped if this is the tagged version. Code adapted from setuptools_git_version which has an MIT license. https://pypi.org/project/setuptools-git-version/ - Note: Only look for tags that start with "2." to avoid tags like "demo-v1.0.1". + Note: Only look for tags that start with "2." to avoid tags of + non-released versions. """ - git_command = "git describe --tags --long --match '2.*' --dirty --always" - fmt = '{tag}+{gitsha}{dirty}' + git_command = "git describe --all --long --match '2.*' --dirty --always" + fmt = '{base_v}+{count}+{gitsha}{dirty}' git_version = subprocess.check_output(git_command, shell=True).decode('utf-8').strip() + # The prefix tags/ may not appear in every context, and should be ignored. match = re.match("(tags/)?(.*)-([\d]+)-g([0-9a-f]+)(-dirty)?", git_version) assert match, ( "Trouble parsing git version output. Got {}, expected 3 or 4 things " - "separated by dashes. This has been caused by the repository having no " - "available tags, which was solved by fetching from the main repo:\n" + "separated by dashes. This has been encountered when the local git repo " + "lacks tags, which can be solved by fetching from the main repo:\n" "`git remote add main https://github.com/switch-model/switch.git && " "git fetch --all`".format(git_version) ) parts = match.groups()[1:] if parts[-1] == '-dirty': - dirty = '+localmod' + dirty = '+dirty' else: dirty = '' - tag, count, sha = parts[:3] + base_v, count, sha = parts[:3] if count == '0' and not dirty: - return tag - return fmt.format(tag=tag, gitsha=sha.lstrip('g'), dirty=dirty) + return base_v + return fmt.format(base_v=base_v, count=count, gitsha=sha, dirty=dirty) def get_and_record_version(repo_path): """