From 552d1126fd63cf7828ee434a325856cdd252f4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrius=20Laukavi=C4=8Dius?= Date: Wed, 13 Mar 2024 13:56:50 +0200 Subject: [PATCH] [ADD] maraplus: install_paths Option to specify paths where modules to install are defined. This allows to reuse list of modules that can be installed between multiple environments. [BRANCH] feature-install-paths-ala --- .gitignore | 1 + README.rst | 5 +- maraplus/config.py | 4 +- maraplus/parser.py | 102 ++++++++++++- setup.cfg | 3 + setup.py | 2 +- tests/conftest.py | 17 +++ tests/data/modules_one.yml | 5 + tests/data/modules_two.yml | 5 + tests/test_install_path_addons.py | 237 ++++++++++++++++++++++++++++++ tests/test_unpack_keys.py | 48 ++++++ 11 files changed, 420 insertions(+), 9 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/data/modules_one.yml create mode 100644 tests/data/modules_two.yml create mode 100644 tests/test_install_path_addons.py create mode 100644 tests/test_unpack_keys.py diff --git a/.gitignore b/.gitignore index 6e69090..cca46ac 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ var/ *.egg-info/ .installed.cfg *.egg +.venv diff --git a/README.rst b/README.rst index db05cba..3a536e3 100644 --- a/README.rst +++ b/README.rst @@ -16,4 +16,7 @@ Wrapper for :code:`marabunta` package, that adds some extra features: * If environment variable key is specified in configuration options, it will be replaced by its value, if such environment variable exists. E.g. if :code:`MY_ENV=test`, :code:`$MY_ENV` would be replaced by :code:`test`. - +* Can specify ``install_paths`` so modules are collected from specified file instead + of needing to explicitly specify in marabunta main yaml file. Modules specified in + these files are added into ``install`` option. If module already exists in ``install`` + option, it is not added multiple times. diff --git a/maraplus/config.py b/maraplus/config.py index cefdf3b..4cb98d8 100644 --- a/maraplus/config.py +++ b/maraplus/config.py @@ -53,8 +53,8 @@ def from_parse_args(cls, args): """Override to create config with extra arguments.""" if not (bool(args.db_password) ^ bool(args.db_password_file)): raise TypeError( - "--db-password and --db-password-file arguments are mutually" + - " exclusive" + "--db-password and --db-password-file arguments are mutually" + + " exclusive" ) return cls( args.migration_file, diff --git a/maraplus/parser.py b/maraplus/parser.py index 6dfdf57..3b00594 100644 --- a/maraplus/parser.py +++ b/maraplus/parser.py @@ -11,23 +11,71 @@ ADDITIVE_STRAT = mergedeep.Strategy.ADDITIVE +# TODO: Should use tuples here to make sure data is immutable. VERSION_LIST_PATHS = [ ['operations', 'pre'], ['operations', 'post'], ['addons', 'install'], ['addons', 'upgrade'], ] +KEY_INSTALL_PATHS = 'install_paths' +# Possible places for install_path argument. +VERSION_INSTALL_PATHS = [ + ['addons', KEY_INSTALL_PATHS], + # Asterisk means, expand to all modes (keys in modes dict) + ['modes', '*', 'addons', KEY_INSTALL_PATHS], +] DEL_OPT_RE = r'DEL->{(.*)}' ENV_OPT_RE = r'(\$([A-Z0-9]+(?:_[A-Z0-9]+)*))' -# TODO: move this to footil. -def _get_from_dict(data: dict, keys: list) -> any: +def get_from_nested_dict(data: dict, keys: list) -> any: """Retrieve value from nested dict.""" return functools.reduce(operator.getitem, keys, data) +def pop_from_nested_dict(data: dict, keys: list): + """Pop element at the end of nested dict using keys list as a path.""" + if len(keys) == 1: + return data.pop(keys[0]) + key = keys[-1] + inner_dct = get_from_nested_dict(data, keys[:-1]) + return inner_dct.pop(key) + + +def unpack_keys_from_nested_dict(data: dict, keys_chain: list): + """Unpack keys lists when any key is asterisk. + + Asterisk means, we have key chains for each key in place of asterisk + + E.g. + - data={'a': {'b1': {'c': 1}, 'b2': {'c': 2}}} + - keys_chain=['a', '*', 'c] + Results in these key chains: + [ + ['a', 'b1', 'c'], + ['a', 'b2', 'c'], + ] + + """ + unpacked = [[]] + for idx, key in enumerate(keys_chain): + if key == '*': + new_unpacked = [] + for path_keys in unpacked: + prev_keys = path_keys[:idx] + asterisk_keys = get_from_nested_dict(data, prev_keys).keys() + new_unpacked.extend( + prev_keys + [k] for k in asterisk_keys + ) + unpacked = new_unpacked + else: + for keys in unpacked: + keys.append(key) + return unpacked + + def _find_data_by_key(datas: list, key: str, val: any) -> dict: for data in datas: if data[key] == val: @@ -46,10 +94,54 @@ def _render_env_placeholders(opt): class YamlParser(parser_orig.YamlParser): """Parser that can additionally parse install addons option.""" + def __init__(self, parsed): + self.postprocess_parsed(parsed) + super().__init__(parsed) + @property def _version_list_paths(self): return VERSION_LIST_PATHS + def postprocess_parsed(self, parsed): + # Handle install_path arg. + self._parse_install_paths(parsed) + + def _parse_install_paths(self, parsed): + versions = parsed['migration']['versions'] + for version in versions: + unpacked_keys = [] + for vpaths in VERSION_INSTALL_PATHS: + # It is expected that some paths might not be defined + # in parsed file. + try: + unpacked_keys.extend(unpack_keys_from_nested_dict(version, vpaths)) + except KeyError: + continue + for keys in unpacked_keys: + try: + paths = pop_from_nested_dict(version, keys) + except KeyError: + continue + # Parent dict is expected to be addons dict. + addons_cfg = get_from_nested_dict(version, keys[:-1]) + for path in paths: + self._parse_install_path(addons_cfg, path) + + def _parse_install_path(self, addons_cfg: dict, path: str): + with open(path, 'r') as f: + modules_dct = yaml.safe_load(f) + try: + modules = modules_dct['install'] + except Exception as e: + raise ParseError(f"install_path file expects 'install' key. Error: {e}") + if not isinstance(modules, list): + raise ParseError("'install_paths' key must be a list") + addons_cfg.setdefault('install', []) + install = addons_cfg['install'] + for module in modules: + if module not in install: + install.append(module) + @classmethod def parser_from_buffer(cls, fp, *extra_fps): """Extend to merge extra yaml.""" @@ -103,12 +195,12 @@ def _merge_yaml(self, fps): def _merge_dict(self, keys, extras): try: - main_dict = _get_from_dict(self.parsed, keys) + main_dict = get_from_nested_dict(self.parsed, keys) except KeyError: return for extra in extras: try: - extra_dict = _get_from_dict(extra, keys) + extra_dict = get_from_nested_dict(extra, keys) mergedeep.merge(main_dict, extra_dict, strategy=ADDITIVE_STRAT) except KeyError: continue @@ -143,7 +235,7 @@ def _update_options(self, update_method): def update_data(data): for keys_path in self._version_list_paths: try: - vals_list = _get_from_dict(data, keys_path) + vals_list = get_from_nested_dict(data, keys_path) update_method(data, keys_path, vals_list) except KeyError: continue diff --git a/setup.cfg b/setup.cfg index 3c6e79c..e01d48e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] universal=1 +[flake8] +ignore = E203,W503 +max-line-length = 88 diff --git a/setup.py b/setup.py index 7c44634..e0b6023 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ license='AGPLv3+', packages=find_packages(), install_requires=[ - 'marabunta>=0.10.6', + 'marabunta>=0.12.0', 'mergedeep>=1.3.4', 'PyYAML>=6.0', ], diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..97cfeba --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +import pathlib +import pytest + + +@pytest.fixture(scope="package") +def path_modules_one(): + yield str(_get_path("modules_one.yml")) + + +@pytest.fixture(scope="package") +def path_modules_two(): + yield str(_get_path("modules_two.yml")) + + +def _get_path(*args) -> pathlib.Path: + p = pathlib.Path(__file__).parent / "data" + return p.joinpath(*args) diff --git a/tests/data/modules_one.yml b/tests/data/modules_one.yml new file mode 100644 index 0000000..a7608da --- /dev/null +++ b/tests/data/modules_one.yml @@ -0,0 +1,5 @@ +--- +install: + - crm + - sale + - mrp diff --git a/tests/data/modules_two.yml b/tests/data/modules_two.yml new file mode 100644 index 0000000..55cff6b --- /dev/null +++ b/tests/data/modules_two.yml @@ -0,0 +1,5 @@ +--- +install: + - sale + - account + - stock diff --git a/tests/test_install_path_addons.py b/tests/test_install_path_addons.py new file mode 100644 index 0000000..1582202 --- /dev/null +++ b/tests/test_install_path_addons.py @@ -0,0 +1,237 @@ +import io + +from maraplus.parser import YamlParser + + +def test_01_parse_install_path_addons_with_install_key(path_modules_one): + # GIVEN + buffer = io.StringIO(f"""--- +migration: + options: + install_command: odoo + versions: + - version: setup + modes: + stage: + addons: + install: [] + addons: + install: + - crm + install_paths: + - {path_modules_one} +""") + # WHEN + res = YamlParser.parser_from_buffer(buffer).parsed + # THEN + assert res == { + 'migration': { + 'options': {'install_command': 'odoo'}, + 'versions': [ + { + 'version': 'setup', + 'modes': { + 'stage': { + 'addons': { + 'install': [], + } + } + }, + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp' + ] + } + } + ], + } + } + + +def test_02_parse_install_path_addons_without_install_key(path_modules_one): + # GIVEN + buffer = io.StringIO(f"""--- +migration: + options: + install_command: odoo + versions: + - version: setup + modes: + stage: + addons: + install: [] + addons: + install_paths: + - {path_modules_one} +""") + # WHEN + res = YamlParser.parser_from_buffer(buffer).parsed + # THEN + assert res == { + 'migration': { + 'options': {'install_command': 'odoo'}, + 'versions': [ + { + 'version': 'setup', + 'modes': { + 'stage': { + 'addons': { + 'install': [], + } + } + }, + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp' + ] + } + } + ], + } + } + + +def test_03_parse_install_path_addons_multiple(path_modules_one, path_modules_two): + # GIVEN + buffer = io.StringIO(f"""--- +migration: + options: + install_command: odoo + versions: + - version: setup + addons: + install_paths: + - {path_modules_one} + - {path_modules_two} +""") + # WHEN + res = YamlParser.parser_from_buffer(buffer).parsed + # THEN + assert res == { + 'migration': { + 'options': {'install_command': 'odoo'}, + 'versions': [ + { + 'version': 'setup', + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp', + 'account', + 'stock', + ] + } + } + ], + } + } + + +def test_04_parse_install_path_addons_in_mode(path_modules_one): + # GIVEN + buffer = io.StringIO(f"""--- +migration: + options: + install_command: odoo + versions: + - version: setup + modes: + stage: + addons: + install: + - crm + install_paths: + - {path_modules_one} +""") + # WHEN + res = YamlParser.parser_from_buffer(buffer).parsed + # THEN + assert res == { + 'migration': { + 'options': {'install_command': 'odoo'}, + 'versions': [ + { + 'version': 'setup', + 'modes': { + 'stage': { + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp' + ], + } + } + }, + } + ], + } + } + + +def test_05_parse_install_path_addons_in_multi_mode(path_modules_one, path_modules_two): + # GIVEN + buffer = io.StringIO(f"""--- +migration: + options: + install_command: odoo + versions: + - version: setup + modes: + stage: + addons: + install_paths: + - {path_modules_one} + prod: + addons: + install_paths: + - {path_modules_two} + addons: + install_paths: + - {path_modules_one} +""") + # WHEN + res = YamlParser.parser_from_buffer(buffer).parsed + # THEN + assert res == { + 'migration': { + 'options': {'install_command': 'odoo'}, + 'versions': [ + { + 'version': 'setup', + 'modes': { + 'stage': { + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp' + ], + } + }, + 'prod': { + 'addons': { + 'install': [ + 'sale', + 'account', + 'stock' + ], + } + }, + }, + 'addons': { + 'install': [ + 'crm', + 'sale', + 'mrp', + ] + } + } + ], + } + } diff --git a/tests/test_unpack_keys.py b/tests/test_unpack_keys.py new file mode 100644 index 0000000..a6572e4 --- /dev/null +++ b/tests/test_unpack_keys.py @@ -0,0 +1,48 @@ +from maraplus.parser import unpack_keys_from_nested_dict + + +def test_01_unpack_keys_no_asterisk(): + # GIVEN + data = {'a': {'b1': {'c': 1}, 'b2': {'c': 2}}} + keys_chain = ['a', 'b1', 'c'] + # WHEN + res = unpack_keys_from_nested_dict(data, keys_chain) + # THEN + assert res == [['a', 'b1', 'c']] + + +def test_02_unpack_keys_one_asterisk(): + # GIVEN + data = {'a': {'b1': {'c': 1}, 'b2': {'c': 2}}} + keys_chain = ['a', '*', 'c'] + # WHEN + res = unpack_keys_from_nested_dict(data, keys_chain) + # THEN + assert res == [['a', 'b1', 'c'], ['a', 'b2', 'c']] + + +def test_03_unpack_keys_two_asterisks(): + # GIVEN + data = {'a': {'b1': {'c': 1}, 'b2': {'c': 2}}} + data = { + 'a': { + 'b1': { + 'c1': {'d': 1}, + 'c2': {'d': 2}, + }, + 'b2': { + 'c3': {'d': 3}, + 'c4': {'d': 4}, + }, + } + } + keys_chain = ['a', '*', '*', 'd'] + # WHEN + res = unpack_keys_from_nested_dict(data, keys_chain) + # THEN + assert res == [ + ['a', 'b1', 'c1', 'd'], + ['a', 'b1', 'c2', 'd'], + ['a', 'b2', 'c3', 'd'], + ['a', 'b2', 'c4', 'd'], + ]