diff --git a/pyaptly/cli.py b/pyaptly/cli.py index 769db5f..830de27 100644 --- a/pyaptly/cli.py +++ b/pyaptly/cli.py @@ -88,7 +88,7 @@ def legacy(passthrough): ) @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True)) @click.argument("task", type=click.Choice(["create"])) -@click.option("--repo-name", "-n", default="all", type=str, help='deafult: "all"') +@click.option("--repo-name", "-n", default="all", type=str, help='default: "all"') def repo(**kwargs): """Create aptly repos.""" from . import main, repo @@ -111,7 +111,7 @@ def repo(**kwargs): ) @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True)) @click.argument("task", type=click.Choice(["create", "update"])) -@click.option("--mirror-name", "-n", default="all", type=str, help='deafult: "all"') +@click.option("--mirror-name", "-n", default="all", type=str, help='default: "all"') def mirror(**kwargs): """Manage aptly mirrors.""" from . import main, mirror @@ -134,7 +134,7 @@ def mirror(**kwargs): ) @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True)) @click.argument("task", type=click.Choice(["create", "update"])) -@click.option("--snapshot-name", "-n", default="all", type=str, help='deafult: "all"') +@click.option("--snapshot-name", "-n", default="all", type=str, help='default: "all"') def snapshot(**kwargs): """Manage aptly snapshots.""" from . import main, snapshot @@ -157,7 +157,7 @@ def snapshot(**kwargs): ) @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True)) @click.argument("task", type=click.Choice(["create", "update"])) -@click.option("--publish-name", "-n", default="all", type=str, help='deafult: "all"') +@click.option("--publish-name", "-n", default="all", type=str, help='default: "all"') def publish(**kwargs): """Manage aptly publishs.""" from . import main, publish diff --git a/pyaptly/command.py b/pyaptly/command.py index d187ff3..189126c 100644 --- a/pyaptly/command.py +++ b/pyaptly/command.py @@ -3,8 +3,6 @@ import collections import logging -from frozendict import frozendict - from . import state_reader, util lg = logging.getLogger(__name__) @@ -27,7 +25,6 @@ def __init__(self, cmd: list[str]): self._finished: bool = False self._known_dependency_types = ( "snapshot", - "mirror", "repo", "publish", "virtual", @@ -84,6 +81,21 @@ def provide(self, type_, identifier): assert type_ in self._known_dependency_types self._provides.add((type_, str(identifier))) + def clear_caches(self): + """Clear state_reader caches of functions which have changed""" + provides = set(p[0] for p in self.get_provides()) + for provide in provides: + lg.debug("clearing cache for " + provide) + match provide: + case "snapshot": + state_reader.state_reader().snapshots.cache_clear() + state_reader.state_reader().snapshot_map.cache_clear() + case "repo": + state_reader.state_reader().repos.cache_clear() + case "publish": + state_reader.state_reader().publishes.cache_clear() + state_reader.state_reader().publish_map.cache_clear() + def execute(self): """Execute the command. Return the return value of the command. @@ -103,6 +115,7 @@ def execute(self): # So I decided to change that. For now we fail hard if a `Command` fails. # I guess we will see in production what happens. util.run_command(self.cmd, check=True) + self.clear_caches() else: lg.info("Pretending to run command: %s", " ".join(self.cmd)) self._finished = True @@ -269,8 +282,6 @@ def order_commands(commands, has_dependency_cb=lambda x: False): # Break out of the requirements loop, as the # command cannot be scheduled anyway. break - # command cannot be scheduled anyway. - break if can_schedule: lg.debug("%s: all dependencies fulfilled" % cmd) @@ -297,70 +308,42 @@ def order_commands(commands, has_dependency_cb=lambda x: False): return scheduled -class FunctionCommand(Command): - """Repesents a function command. +class DummyCommand(Command): + """Represents a dummy command. - Is used to resolve dependencies between such commands. This command executes - the given function. *args and **kwargs are passed through. + Is used to resolve dependencies between commands, but does nothing itself :param func: The function to execute :type func: callable """ - def __init__(self, func, *args, **kwargs): + def __init__(self, identifier: str): super().__init__([]) - - assert callable(func) - self.cmd = [str(id(func))] - self.func = func - self.args = args - self.kwargs = kwargs + self.identifier = identifier def freeze(self): """Freeze the class to make it hashable.""" self._freeze_common() - # manually checking using self.frozen - self.kwargs = frozendict(self.kwargs) # type: ignore def __hash__(self): """Hash the class.""" dependencies_hash = self._hash_base() - return hash((id(self.func), self.args, self.kwargs, dependencies_hash)) + return hash((self.identifier, dependencies_hash)) def __eq__(self, other): """Compare the class.""" - return ( - self._eq_base(other) - and id(self.func) == id(other.func) - and self.args == other.args - and self.kwargs == other.kwargs - ) + return self._eq_base(other) and self.identifier == other.identifier def execute(self): - """Execute the command. - - Call the function. - """ + """Mark command as executed""" if self._finished: # pragma: no cover return self._finished if not Command.pretend_mode: - lg.debug( - "Running code: %s(args=%s, kwargs=%s)", - self.func.__name__, - repr(self.args), - repr(self.kwargs), - ) - - self.func(*self.args, **self.kwargs) + lg.debug("Running dummy Command with provides %s", self._provides) self._finished = True else: # pragma: no cover - lg.info( - "Pretending to run code: %s(args=%s, kwargs=%s)", - self.repr_cmd(), - repr(self.args), - repr(self.kwargs), - ) + lg.info("Pretending to run dummy Command with provides: %s", self._provides) return self._finished @@ -369,13 +352,11 @@ def repr_cmd(self): :rtype: str """ - # We need to "id" ourselves here so that multiple commands that call a - # function with the same name won't be shown as being equal. - return "%s|%s" % (self.func.__name__, id(self)) + return self.identifier def __repr__(self): - """Show repr for FunctionCommand.""" - return "FunctionCommand<%s requires %s, provides %s>\n" % ( + """Show repr for DummyCommand.""" + return "DummyCommand<%s requires %s, provides %s>\n" % ( self.repr_cmd(), ", ".join([repr(x) for x in self._requires]), ", ".join([repr(x) for x in self._provides]), diff --git a/pyaptly/conftest.py b/pyaptly/conftest.py index 6eda42a..2e383b5 100644 --- a/pyaptly/conftest.py +++ b/pyaptly/conftest.py @@ -65,6 +65,14 @@ def environment(debug_mode): os.environ["GNUPGHOME"] = str(gnupg) util._PYTEST_KEYSERVER = "hkp://127.0.0.1:8080" + # Make sure we start with a clean slate + state_reader.state_reader().mirrors.cache_clear() + state_reader.state_reader().snapshots.cache_clear() + state_reader.state_reader().snapshot_map.cache_clear() + state_reader.state_reader().repos.cache_clear() + state_reader.state_reader().publishes.cache_clear() + state_reader.state_reader().publish_map.cache_clear() + try: yield finally: @@ -78,6 +86,7 @@ def test_key_03(environment): """Get test gpg-key number 3.""" util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True) util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True) + state_reader.state_reader().gpg_keys.cache_clear() @pytest.fixture() @@ -120,11 +129,9 @@ def mirror_update(environment, config): """Test if updating mirrors works.""" args = ["-c", config, "mirror", "create"] state = state_reader.SystemStateReader() - state.read() - assert "fakerepo01" not in state.mirrors + assert "fakerepo01" not in state.mirrors() main.main(args) - state.read() - assert "fakerepo01" in state.mirrors + assert "fakerepo01" in state.mirrors() args[3] = "update" main.main(args) args = [ @@ -144,9 +151,8 @@ def snapshot_create(config, mirror_update, freeze): args = ["-c", config, "snapshot", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() assert set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]).issubset( - state.snapshots + state.snapshots() ) yield state @@ -162,14 +168,13 @@ def snapshot_update_rotating(config, mirror_update, freeze): ] main.main(args) state = state_reader.SystemStateReader() - state.read() assert set( [ "fake-current", "fakerepo01-current", "fakerepo02-current", ] - ).issubset(state.snapshots) + ).issubset(state.snapshots()) args = [ "-c", config, @@ -177,14 +182,13 @@ def snapshot_update_rotating(config, mirror_update, freeze): "update", ] main.main(args) - state.read() assert set( [ "fake-current", "fakerepo01-current-rotated-20121010T1010Z", "fakerepo02-current-rotated-20121010T1010Z", ] - ).issubset(state.snapshots) + ).issubset(state.snapshots()) expected = { "fake-current": set(["fakerepo01-current", "fakerepo02-current"]), "fake-current-rotated-20121010T1010Z": set( @@ -198,7 +202,7 @@ def snapshot_update_rotating(config, mirror_update, freeze): "fakerepo02-current": set([]), "fakerepo02-current-rotated-20121010T1010Z": set([]), } - assert state.snapshot_map == expected + assert state.snapshot_map() == expected @pytest.fixture() @@ -207,7 +211,6 @@ def repo_create(environment, config, test_key_03): args = ["-c", config, "repo", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() util.run_command( [ "aptly", @@ -217,7 +220,8 @@ def repo_create(environment, config, test_key_03): "/source/compose/setup/hellome_0.1-1_amd64.deb", ] ) - assert set(["centrify"]) == state.repos + state_reader.state_reader().repos.cache_clear() + assert set(["centrify"]) == state.repos() @pytest.fixture() @@ -226,13 +230,12 @@ def publish_create(config, snapshot_create, test_key_03): args = ["-c", config, "publish", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes + assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes() expect = { "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]), "fakerepo01 main": set(["fakerepo01-20121010T0000Z"]), } - assert expect == state.publish_map + assert expect == state.publish_map() @pytest.fixture() @@ -241,7 +244,6 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03): args = ["-c", config, "publish", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() assert ( set( [ @@ -250,14 +252,14 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03): "fakerepo02/current stable", ] ) - == state.publishes + == state.publishes() ) expect = { "fake/current stable": set(["fake-current"]), "fakerepo01/current stable": set(["fakerepo01-current"]), "fakerepo02/current stable": set(["fakerepo02-current"]), } - assert expect == state.publish_map + assert expect == state.publish_map() @pytest.fixture() @@ -277,5 +279,4 @@ def publish_create_republish(config, publish_create, caplog): ] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert "fakerepo01-stable main" in state.publishes + assert "fakerepo01-stable main" in state.publishes() diff --git a/pyaptly/main.py b/pyaptly/main.py index 03cf44d..8da90a0 100755 --- a/pyaptly/main.py +++ b/pyaptly/main.py @@ -13,7 +13,6 @@ publish, repo, snapshot, - state_reader, ) _logging_setup = False @@ -48,7 +47,6 @@ def prepare(args): with open(args.config, "rb") as f: cfg = tomli.load(f) - state_reader.state_reader().read() return cfg diff --git a/pyaptly/mirror.py b/pyaptly/mirror.py index 26918a5..dfbc1ad 100644 --- a/pyaptly/mirror.py +++ b/pyaptly/mirror.py @@ -33,7 +33,7 @@ def add_gpg_keys(mirror_config): keys_urls[key] = None for key in keys_urls.keys(): - if key in state_reader.state_reader().gpg_keys: + if key in state_reader.state_reader().gpg_keys(): continue try: key_command = [ @@ -59,7 +59,7 @@ def add_gpg_keys(mirror_config): util.run_command(["bash", "-c", key_shell], check=True) else: raise - state_reader.state_reader().read_gpg() + state_reader.state_reader().gpg_keys.cache_clear() def mirror(cfg, args): @@ -102,7 +102,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config): :param mirror_config: Configuration of the snapshot from the toml file. :type mirror_config: dict """ - if mirror_name in state_reader.state_reader().mirrors: # pragma: no cover + if mirror_name in state_reader.state_reader().mirrors(): # pragma: no cover return add_gpg_keys(mirror_config) @@ -130,6 +130,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config): lg.debug("Running command: %s", " ".join(aptly_cmd)) util.run_command(aptly_cmd, check=True) + state_reader.state_reader().mirrors.cache_clear() def cmd_mirror_update(cfg, mirror_name, mirror_config): @@ -142,7 +143,7 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config): :param mirror_config: Configuration of the snapshot from the toml file. :type mirror_config: dict """ - if mirror_name not in state_reader.state_reader().mirrors: # pragma: no cover + if mirror_name not in state_reader.state_reader().mirrors(): # pragma: no cover raise Exception("Mirror not created yet") add_gpg_keys(mirror_config) aptly_cmd = ["aptly", "mirror", "update"] @@ -152,3 +153,4 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config): aptly_cmd.append(mirror_name) lg.debug("Running command: %s", " ".join(aptly_cmd)) util.run_command(aptly_cmd, check=True) + state_reader.state_reader().mirrors.cache_clear() diff --git a/pyaptly/publish.py b/pyaptly/publish.py index 0bd62a7..8f914af 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -80,10 +80,12 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) if "repo" in publish_config: publish_cmd.append("update") - return command.Command(publish_cmd + options + args) + cmd = command.Command(publish_cmd + options + args) + cmd.provide("publish", publish_name) + return cmd publish_fullname = "%s %s" % (publish_name, publish_config["distribution"]) - current_snapshots = state_reader.state_reader().publish_map[publish_fullname] + current_snapshots = state_reader.state_reader().publish_map()[publish_fullname] if "snapshots" in publish_config: snapshots_config = publish_config["snapshots"] new_snapshots = [ @@ -97,7 +99,7 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) if publish["distribution"] == distribution: snapshots_config.extend(publish["snapshots"]) break - new_snapshots = list(state_reader.state_reader().publish_map[conf_value]) + new_snapshots = list(state_reader.state_reader().publish_map()[conf_value]) else: # pragma: no cover raise ValueError( "No snapshot references configured in publish %s" % publish_name @@ -122,7 +124,7 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) "%T", date_tools.format_timestamp(datetime.datetime.now()) ) if ( - archive in state_reader.state_reader().snapshots + archive in state_reader.state_reader().snapshots() ): # pragma: no cover continue prefix_to_search = re.sub("%T$", "", snap["name"]) @@ -141,7 +143,9 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) if "skip-contents" in publish_config and publish_config["skip-contents"]: options.append("-skip-contents=true") - return command.Command(publish_cmd + options + args + new_snapshots) + cmd = command.Command(publish_cmd + options + args + new_snapshots) + cmd.provide("publish", publish_fullname) + return cmd def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False): @@ -158,7 +162,7 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False) """ publish_fullname = "%s %s" % (publish_name, publish_config["distribution"]) if ( - publish_fullname in state_reader.state_reader().publishes + publish_fullname in state_reader.state_reader().publishes() and not ignore_existing ): # Nothing to do, publish already created @@ -233,7 +237,7 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False) conf_value = " ".join(conf_value.split("/")) source_args.append("snapshot") try: - sources = state_reader.state_reader().publish_map[conf_value] + sources = state_reader.state_reader().publish_map()[conf_value] except KeyError: lg.critical( ( @@ -256,4 +260,6 @@ def publish_cmd_create(cfg, publish_name, publish_config, ignore_existing=False) assert has_source assert len(components) == num_sources - return command.Command(publish_cmd + options + source_args + endpoint_args) + cmd = command.Command(publish_cmd + options + source_args + endpoint_args) + cmd.provide("publish", publish_fullname) + return cmd diff --git a/pyaptly/repo.py b/pyaptly/repo.py index d239ab2..661dead 100644 --- a/pyaptly/repo.py +++ b/pyaptly/repo.py @@ -57,7 +57,7 @@ def repo_cmd_create(cfg, repo_name, repo_config): :param repo_config: Configuration of the repo from the toml file. :type repo_config: dict """ - if repo_name in state_reader.state_reader().repos: # pragma: no cover + if repo_name in state_reader.state_reader().repos(): # pragma: no cover # Nothing to do, repo already created return @@ -86,4 +86,6 @@ def repo_cmd_create(cfg, repo_name, repo_config): ) ) - return command.Command(repo_cmd + options + endpoint_args) + cmd = command.Command(repo_cmd + options + endpoint_args) + cmd.provide("repo", repo_name) + return cmd diff --git a/pyaptly/snapshot.py b/pyaptly/snapshot.py index d574037..a8c1b83 100644 --- a/pyaptly/snapshot.py +++ b/pyaptly/snapshot.py @@ -115,9 +115,7 @@ def dependents_of_snapshot(snapshot_name): :rtype: generator """ - for dependent in state_reader.state_reader().snapshot_map.get( - snapshot_name, [] - ): + for dependent in state_reader.state_reader().snapshot_map().get(snapshot_name, []): yield dependent # TODO I fixed a bug, but there is no test. We do not test recursive dependants yield from dependents_of_snapshot(dependent) @@ -142,14 +140,13 @@ def rotate_snapshot(cfg, snapshot_name): # First, verify that our snapshot environment is in a sane state_reader.state. # Fixing the environment is not currently our task. - if rotated_name in state_reader.state_reader().snapshots: # pragma: no cover + if rotated_name in state_reader.state_reader().snapshots(): # pragma: no cover raise Exception( "Cannot update snapshot %s - rotated name %s already exists" % (snapshot_name, rotated_name) ) cmd = command.Command(["aptly", "snapshot", "rename", snapshot_name, rotated_name]) - cmd.provide("virtual", rotated_name) return cmd @@ -181,13 +178,12 @@ def cmd_snapshot_update( affected_snapshots.extend(list(dependents_of_snapshot(snapshot_name))) # TODO: rotated snapshots should be identified by configuration option, not - # just by "not being timestamped + # just by "not being timestamped" rename_cmds = [rotate_snapshot(cfg, snap) for snap in affected_snapshots] - # The "intermediate" command causes the state reader to refresh. At the - # same time, it provides a collection point for dependency handling. - intermediate = command.FunctionCommand(state_reader.state_reader().read) + # The "intermediate" command is used as collection point for dependencies + intermediate = command.DummyCommand("intermediate") intermediate.provide("virtual", "all-snapshots-rotated") for cmd in rename_cmds: @@ -201,8 +197,7 @@ def cmd_snapshot_update( intermediate.require("virtual", provide) # Same as before - create a focal point to "collect" dependencies - # after the snapshots have been rebuilt. Also reload state once again - intermediate2 = command.FunctionCommand(state_reader.state_reader().read) + intermediate2 = command.DummyCommand("intermediate2") intermediate2.provide("virtual", "all-snapshots-rebuilt") create_cmds = [] @@ -247,7 +242,7 @@ def cmd_snapshot_update( def is_publish_affected(name, publish_info): if ( "%s %s" % (name, publish_info["distribution"]) - in state_reader.state_reader().publishes + in state_reader.state_reader().publishes() ): try: for snap in publish_info["snapshots"]: @@ -318,10 +313,7 @@ def cmd_snapshot_create( snapshot_name = date_tools.expand_timestamped_name(snapshot_name, snapshot_config) - if ( - snapshot_name in state_reader.state_reader().snapshots - and not ignore_existing - ): + if snapshot_name in state_reader.state_reader().snapshots() and not ignore_existing: return [] default_aptly_cmd = ["aptly", "snapshot", "create"] diff --git a/pyaptly/state_reader.py b/pyaptly/state_reader.py index f4a66c2..5a62fcb 100644 --- a/pyaptly/state_reader.py +++ b/pyaptly/state_reader.py @@ -2,6 +2,7 @@ import logging import re +from functools import lru_cache from . import util @@ -13,19 +14,11 @@ class SystemStateReader(object): To find out what operations have to be performed to reach the state defined in the toml config-file. + Functions are cached and execution commands clear the cache of functions which need to be rerun """ known_dependency_types = ("repo", "snapshot", "mirror", "gpg_key") - def __init__(self): - self.gpg_keys = set() - self.mirrors = set() - self.repos = set() - self.snapshots = set() - self.snapshot_map = {} - self.publishes = set() - self.publish_map = {} - def _extract_sources(self, data): """Extract sources from data. @@ -50,19 +43,10 @@ def _extract_sources(self, data): return sources - def read(self): - """Read all available system states.""" - self.read_gpg() - self.read_repos() - self.read_mirror() - self.read_snapshot() - self.read_snapshot_map() - self.read_publishes() - self.read_publish_map() - - def read_gpg(self): - """Read all trusted keys in gpg.""" - self.gpg_keys = set() + @lru_cache(maxsize=None) + def gpg_keys(self): + """Read all trusted keys in gp and cache in lru_cache.""" + gpg_keys = set() cmd = [ "gpg", "--no-default-keyring", @@ -77,65 +61,71 @@ def read_gpg(self): if field[0] in ("pub", "sub"): key = field[4] key_short = key[8:] - self.gpg_keys.add(key) - self.gpg_keys.add(key_short) - - def read_publish_map(self): - """Create a publish map. publish -> snapshots.""" - self.publish_map = {} - # match example: main: test-snapshot [snapshot] + gpg_keys.add(key) + gpg_keys.add(key_short) + return gpg_keys + + @lru_cache(maxsize=None) + def publish_map(self): + """Create a publish map. publish -> snapshots. + Cached in the lru_cache. + """ + publish_map = {} re_snap = re.compile(r"\s+[\w\d-]+\:\s([\w\d-]+)\s\[snapshot\]") - for publish in self.publishes: + for publish in self.publishes(): prefix, dist = publish.split(" ") cmd = ["aptly", "publish", "show", dist, prefix] result = util.run_command(cmd, stdout=util.PIPE, check=True) sources = self._extract_sources(result.stdout) matches = [re_snap.match(source) for source in sources] snapshots = [match.group(1) for match in matches if match] - self.publish_map[publish] = set(snapshots) + publish_map[publish] = set(snapshots) - lg.debug("Joined snapshots and publishes: %s", self.publish_map) + lg.debug("Joined snapshots and publishes: %s", publish_map) + return publish_map - def read_snapshot_map(self): + @lru_cache(maxsize=None) + def snapshot_map(self): """Create a snapshot map. snapshot -> snapshots. - This is also called merge-tree. + Cached in the lru_cache. """ - self.snapshot_map = {} + snapshot_map = {} # match example: test-snapshot [snapshot] re_snap = re.compile(r"\s+([\w\d-]+)\s\[snapshot\]") - for snapshot_outer in self.snapshots: + for snapshot_outer in self.snapshots(): cmd = ["aptly", "snapshot", "show", snapshot_outer] result = util.run_command(cmd, stdout=util.PIPE, check=True) sources = self._extract_sources(result.stdout) matches = [re_snap.match(source) for source in sources] snapshots = [match.group(1) for match in matches if match] - self.snapshot_map[snapshot_outer] = set(snapshots) + snapshot_map[snapshot_outer] = set(snapshots) - lg.debug("Joined snapshots with self(snapshots): %s", self.snapshot_map) + lg.debug("Joined snapshots with self(snapshots): %s", snapshot_map) + return snapshot_map - def read_publishes(self): - """Read all available publishes.""" - self.publishes = set() - self.read_aptly_list("publish", self.publishes) + @lru_cache(maxsize=None) + def publishes(self): + """Read all available publishes and cache in lru_cache""" + return self.read_aptly_list("publish") - def read_repos(self): - """Read all available repos.""" - self.repos = set() - self.read_aptly_list("repo", self.repos) + @lru_cache(maxsize=None) + def repos(self): + """Read all available repo and cache in lru_cache.""" + return self.read_aptly_list("repo") - def read_mirror(self): - """Read all available mirrors.""" - self.mirrors = set() - self.read_aptly_list("mirror", self.mirrors) + @lru_cache(maxsize=None) + def mirrors(self): + """Read all available mirror and cache in lru_cache.""" + return self.read_aptly_list("mirror") - def read_snapshot(self): - """Read all available snapshots.""" - self.snapshots = set() - self.read_aptly_list("snapshot", self.snapshots) + @lru_cache(maxsize=None) + def snapshots(self): + """Read all available snapshot and cache in lru_cache.""" + return self.read_aptly_list("snapshot") - def read_aptly_list(self, type_, list_): + def read_aptly_list(self, type_): """Read lists from aptly. :param type_: The type of list to read ie. snapshot @@ -144,11 +134,13 @@ def read_aptly_list(self, type_, list_): :param list_: list """ cmd = ["aptly", type_, "list", "-raw"] + clean_lines = set() result = util.run_command(cmd, stdout=util.PIPE, check=True) for line in result.stdout.split("\n"): clean_line = line.strip() if clean_line: - list_.add(clean_line) + clean_lines.add(clean_line) + return clean_lines def has_dependency(self, dependency): """Check system state dependencies. @@ -159,13 +151,13 @@ def has_dependency(self, dependency): type_, name = dependency if type_ == "repo": # pragma: no cover - return name in self.repos + return name in self.repos() if type_ == "mirror": # pragma: no cover - return name in self.mirrors + return name in self.mirrors() elif type_ == "snapshot": - return name in self.snapshots # pragma: no cover + return name in self.snapshots() # pragma: no cover elif type_ == "gpg_key": # pragma: no cover - return name in self.gpg_keys # Not needed ATM + return name in self.gpg_keys() # Not needed ATM elif type_ == "virtual": # virtual dependencies can never be resolved by the # system state reader - they are used for internal diff --git a/pyaptly/tests/test_graph.py b/pyaptly/tests/test_graph.py index 8fb4d7e..4194df6 100644 --- a/pyaptly/tests/test_graph.py +++ b/pyaptly/tests/test_graph.py @@ -1,7 +1,6 @@ """Testing dependency graphs.""" import random -from functools import partial from typing import Union from hypothesis import given, settings @@ -105,16 +104,10 @@ def run_graph(tree): commands = [] index = list(range(len(tree[0]))) random.shuffle(index) - cmd: Union[command.Command, command.FunctionCommand] + cmd: Union[command.Command, command.DummyCommand] for i in index: - - def dummy(i): # pragma: no cover - return i - if tree[2][i]: - func = partial(dummy, i) - func.__name__ = dummy.__name__ # type: ignore - cmd = command.FunctionCommand(func) + cmd = command.DummyCommand("dummy %s" % i) else: cmd = command.Command([str(i)]) for provides in tree[0][i]: diff --git a/pyaptly/tests/test_mirror.py b/pyaptly/tests/test_mirror.py index 22efed4..12a0571 100644 --- a/pyaptly/tests/test_mirror.py +++ b/pyaptly/tests/test_mirror.py @@ -43,8 +43,7 @@ def test_mirror_create(environment, config, caplog): assert len(keys_added) == len(set(keys_added)), "Key multiple times added" state = state_reader.SystemStateReader() - state.read() - assert state.mirrors == {"fakerepo03"} + assert state.mirrors() == {"fakerepo03"} @pytest.mark.parametrize("config", ["mirror-basic.toml"], indirect=True) diff --git a/pyaptly/tests/test_publish.py b/pyaptly/tests/test_publish.py index 06bd410..ad54926 100644 --- a/pyaptly/tests/test_publish.py +++ b/pyaptly/tests/test_publish.py @@ -16,10 +16,9 @@ def test_publish_create_single(config, snapshot_create, test_key_03, repo): return main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set(["fakerepo01 main"]) == state.publishes + assert set(["fakerepo01 main"]) == state.publishes() expect = {"fakerepo01 main": set(["fakerepo01-20121010T0000Z"])} - assert expect == state.publish_map + assert expect == state.publish_map() @pytest.mark.parametrize("config", ["publish.toml"], indirect=True) @@ -53,9 +52,8 @@ def test_pretend(config, snapshot_create, test_key_03): ] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set() == state.publishes - assert {} == state.publish_map + assert set() == state.publishes() + assert {} == state.publish_map() assert command.Command.pretend_mode @@ -77,9 +75,8 @@ def test_publish_create_repo(config, repo_create): ] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set(["centrify latest"]) == state.publishes - assert {"centrify latest": set([])} == state.publish_map + assert set(["centrify latest"]) == state.publishes() + assert {"centrify latest": set([])} == state.publish_map() @pytest.mark.parametrize("config", ["publish.toml"], indirect=True) @@ -96,13 +93,12 @@ def test_publish_update_rotating(config, freeze, publish_create_rotating, via): args = ["-c", config, via, "update"] main.main(args) state = state_reader.SystemStateReader() - state.read() expect = { "fake/current stable": set(["fake-current"]), "fakerepo01/current stable": set(["fakerepo01-current"]), "fakerepo02/current stable": set(["fakerepo02-current"]), } - assert expect == state.publish_map + assert expect == state.publish_map() if via == "snapshot": expect2 = { "fakerepo02-current", @@ -115,7 +111,7 @@ def test_publish_update_rotating(config, freeze, publish_create_rotating, via): "fake-current-rotated-20121010T1010Z", "fake-current-rotated-20121011T1010Z", } - assert expect2 == state.snapshots + assert expect2 == state.snapshots() @pytest.mark.parametrize("config", ["publish-current.toml"], indirect=True) @@ -139,8 +135,7 @@ def test_publish_update_republish(config, publish_create_republish, freeze): args = ["-c", config, "publish", "update"] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert "fakerepo01-stable main" in state.publishes + assert "fakerepo01-stable main" in state.publishes() # As you see fakerepo01-stable main points to the old snapshot # this is theoretically not correct, but it will be fixed with # the next call to publish update. Since we use this from a hourly cron @@ -152,7 +147,7 @@ def test_publish_update_republish(config, publish_create_republish, freeze): "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]), "fakerepo01 main": set(["fakerepo01-20121011T0000Z"]), } - assert expect == state.publish_map + assert expect == state.publish_map() @pytest.mark.parametrize("config", ["publish.toml"], indirect=True) @@ -164,7 +159,6 @@ def test_publish_updating_basic(config, publish_create, freeze): args = ["-c", config, "publish", "update"] main.main(args) state = state_reader.SystemStateReader() - state.read() expect = set( [ "archived-fakerepo01-20121011T1010Z", @@ -173,12 +167,12 @@ def test_publish_updating_basic(config, publish_create, freeze): "fakerepo01-20121010T0000Z", ] ) - assert expect == state.snapshots + assert expect == state.snapshots() expect2 = { "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]), "fakerepo01 main": set(["fakerepo01-20121011T0000Z"]), } - assert expect2 == state.publish_map + assert expect2 == state.publish_map() @pytest.mark.parametrize("repo", ["centrify", "asdfasdf"]) @@ -192,5 +186,4 @@ def test_repo_create_single(config, repo, test_key_03): return main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set(["centrify"]) == state.repos + assert set(["centrify"]) == state.repos() diff --git a/pyaptly/tests/test_snapshot.py b/pyaptly/tests/test_snapshot.py index a97fbb5..66791cf 100644 --- a/pyaptly/tests/test_snapshot.py +++ b/pyaptly/tests/test_snapshot.py @@ -10,7 +10,7 @@ def test_snapshot_create_basic(config, snapshot_create): """Test if snapshot create works.""" assert ( set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]) - == snapshot_create.snapshots + == snapshot_create.snapshots() ) @@ -36,14 +36,13 @@ def test_snapshot_create_rotating(mirror_update, config): args = ["-c", config, "snapshot", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() assert set( [ "fake-current", "fakerepo01-current", "fakerepo02-current", ] - ).issubset(state.snapshots) + ).issubset(state.snapshots()) @pytest.mark.parametrize("config", ["snapshot-current.toml"], indirect=True) @@ -64,7 +63,6 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f ] main.main(args) state = state_reader.SystemStateReader() - state.read() assert set( [ "fake-current", @@ -73,7 +71,7 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f "fakerepo01-current-rotated-20121011T1010Z", "fakerepo02-current-rotated-20121011T1010Z", ] - ).issubset(state.snapshots) + ).issubset(state.snapshots()) expected = { "fake-current": set(["fakerepo01-current", "fakerepo02-current"]), "fake-current-rotated-20121010T1010Z": set( @@ -95,7 +93,7 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f "fakerepo02-current-rotated-20121010T1010Z": set([]), "fakerepo02-current-rotated-20121011T1010Z": set([]), } - assert state.snapshot_map == expected + assert state.snapshot_map() == expected freeze.move_to("2012-10-12 10:10:10") args = [ @@ -106,7 +104,6 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f ] main.main(args) state = state_reader.SystemStateReader() - state.read() assert set( [ "fake-current", @@ -115,7 +112,7 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f "fakerepo01-current-rotated-20121012T1010Z", "fakerepo02-current-rotated-20121012T1010Z", ] - ).issubset(state.snapshots) + ).issubset(state.snapshots()) expected = { "fake-current": set(["fakerepo01-current", "fakerepo02-current"]), "fake-current-rotated-20121010T1010Z": set( @@ -145,7 +142,7 @@ def test_snapshot_update_threetimes_rotating(snapshot_update_rotating, config, f "fakerepo02-current-rotated-20121011T1010Z": set([]), "fakerepo02-current-rotated-20121012T1010Z": set([]), } - assert state.snapshot_map == expected + assert state.snapshot_map() == expected @pytest.mark.parametrize("config", ["snapshot-repo.toml"], indirect=True) @@ -154,9 +151,7 @@ def test_snapshot_create_repo(config, repo_create): args = ["-c", config, "snapshot", "create"] main.main(args) state = state_reader.SystemStateReader() - state.read() - assert set(["centrify-latest"]).issubset(state.snapshots) - return state + assert set(["centrify-latest"]).issubset(state.snapshots()) @pytest.mark.parametrize("config", ["snapshot-merge.toml"], indirect=True) @@ -170,7 +165,7 @@ def test_snapshot_create_merge(config, snapshot_create): "superfake-20121010T0000Z", ] ) - == snapshot_create.snapshots + == snapshot_create.snapshots() ) expect = { "fakerepo01-20121010T0000Z": set([]), @@ -179,7 +174,7 @@ def test_snapshot_create_merge(config, snapshot_create): ["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"] ), } - assert expect == snapshot_create.snapshot_map + assert expect == snapshot_create.snapshot_map() @pytest.mark.parametrize("config", ["snapshot-filter.toml"], indirect=True)