feat: Replace SystemStateReader.read() with lru_cache

pyaptly/cli.py

@@ -88,7 +88,7 @@ def legacy(passthrough):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create"]))
-@click.option("--repo-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--repo-name", "-n", default="all", type=str, help='default: "all"')
 def repo(**kwargs):
     """Create aptly repos."""
     from . import main, repo
@@ -111,7 +111,7 @@ def repo(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--mirror-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--mirror-name", "-n", default="all", type=str, help='default: "all"')
 def mirror(**kwargs):
     """Manage aptly mirrors."""
     from . import main, mirror
@@ -134,7 +134,7 @@ def mirror(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--snapshot-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--snapshot-name", "-n", default="all", type=str, help='default: "all"')
 def snapshot(**kwargs):
     """Manage aptly snapshots."""
     from . import main, snapshot
@@ -157,7 +157,7 @@ def snapshot(**kwargs):
 )
 @click.argument("config", type=click.Path(file_okay=True, dir_okay=False, exists=True))
 @click.argument("task", type=click.Choice(["create", "update"]))
-@click.option("--publish-name", "-n", default="all", type=str, help='deafult: "all"')
+@click.option("--publish-name", "-n", default="all", type=str, help='default: "all"')
 def publish(**kwargs):
     """Manage aptly publishs."""
     from . import main, publish

pyaptly/command.py

@@ -84,6 +84,23 @@ def provide(self, type_, identifier):
         assert type_ in self._known_dependency_types
         self._provides.add((type_, str(identifier)))
 
+    def clear_caches(self):
+        """Clear state_reader caches of functions which have changed"""
+        provides = set(p[0] for p in self.get_provides())
+        for provide in provides:
+            lg.debug("clearing cache for " + provide)
+            match provide:
+                case "mirrors":
+                    state_reader.state_reader().mirrors.cache_clear()
+                case "snapshot":
+                    state_reader.state_reader().snapshots.cache_clear()
+                    state_reader.state_reader().snapshot_map.cache_clear()
+                case "repo":
+                    state_reader.state_reader().repos.cache_clear()
+                case "publish":
+                    state_reader.state_reader().publishes.cache_clear()
+                    state_reader.state_reader().publish_map.cache_clear()
+
     def execute(self):
         """Execute the command. Return the return value of the command.
 
@@ -103,6 +120,7 @@ def execute(self):
             # So I decided to change that. For now we fail hard if a `Command` fails.
             # I guess we will see in production what happens.
             util.run_command(self.cmd, check=True)
+            self.clear_caches()
         else:
             lg.info("Pretending to run command: %s", " ".join(self.cmd))
         self._finished = True
@@ -269,8 +287,6 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
                             # Break out of the requirements loop, as the
                             # command cannot be scheduled anyway.
                             break
-                            # command cannot be scheduled anyway.
-                            break
 
                 if can_schedule:
                     lg.debug("%s: all dependencies fulfilled" % cmd)
@@ -297,69 +313,50 @@ def order_commands(commands, has_dependency_cb=lambda x: False):
         return scheduled
 
 
-class FunctionCommand(Command):
-    """Repesents a function command.
+class DummyCommand(Command):
+    """Represents a dummy command.
 
-    Is used to resolve dependencies between such commands. This command executes
-    the given function. *args and **kwargs are passed through.
+    Is used to resolve dependencies between commands, but does nothing itself
 
     :param func: The function to execute
     :type  func: callable
     """
 
-    def __init__(self, func, *args, **kwargs):
+    def __init__(self, identifier: str):
         super().__init__([])
-
-        assert callable(func)
-        self.cmd = [str(id(func))]
-        self.func = func
-        self.args = args
-        self.kwargs = kwargs
+        self.identifier = identifier
 
     def freeze(self):
         """Freeze the class to make it hashable."""
         self._freeze_common()
-        # manually checking using self.frozen
-        self.kwargs = frozendict(self.kwargs)  # type: ignore
 
     def __hash__(self):
         """Hash the class."""
         dependencies_hash = self._hash_base()
-        return hash((id(self.func), self.args, self.kwargs, dependencies_hash))
+        return hash((self.identifier, dependencies_hash))
 
     def __eq__(self, other):
         """Compare the class."""
         return (
             self._eq_base(other)
-            and id(self.func) == id(other.func)
-            and self.args == other.args
-            and self.kwargs == other.kwargs
+            and self.identifier == other.identifier
         )
 
     def execute(self):
-        """Execute the command.
-
-        Call the function.
-        """
+        """Mark command as executed"""
         if self._finished:  # pragma: no cover
             return self._finished
         if not Command.pretend_mode:
             lg.debug(
-                "Running code: %s(args=%s, kwargs=%s)",
-                self.func.__name__,
-                repr(self.args),
-                repr(self.kwargs),
+                "Running dummy Command with provides %s",
+                self._provides
             )
 
-            self.func(*self.args, **self.kwargs)
-
             self._finished = True
         else:  # pragma: no cover
             lg.info(
-                "Pretending to run code: %s(args=%s, kwargs=%s)",
-                self.repr_cmd(),
-                repr(self.args),
-                repr(self.kwargs),
+                "Pretending to run dummy Command with provides: %s",
+                self._provides
             )
 
         return self._finished
@@ -369,13 +366,11 @@ def repr_cmd(self):
 
         :rtype: str
         """
-        # We need to "id" ourselves here so that multiple commands that call a
-        # function with the same name won't be shown as being equal.
-        return "%s|%s" % (self.func.__name__, id(self))
+        return self.identifier
 
     def __repr__(self):
-        """Show repr for FunctionCommand."""
-        return "FunctionCommand<%s requires %s, provides %s>\n" % (
+        """Show repr for DummyCommand."""
+        return "DummyCommand<%s requires %s, provides %s>\n" % (
             self.repr_cmd(),
             ", ".join([repr(x) for x in self._requires]),
             ", ".join([repr(x) for x in self._provides]),

pyaptly/conftest.py

@@ -65,6 +65,14 @@ def environment(debug_mode):
     os.environ["GNUPGHOME"] = str(gnupg)
     util._PYTEST_KEYSERVER = "hkp://127.0.0.1:8080"
 
+    # Make sure we start with a clean slate
+    state_reader.state_reader().mirrors.cache_clear()
+    state_reader.state_reader().snapshots.cache_clear()
+    state_reader.state_reader().snapshot_map.cache_clear()
+    state_reader.state_reader().repos.cache_clear()
+    state_reader.state_reader().publishes.cache_clear()
+    state_reader.state_reader().publish_map.cache_clear()
+
     try:
         yield
     finally:
@@ -78,6 +86,7 @@ def test_key_03(environment):
     """Get test gpg-key number 3."""
     util.run_command(["gpg", "--import", setup_base / "test03.key"], check=True)
     util.run_command(["gpg", "--import", setup_base / "test03.pub"], check=True)
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 
 @pytest.fixture()
@@ -120,11 +129,9 @@ def mirror_update(environment, config):
     """Test if updating mirrors works."""
     args = ["-c", config, "mirror", "create"]
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01" not in state.mirrors
+    assert "fakerepo01" not in state.mirrors()
     main.main(args)
-    state.read()
-    assert "fakerepo01" in state.mirrors
+    assert "fakerepo01" in state.mirrors()
     args[3] = "update"
     main.main(args)
     args = [
@@ -144,9 +151,8 @@ def snapshot_create(config, mirror_update, freeze):
     args = ["-c", config, "snapshot", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(["fakerepo01-20121010T0000Z", "fakerepo02-20121006T0000Z"]).issubset(
-        state.snapshots
+        state.snapshots()
     )
     yield state
 
@@ -162,29 +168,27 @@ def snapshot_update_rotating(config, mirror_update, freeze):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current",
             "fakerepo02-current",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     args = [
         "-c",
         config,
         "snapshot",
         "update",
     ]
     main.main(args)
-    state.read()
     assert set(
         [
             "fake-current",
             "fakerepo01-current-rotated-20121010T1010Z",
             "fakerepo02-current-rotated-20121010T1010Z",
         ]
-    ).issubset(state.snapshots)
+    ).issubset(state.snapshots())
     expected = {
         "fake-current": set(["fakerepo01-current", "fakerepo02-current"]),
         "fake-current-rotated-20121010T1010Z": set(
@@ -198,7 +202,7 @@ def snapshot_update_rotating(config, mirror_update, freeze):
         "fakerepo02-current": set([]),
         "fakerepo02-current-rotated-20121010T1010Z": set([]),
     }
-    assert state.snapshot_map == expected
+    assert state.snapshot_map() == expected
 
 
 @pytest.fixture()
@@ -207,7 +211,6 @@ def repo_create(environment, config, test_key_03):
     args = ["-c", config, "repo", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     util.run_command(
         [
             "aptly",
@@ -217,7 +220,8 @@ def repo_create(environment, config, test_key_03):
             "/source/compose/setup/hellome_0.1-1_amd64.deb",
         ]
     )
-    assert set(["centrify"]) == state.repos
+    state_reader.state_reader().repos.cache_clear()
+    assert set(["centrify"]) == state.repos()
 
 
 @pytest.fixture()
@@ -226,13 +230,12 @@ def publish_create(config, snapshot_create, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes
+    assert set(["fakerepo02 main", "fakerepo01 main"]) == state.publishes()
     expect = {
         "fakerepo02 main": set(["fakerepo02-20121006T0000Z"]),
         "fakerepo01 main": set(["fakerepo01-20121010T0000Z"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -241,7 +244,6 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
     args = ["-c", config, "publish", "create"]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
     assert (
         set(
             [
@@ -250,14 +252,14 @@ def publish_create_rotating(config, snapshot_update_rotating, test_key_03):
                 "fakerepo02/current stable",
             ]
         )
-        == state.publishes
+        == state.publishes()
     )
     expect = {
         "fake/current stable": set(["fake-current"]),
         "fakerepo01/current stable": set(["fakerepo01-current"]),
         "fakerepo02/current stable": set(["fakerepo02-current"]),
     }
-    assert expect == state.publish_map
+    assert expect == state.publish_map()
 
 
 @pytest.fixture()
@@ -277,5 +279,4 @@ def publish_create_republish(config, publish_create, caplog):
     ]
     main.main(args)
     state = state_reader.SystemStateReader()
-    state.read()
-    assert "fakerepo01-stable main" in state.publishes
+    assert "fakerepo01-stable main" in state.publishes()

pyaptly/main.py

@@ -48,7 +48,6 @@ def prepare(args):
 
     with open(args.config, "rb") as f:
         cfg = tomli.load(f)
-    state_reader.state_reader().read()
     return cfg
 
 

pyaptly/mirror.py

@@ -33,7 +33,7 @@ def add_gpg_keys(mirror_config):
                 keys_urls[key] = None
 
     for key in keys_urls.keys():
-        if key in state_reader.state_reader().gpg_keys:
+        if key in state_reader.state_reader().gpg_keys():
             continue
         try:
             key_command = [
@@ -59,8 +59,7 @@ def add_gpg_keys(mirror_config):
                 util.run_command(["bash", "-c", key_shell], check=True)
             else:
                 raise
-    state_reader.state_reader().read_gpg()
-
+    state_reader.state_reader().gpg_keys.cache_clear()
 
 def mirror(cfg, args):
     """Create mirror commands, orders and executes them.
@@ -102,7 +101,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config):
     :param mirror_config: Configuration of the snapshot from the toml file.
     :type  mirror_config: dict
     """
-    if mirror_name in state_reader.state_reader().mirrors:  # pragma: no cover
+    if mirror_name in state_reader.state_reader().mirrors():  # pragma: no cover
         return
 
     add_gpg_keys(mirror_config)
@@ -130,6 +129,7 @@ def cmd_mirror_create(cfg, mirror_name, mirror_config):
 
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()
 
 
 def cmd_mirror_update(cfg, mirror_name, mirror_config):
@@ -142,7 +142,7 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config):
     :param mirror_config: Configuration of the snapshot from the toml file.
     :type  mirror_config: dict
     """
-    if mirror_name not in state_reader.state_reader().mirrors:  # pragma: no cover
+    if mirror_name not in state_reader.state_reader().mirrors():  # pragma: no cover
         raise Exception("Mirror not created yet")
     add_gpg_keys(mirror_config)
     aptly_cmd = ["aptly", "mirror", "update"]
@@ -152,3 +152,4 @@ def cmd_mirror_update(cfg, mirror_name, mirror_config):
     aptly_cmd.append(mirror_name)
     lg.debug("Running command: %s", " ".join(aptly_cmd))
     util.run_command(aptly_cmd, check=True)
+    state_reader.state_reader().mirrors.cache_clear()