Remove copy mechanism for now.

Author: riga

order/__init__.py

@@ -13,7 +13,6 @@
     "Adapter", "Materialized", "DataProvider",
     "UniqueObject", "LazyUniqueObject", "UniqueObjectIndex",
     "DuplicateObjectException", "DuplicateNameException", "DuplicateIdException",
-    "CopyMixin",
     "ProcessIndex", "Process", "LazyProcess",
     "DatasetIndex", "Dataset", "LazyDataset", "DatasetVariation", "GenOrder",
     "Campaign",
@@ -35,7 +34,6 @@
     UniqueObject, LazyUniqueObject, UniqueObjectIndex, DuplicateObjectException,
     DuplicateNameException, DuplicateIdException,
 )
-from order.models.mixins import CopyMixin
 from order.models.process import ProcessIndex, Process, LazyProcess
 from order.models.dataset import DatasetIndex, Dataset, LazyDataset, DatasetVariation, GenOrder
 from order.models.campaign import Campaign

order/__meta__.py

@@ -6,7 +6,7 @@
 
 __author__ = "CMS Collaboration"
 __email__ = "[email protected]"
-__copyright__ = "Copyright 2023, CMS Collaboration"
+__copyright__ = "Copyright 2023-2024, CMS Collaboration"
 __credits__ = ["Marcel Rieger", "Davide Valsecchi"]
 __contact__ = "https://github.com/cms-cat/order"
 __license__ = "BSD-3-Clause"

order/adapters/base.py

@@ -236,6 +236,9 @@ def check_cache(
         cre = re.compile(rf"^{h}(|_\d+)\.json$")
 
         def find(directory: str, ts: int, invalidate: bool) -> str | None:
+            if not os.path.exists(directory):
+                return None
+
             files = {
                 int(m.group(1)[1:] or 0): os.path.join(directory, elem)
                 for elem in os.listdir(directory)

order/magics.py

@@ -6,10 +6,8 @@
 
 from __future__ import annotations
 
-
 __all__ = []
 
-
 import argparse
 import shlex
 

order/models/base.py

@@ -6,10 +6,8 @@
 
 from __future__ import annotations
 
-
 __all__ = ["BaseModel", "Model", "AdapterModel"]
 
-
 from contextlib import contextmanager
 
 from pydantic import BaseModel as PDBaseModel, ConfigDict, Field
@@ -168,10 +166,6 @@ class BaseModel(PDBaseModel):
     def __init__(self: BaseModel, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 
-        # flag that is True for the duration of the model_copy base call to better control __copy__
-        # and __deepcopy__ depending on whether mode_copy or another object triggered the copy
-        self._copy_triggered_by_model = False
-
         # setup objects
         self._setup_objects()
 
@@ -237,35 +231,6 @@ def __repr_adapter__(self, adapter_model: "AdapterModel") -> str | Repr:
     def __repr_circular__(self) -> str:
         return self.__class__.__name__
 
-    def model_copy(
-        self,
-        *,
-        update: dict[str, Any] | None = None,
-        deep: bool = False,
-    ) -> "BaseModel":
-        # set the copy flag since model_copy triggered the copy
-        orig_copy_flag = self._copy_triggered_by_model
-        self._copy_triggered_by_model = True
-
-        # super call
-        copied = super().model_copy(update=update, deep=deep)
-
-        # reset the copy flag of _this_ and the copied instance
-        copied._copy_triggered_by_model = orig_copy_flag
-        self._copy_triggered_by_model = orig_copy_flag
-
-        # the copied instance already contains updated variables, but pydantic does not re-validate
-        # them, so we need to do this manually here
-        if update:
-            for f, v in update.items():
-                with copied._unfreeze_field(f):
-                    setattr(copied, f, v)
-
-        # setup objects
-        copied._setup_objects()
-
-        return copied
-
     def model_show(
         self,
         *,

order/models/campaign.py

@@ -12,7 +12,7 @@
 
 from pydantic import Field
 
-from order.types import Lazy, NonEmptyStrictStr, StrictFloat, Any
+from order.types import Lazy, NonEmptyStrictStr, StrictFloat
 from order.util import has_attr
 from order.models.unique import UniqueObject
 from order.models.dataset import DatasetIndex, Dataset, LazyDataset
@@ -52,28 +52,6 @@ def _setup_datasets(self) -> None:
         for dataset in self.datasets.objects:
             self._dataset_add_callback(dataset)
 
-    def __copy__(self) -> Campaign:
-        copied = super().__copy__()
-
-        # drop references to datasets for consistency as they still point to the original campaign
-        with copied._unfreeze_field("datasets"):
-            copied.datasets = copied.model_fields["datasets"].default_factory()
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
-    def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Campaign:
-        copied = super().__deepcopy__(memo=memo)
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
     def _dataset_materialize_callback(self, dataset: Dataset) -> None:
         dataset.campaign = self
 

order/models/dataset.py

@@ -7,17 +7,15 @@
 
 
 import enum
-import copy
 
 from pydantic import Field, field_validator
 
 from order.types import (
-    Union, List, Dict, NonEmptyStrictStr, PositiveStrictInt, Lazy, ClassVar, GeneratorType, Any,
+    Union, List, Dict, NonEmptyStrictStr, PositiveStrictInt, Lazy, ClassVar, GeneratorType,
 )
-from order.util import has_attr, validated
+from order.util import validated
 from order.models.base import Model
 from order.models.unique import UniqueObjectBase, UniqueObject, LazyUniqueObject, UniqueObjectIndex
-# from order.models.mixins import CopyMixin
 
 
 class DatasetIndex(UniqueObjectIndex):
@@ -83,19 +81,6 @@ def validate_gen_order(cls, gen_order: str) -> str:
         except KeyError:
             raise ValueError(f"unknown gen_order '{gen_order}'")
 
-    def __copy__(self) -> "DatasetVariation":
-        inst = super().__copy__()
-
-        # shallow copy of keys
-        with self._unfreeze_field("keys"):
-            inst.keys = copy.copy(inst.keys)
-
-        # shallow copy of the lfns if not materialized yet
-        if isinstance(inst.lazy_lfns, list):
-            inst.lfns = copy.copy(inst.lfns)
-
-        return inst
-
 
 class Dataset(UniqueObject):
 
@@ -127,42 +112,6 @@ def campaign(self, campaign: "Campaign" | None) -> "Campaign":
 
         return campaign
 
-    def __copy__(self) -> "Dataset":
-        copied = super().__copy__()
-
-        # shallow copy the variations dict
-        with copied._unfreeze_field("variations"):
-            copied.variations = copy.copy(copied.variations)
-
-        # drop the campaign reference
-        copied.campaign = None
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
-    def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Dataset:
-        copied = super().__deepcopy__(memo=memo)
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
-    def model_copy(self, *args, **kwargs) -> "Dataset":
-        copied = super().model_copy(*args, **kwargs)
-
-        # since campaign is not a proper field, a duplicate of the copied instance is added to the
-        # campaign's dataset index, so replace it
-        if has_attr(getattr(copied, "campaign", None), "datasets"):
-            copied.campaign._setup_datasets()
-            copied.campaign.datasets.add(copied, overwrite=True)
-
-        return copied
-
     def __getitem__(self, name: str) -> DatasetVariation:
         return self.get_info(name)
 

order/models/mixins.py

@@ -2,38 +2,7 @@
 
 from __future__ import annotations
 
+__all__ = []
 
-__all__ = ["CopyMixin"]
-
-
-from order.types import Any
-from order.models.base import Model
-
-
-class CopyMixin(Model):
-    """
-    TODO.
-    """
-
-    # shallow specs:
-    # set_none
-    # re_copy
-    # field_default (default if not PydanticUndefined, otherwise default_factory())
-    # custom: e.g. run setup
-
-    # deep specs:
-    # set_none
-    # field_default
-    # custom: e.g. re-add to index to overwrite duplicate copy, see datasets
-
-    # create a way to have specs apply to both shallow and deep copies, or to just one!
-
-    # also, there should be an easy way to inject conditions for applying specs in the first place,
-    # such as laziness checks, which are done on the corresponding lazy attribute to avoid
-    # materialization, see e.g. DatasetInfo.lfns
-
-    def __copy__(self) -> "CopyMixin":
-        return super().__copy__()
-
-    def __deepcopy__(self, memo: dict[int, Any] | None = None) -> "CopyMixin":
-        return super().__deepcopy__(memo=memo)
+from order.types import Any  # noqa
+from order.models.base import Model  # noqa

order/models/process.py

@@ -9,7 +9,7 @@
 from pydantic import Field
 
 from order.types import (
-    Union, List, Dict, NonEmptyStrictStr, Lazy, ClassVar, Any, StrictFloat,
+    Union, List, Dict, NonEmptyStrictStr, Lazy, ClassVar, StrictFloat,
 )
 from order.util import has_attr
 from order.models.unique import UniqueObjectBase, UniqueObject, LazyUniqueObject, UniqueObjectIndex
@@ -49,8 +49,16 @@ class Process(UniqueObject):
         default_factory=dict,
         frozen=True,
     )
-    processes: ProcessIndex = Field(default_factory=ProcessIndex, frozen=True)
-    parent_processes: ProcessIndex = Field(default_factory=ProcessIndex, frozen=True)
+    processes: ProcessIndex = Field(
+        default_factory=ProcessIndex,
+        frozen=True,
+    )
+    parent_processes: ProcessIndex = Field(
+        default_factory=ProcessIndex,
+        frozen=True,
+        # exclude from exports to avoid circular references
+        exclude=True,
+    )
 
     lazy_cls: ClassVar[UniqueObjectBase] = LazyProcess
 
@@ -97,33 +105,6 @@ def _setup_parent_processes(self) -> None:
         for parent_process in self.parent_processes.objects:
             self._parent_process_add_callback(parent_process)
 
-    def __copy__(self) -> "Process":
-        copied = super().__copy__()
-
-        # TODO: handle cross sections here after above TODOs are resolved
-
-        # drop references to child and parent processes for consistency
-        with copied._unfreeze_field("processes"):
-            copied.processes = copied.model_fields["processes"].default_factory()
-
-        with copied._unfreeze_field("parent_processes"):
-            copied.parent_processes = copied.model_fields["parent_processes"].default_factory()
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
-    def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Process:
-        copied = super().__deepcopy__(memo=memo)
-
-        # setup objects if not triggered by model_copy
-        if not self._copy_triggered_by_model:
-            copied._setup_objects()
-
-        return copied
-
     def _process_materialize_callback(self, process: "Process") -> None:
         process.parent_processes.add(self, overwrite=True)
 

order/models/unique.py

@@ -6,13 +6,11 @@
 
 from __future__ import annotations
 
-
 __all__ = [
     "UniqueObject", "LazyUniqueObject", "UniqueObjectIndex",
     "DuplicateObjectException", "DuplicateNameException", "DuplicateIdException",
 ]
 
-
 from abc import abstractmethod
 from contextlib import contextmanager
 
@@ -242,7 +240,7 @@ class LazyUniqueObject(UniqueObjectBase, WrapsUniqueClass):
     @abstractmethod
     def create_lazy_dict(cls) -> dict[str, Any]:
         # must be implemented by subclasses
-        return
+        ...
 
     @classmethod
     def create(cls, *args, **kwargs) -> "LazyUniqueObject":
@@ -368,25 +366,6 @@ def __init__(self, *args, **kwargs) -> None:
         # sync indices initially
         self._sync_indices()
 
-    def __copy__(self) -> "UniqueObjectIndex":
-        inst = super().__copy__()
-
-        # make a copy of the objects list
-        inst.objects = list(inst.objects)
-
-        # fully reset indices
-        self._reset_indices()
-
-        return inst
-
-    def __deepcopy__(self, memo: dict[int, Any] | None = None) -> "UniqueObjectIndex":
-        inst = super().__deepcopy__(memo=memo)
-
-        # fully reset indices
-        self._reset_indices()
-
-        return inst
-
     def __len__(self) -> int:
         """
         Returns the number of objects in the index.

requirements_dev.txt

@@ -1,6 +1,9 @@
+ipython~=8.8
+pytest~=7.1
+pytest-cov~=3.0
+flake8~=5.0
+flake8-commas~=2.1
+flake8-quotes~=3.3
+pipdeptree~=2.13
 cloudpickle>=1.3
-flake8>=4.0,<6.0
-flake8-commas>=2.1
-flake8-quotes>=3.3
-pytest-cov>=3.0
 mermaidmro~=0.1.2