Restructure (#41)

* moved examples

* remove unused file

* moved schema

* moved util

* fix basleine save test

* fix update and prov test

* fix file persistence

* fix size tests

* reformat

* add utils

Author: slin96

.gitignore

@@ -143,10 +143,10 @@ cython_debug/
 # build lib
 build/
 dist/
-/examples/summary
+/mmlib/examples/summary
 /tests/filesystem-tmp/
 /tests/saveservice-tmp/
 /tests/tmp-data/
 /tests/example_files/data/reduced-custom-coco-data.zip
-/examples/filesystem-tmp/
+/mmlib/examples/filesystem-tmp/
 /tests/example_files/local-config.ini

README.md

@@ -2,7 +2,7 @@
 
 import torch
 
-from util.helper import get_device
+from mmlib.util.helper import get_device
 
 
 def blackbox_model_equal(m1: torch.nn.Module, m2: torch.nn.Module, produce_input: Callable[[], torch.tensor],

mmlib/equal.py

@@ -3,13 +3,13 @@
 from mmlib.equal import model_equal
 from mmlib.persistence import FileSystemPersistenceService, MongoDictPersistenceService
 from mmlib.save import BaselineSaveService
-from schema.save_info_builder import ModelSaveInfoBuilder
+from mmlib.schema import ModelSaveInfoBuilder
+from mmlib.util.dummy_data import imagenet_input
 from tests.example_files.mynets.mobilenet import mobilenet_v2
-from util.dummy_data import imagenet_input
 
 CONTAINER_NAME = 'mongo-test'
 
-TARGET_FILE_SYSTEM_DIR = './filesystem-tmp'
+TARGET_FILE_SYSTEM_DIR = 'filesystem-tmp'
 
 if __name__ == '__main__':
     # initialize a service to store files

examples/README.md mmlib/examples/README.md

@@ -5,7 +5,7 @@
 from mmlib.deterministic import set_deterministic
 from mmlib.equal import model_equal, blackbox_model_equal, whitebox_model_equal
 from mmlib.probe import ProbeInfo, probe_inference, probe_training
-from util.dummy_data import imagenet_input, imagenet_target
+from mmlib.util.dummy_data import imagenet_input, imagenet_target
 
 MODEL = models.googlenet
 

schema/__init__.py mmlib/examples/__init__.py

@@ -1,6 +1,6 @@
 import argparse
 
-from examples.probe_store import _generate_probe_training_summary
+from mmlib.examples.probe_store import _generate_probe_training_summary
 from mmlib.probe import ProbeSummary, ProbeInfo
 
 

examples/baseline_save.py mmlib/examples/baseline_save.py

@@ -7,7 +7,7 @@
 
 from mmlib.deterministic import set_deterministic
 from mmlib.probe import probe_training
-from util.dummy_data import imagenet_input, imagenet_target
+from mmlib.util.dummy_data import imagenet_input, imagenet_target
 
 
 def main(args):

examples/probe_example.py mmlib/examples/probe_example.py

@@ -8,16 +8,16 @@
 from mmlib.equal import model_equal
 from mmlib.persistence import FileSystemPersistenceService, MongoDictPersistenceService
 from mmlib.save import ProvenanceSaveService
+from mmlib.schema import ModelSaveInfoBuilder
+from mmlib.schema.restorable_object import RestorableObjectWrapper, OptimizerWrapper
 from mmlib.track_env import track_current_environment
-from schema.restorable_object import RestorableObjectWrapper, OptimizerWrapper
-from schema.save_info_builder import ModelSaveInfoBuilder
+from mmlib.util.dummy_data import imagenet_input
 from tests.example_files.data.custom_coco import TrainCustomCoco
 from tests.example_files.imagenet_train import ImagenetTrainService, DATALOADER, OPTIMIZER, ImagenetTrainWrapper, DATA
 from tests.example_files.mynets.resnet18 import resnet18
-from util.dummy_data import imagenet_input
 
 CONTAINER_NAME = 'mongo-test'
-TARGET_FILE_SYSTEM_DIR = './filesystem-tmp'
+TARGET_FILE_SYSTEM_DIR = 'filesystem-tmp'
 
 
 def initi_train_service():

examples/probe_load_compare.py mmlib/examples/probe_load_compare.py

@@ -7,9 +7,9 @@
 
 from bson import ObjectId
 
-from schema.file_reference import FileReference
-from util.helper import find_file, log_stop, START_STOP, TIME
-from util.mongo import MongoService
+from mmlib.schema.file_reference import FileReference
+from mmlib.util.helper import find_file, log_stop, START_STOP, TIME
+from mmlib.util.mongo import MongoService
 
 MMLIB_FILE_PERS = 'mmlib_file_pers'
 

examples/probe_store.py mmlib/examples/probe_store.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 from colorama import Fore, Style
 
-from util.helper import print_info, get_device
+from mmlib.util.helper import print_info, get_device
 
 
 class ProbeInfo(Enum):

examples/provenance_save.py mmlib/examples/provenance_save.py

@@ -8,17 +8,17 @@
 from mmlib.equal import tensor_equal
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
 from mmlib.save_info import ModelSaveInfo, ProvModelSaveInfo
+from mmlib.schema.dataset import Dataset
+from mmlib.schema.file_reference import FileReference
+from mmlib.schema.model_info import ModelInfo, MODEL_INFO
+from mmlib.schema.recover_info import FullModelRecoverInfo, WeightsUpdateRecoverInfo, ProvenanceRecoverInfo
+from mmlib.schema.restorable_object import RestoredModelInfo
+from mmlib.schema.store_type import ModelStoreType
+from mmlib.schema.train_info import TrainInfo
 from mmlib.track_env import compare_env_to_current
-from schema.dataset import Dataset
-from schema.file_reference import FileReference
-from schema.model_info import ModelInfo, MODEL_INFO
-from schema.recover_info import FullModelRecoverInfo, ProvenanceRecoverInfo, WeightsUpdateRecoverInfo
-from schema.restorable_object import RestoredModelInfo
-from schema.store_type import ModelStoreType
-from schema.train_info import TrainInfo
-from util.helper import log_start, log_stop
-from util.init_from_file import create_object, create_type
-from util.weight_dict_merkle_tree import WeightDictMerkleTree, THIS, OTHER
+from mmlib.util.helper import log_start, log_stop
+from mmlib.util.init_from_file import create_object, create_type
+from mmlib.util.weight_dict_merkle_tree import WeightDictMerkleTree, THIS, OTHER
 
 PROVENANCE = 'provenance'
 

mmlib/log.py

@@ -1,8 +1,8 @@
 import torch
 
-from schema.environment import Environment
-from schema.restorable_object import StateDictRestorableObjectWrapper
-from util.helper import class_name, source_file
+from mmlib.schema.environment import Environment
+from mmlib.schema.restorable_object import StateDictRestorableObjectWrapper
+from mmlib.util.helper import class_name, source_file
 
 
 class TrainSaveInfo:

mmlib/persistence.py

@@ -1,9 +1,9 @@
 import os
 
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.file_reference import FileReference
-from schema.schema_obj import SchemaObj
-from util.zip import zip_path, unzip
+from mmlib.schema.file_reference import FileReference
+from mmlib.schema.schema_obj import SchemaObj
+from mmlib.util.zip import zip_path, unzip
 
 RAW_DATA = 'raw_data'
 

mmlib/probe.py

@@ -1,5 +1,5 @@
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.schema_obj import SchemaObj
+from mmlib.schema.schema_obj import SchemaObj
 
 PYTHON_VERSION = 'python_version'
 PYTORCH_VERSION = 'pytorch_version'

mmlib/save.py

@@ -1,9 +1,9 @@
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.recover_info import FullModelRecoverInfo, AbstractRecoverInfo, WeightsUpdateRecoverInfo, \
+from mmlib.schema.recover_info import FullModelRecoverInfo, AbstractRecoverInfo, WeightsUpdateRecoverInfo, \
     ProvenanceRecoverInfo, RECOVER_INFO
-from schema.schema_obj import SchemaObj
-from schema.store_type import ModelStoreType
-from util.weight_dict_merkle_tree import WeightDictMerkleTree
+from mmlib.schema.schema_obj import SchemaObj
+from mmlib.schema.store_type import ModelStoreType
+from mmlib.util.weight_dict_merkle_tree import WeightDictMerkleTree
 
 STORE_TYPE = 'store_type'
 RECOVER_INFO_ID = 'recover_info_id'

mmlib/save_info.py

@@ -4,12 +4,12 @@
 
 from mmlib.constants import MMLIB_CONFIG, CURRENT_DATA_ROOT, VALUES
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.dataset import Dataset
-from schema.environment import Environment
-from schema.file_reference import FileReference
-from schema.schema_obj import SchemaObj
-from schema.train_info import TrainInfo
-from util.helper import copy_all_data, clean
+from mmlib.schema.dataset import Dataset
+from mmlib.schema.environment import Environment
+from mmlib.schema.file_reference import FileReference
+from mmlib.schema.schema_obj import SchemaObj
+from mmlib.schema.train_info import TrainInfo
+from mmlib.util.helper import copy_all_data, clean
 
 RECOVER_INFO = 'recover_info'
 

util/__init__.py mmlib/schema/__init__.py

@@ -9,10 +9,10 @@
 
 from mmlib.constants import MMLIB_CONFIG, VALUES, ID
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.file_reference import FileReference
-from schema.schema_obj import SchemaObj
-from util.helper import class_name, source_file
-from util.init_from_file import create_object_with_parameters
+from mmlib.schema.file_reference import FileReference
+from mmlib.schema.schema_obj import SchemaObj
+from mmlib.util.helper import class_name, source_file
+from mmlib.util.init_from_file import create_object_with_parameters
 
 STATE_DICT = 'state_dict'
 

schema/dataset.py mmlib/schema/dataset.py

@@ -1,8 +1,8 @@
 import torch
 
 from mmlib.save_info import ModelSaveInfo, TrainSaveInfo, ProvModelSaveInfo
-from schema.environment import Environment
-from schema.restorable_object import StateDictRestorableObjectWrapper
+from mmlib.schema.environment import Environment
+from mmlib.schema.restorable_object import StateDictRestorableObjectWrapper
 
 
 class ModelSaveInfoBuilder:

schema/environment.py mmlib/schema/environment.py

@@ -6,7 +6,7 @@
 
 from mmlib.constants import ID
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from util.helper import log_stop, START_STOP, START, TIME
+from mmlib.util.helper import log_stop, START_STOP, START, TIME
 
 METADATA_SIZE = 'metadata_size'
 

schema/file_reference.py mmlib/schema/file_reference.py

@@ -1,10 +1,10 @@
 import os
 
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.file_reference import FileReference
-from schema.restorable_object import StateDictRestorableObjectWrapper
-from schema.schema_obj import SchemaObj
-from util.init_from_file import create_type
+from mmlib.schema.file_reference import FileReference
+from mmlib.schema.restorable_object import StateDictRestorableObjectWrapper
+from mmlib.schema.schema_obj import SchemaObj
+from mmlib.util.init_from_file import create_type
 
 TRAIN_SERVICE = 'train_service'
 TRAIN_KWARGS = 'train_kwargs'

schema/model_info.py mmlib/schema/model_info.py

@@ -6,7 +6,7 @@
 import torch
 from torch.utils.collect_env import get_pretty_env_info
 
-from schema.environment import Environment
+from mmlib.schema.environment import Environment
 
 ARCHITECTURE = 'architecture'
 MACHINE = 'machine'

schema/recover_info.py mmlib/schema/recover_info.py

@@ -3,7 +3,7 @@
 import torch
 
 from mmlib.deterministic import set_deterministic
-from util.helper import get_device
+from mmlib.util.helper import get_device
 
 
 def inference_hash(model: torch.nn.Module, dummy_input_shape: [int]):

schema/restorable_object.py mmlib/schema/restorable_object.py

@@ -1,6 +1,6 @@
 import math
 
-from util.hash import hash_string, tensor_hash
+from mmlib.util.hash import hash_string, tensor_hash
 
 OTHER = 'other'
 

schema/save_info_builder.py mmlib/schema/save_info_builder.py

@@ -1,7 +1,7 @@
 import os
 import zipfile
 
-from util.helper import zip_dir
+from mmlib.util.helper import zip_dir
 
 
 def zip_path(save_path: str) -> str:

schema/schema_obj.py mmlib/schema/schema_obj.py

@@ -17,7 +17,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/slin96/mmlib",
-    packages=['mmlib', 'util', 'schema'],
+    packages=['mmlib', 'mmlib.schema', 'mmlib.util'],
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",

schema/store_type.py mmlib/schema/store_type.py

@@ -1,7 +1,7 @@
 import torch
 from torch.optim import SGD
 
-from schema.restorable_object import StateFileRestorableObject
+from mmlib.schema.restorable_object import StateFileRestorableObject
 
 
 class ImagenetOptimizer(SGD, StateFileRestorableObject):

schema/train_info.py mmlib/schema/train_info.py

@@ -2,9 +2,9 @@
 
 from mmlib.deterministic import set_deterministic
 from mmlib.persistence import FilePersistenceService, DictPersistenceService
-from schema.restorable_object import TrainService, StateDictRestorableObjectWrapper, \
+from mmlib.schema.restorable_object import TrainService, StateDictRestorableObjectWrapper, \
     RestorableObjectWrapper, StateFileRestorableObjectWrapper
-from util.init_from_file import create_object
+from mmlib.util.init_from_file import create_object
 
 DATA = 'data'
 DATALOADER = 'dataloader'

mmlib/track_env.py

@@ -2,10 +2,10 @@
 import unittest
 
 from mmlib.equal import model_equal
+from mmlib.util.dummy_data import imagenet_input
+from mmlib.util.init_from_file import create_object, create_object_with_parameters
 from tests.example_files.mynets.resnet18 import resnet18
 from tests.init_from_file.dummy_classes import DummyA
-from util.dummy_data import imagenet_input
-from util.init_from_file import create_object, create_object_with_parameters
 
 CODE = os.path.join(os.path.dirname(os.path.realpath(__file__)), './dummy_classes.py')
 

mmlib/util/__init__.py

@@ -5,8 +5,8 @@
 
 from mmlib.deterministic import set_deterministic
 from mmlib.equal import state_dict_equal, model_equal
-from util.dummy_data import imagenet_input
-from util.hash import state_dict_hash, tensor_hash
+from mmlib.util import state_dict_hash, tensor_hash
+from mmlib.util.dummy_data import imagenet_input
 
 
 class TestStateDictEqual(unittest.TestCase):

util/dummy_data.py mmlib/util/dummy_data.py

@@ -4,7 +4,7 @@
 import unittest
 
 from mmlib.persistence import FileSystemPersistenceService
-from schema.file_reference import FileReference
+from mmlib.schema.file_reference import FileReference
 
 
 class TestPersistence(unittest.TestCase):

util/hash.py mmlib/util/hash.py

@@ -7,7 +7,7 @@
 
 from mmlib.deterministic import set_deterministic
 from mmlib.probe import ProbeSummary, probe_inference, probe_training
-from util.dummy_data import imagenet_input, imagenet_target
+from mmlib.util.dummy_data import imagenet_input, imagenet_target
 
 TMP_SUMMARY_PATH = './tmp-summary'
 

util/helper.py mmlib/util/helper.py

@@ -5,13 +5,13 @@
 from mmlib.equal import model_equal
 from mmlib.persistence import FileSystemPersistenceService, MongoDictPersistenceService
 from mmlib.save import BaselineSaveService
+from mmlib.schema.save_info_builder import ModelSaveInfoBuilder
 from mmlib.track_env import track_current_environment
-from schema.save_info_builder import ModelSaveInfoBuilder
+from mmlib.util.dummy_data import imagenet_input
+from mmlib.util.mongo import MongoService
 from tests.example_files.mynets.googlenet import googlenet
 from tests.example_files.mynets.mobilenet import mobilenet_v2
 from tests.example_files.mynets.resnet18 import resnet18
-from util.dummy_data import imagenet_input
-from util.mongo import MongoService
 
 FILE_PATH = os.path.dirname(os.path.realpath(__file__))
 NETWORK_CODE_TEMPLATE = os.path.join(FILE_PATH, '../example_files/mynets/{}.py')