baseline functionality (#7)

* very basic skelton for usecase1

* develop and test basic functionality to insert json to mongo db and retrieve by id

* remove @classmethod

* test and dev add attribute

* dev and test save model

* added two placeholder savemodel methods

* make mongo service more genral

* moved mongo servcie to separate file

* dev and test saved_ids

* restore model

* typo

* fix test

* extract mongo id helper

* save matehod with pickle and code zipped

* save and restore working

* save and restore all test running

* equals -> equal

* script to build mmlib in docker container

* script to build mmlib in docker container

* added documentation: build with docker

* fix probe example

* added section on repo content

* documented examples

* documented examples

* added comment for model

* use python warning istead of custom method

* updated requirements

* added autor email

* updated setup.py

* restructuring

* added comments to save.py

* introduced abstract classes for Save and Recover Service

* adjusted comments

* added types for helper

* added types for public methods from model_equal and probe

* updated Readme

* updated Readme

* fixed str vs ObjectIds

* include util in library

* refactoring of equal methods

* added TODOs to use protocols

* simple version of use model

* readme adjustments

* rename ProbeTest -> SaveTest

Author: slin96

README.md

@@ -1,7 +1,34 @@
-# MM-LIB
+# mmlib
+
+- A library for model management and related tasks.
+
+## Installation
+
+### Option 1: Docker
+
+- **Requirements**: Docker installed
+- **Build Library**
+    - clone this repo
+    - run the script `generate-archives-docker.sh`
+      - it runs a docker container and builds the *mmlib* in it.
+      - the created `dist` directory is copied back to repository root
+      - it contains the `.whl` file that can be used to install the library with pip (see below)
+- **Install**
+    - to install mmlib run: `pip install <PATH>/dist/mmlib-0.0.1-py3-none-any.whl`
+
+### Option 2: Local Build
+
+- **Requirements**: Python 3.8
+- **Build Library**
+    - run the script `generate-archives.sh`
+      - it creates a virtual environment, activates it, and installs all requirements
+      - afterward it builds the library, and a `dist` directory containing the `.whl` file is created
+- **Install**
+    - to install mmlib run: `pip install <PATH>/dist/mmlib-0.0.1-py3-none-any.whl`
+
+## Examples
+
+- For examples on how to use mmlib checkout the [examples](./examples) directory.
+
 
-## installation
 
-- to build the lib run: `generate-archives.sh`
-- to install it run: `pip install <PATH>/dist/mmlib-0.0.1-py3-none-any.whl
-  `

examples/README.md

@@ -0,0 +1,29 @@
+# Examples
+
+This directory contains examples of how to use the functionality offered by the *mmlib*.
+
+- *probe_store.py* - Creates and stores a probe summary of the training process of a GoogLeNet.
+    - execution: `python probe_store.py --path <optional path to store probe summary>`
+- *probe_load_compare.py* - Creates a probe summary of the training process of a GoogLeNet and compares it to a stored
+  probe summary
+    - execution: `python probe_load_compare.py --path <path to the already stored probe summary>`
+    - note: To generate and store a probe summary to compare to use the *probe_store.py* script.
+- *probe_example.py* - Shows extensively how the probe functionality offered by the *mmlib* can be used to make the
+  PyTorch implementation of GoogLeNet reproducible. It runs the following steps:
+    - simple summary
+        - creates a probe summary for the inference mode and prints the representation
+    - probe inference
+        - creates two instances of the same model
+        - creates inference mode probe summaries (covering forward path) for them
+        - compares the probe summaries
+    - probe training
+        - creates two instances of the same model
+        - creates training mode probe summaries (covering forward and backward path)
+        - compares the probe summaries
+    - probe reproducible training
+        - creates two instances of the same model
+        - uses *set_deterministic* functionality offered by the *mmlib* to make the training process of both models
+          reproducible
+        - creates training mode probe summaries (covering forward and backward path)
+        - compares the probe summaries
+        - compares both models using the methods *blackbox_model_equal*, *whitebox_model_equal*, and *model_equal* offered by the *mmlib*.

examples/probe_example.py

@@ -4,7 +4,7 @@
 
 from mmlib.deterministic import set_deterministic
 from mmlib.helper import imagenet_input, imagenet_target
-from mmlib.model_equals import equals, whitebox_equals, blackbox_equals
+from mmlib.equal import model_equal, blackbox_model_equal, whitebox_model_equal
 from mmlib.probe import ProbeInfo, probe_inference, probe_training
 
 MODEL = models.googlenet
@@ -96,13 +96,13 @@ def deterministic_backward_compare(device, forward_indices=None):
     summary1.compare_to(summary2, common, compare)
 
     # also the models should be equal
-    blackbox_equal = blackbox_equals(model1, model2, imagenet_input)
-    whitebox_equal = whitebox_equals(model1, model2)
-    models_are_equal = equals(model1, model2, imagenet_input)
+    blackbox_eq = blackbox_model_equal(model1, model2, imagenet_input)
+    whitebox_eq = whitebox_model_equal(model1, model2)
+    models_are_equal = model_equal(model1, model2, imagenet_input)
     print()
     print('Also the models should be the same - compare the models')
-    print('models_are_equal (blackbox): {}'.format(blackbox_equal))
-    print('models_are_equal (whitebox): {}'.format(whitebox_equal))
+    print('models_are_equal (blackbox): {}'.format(blackbox_eq))
+    print('models_are_equal (whitebox): {}'.format(whitebox_eq))
     print('models_are_equal: {}'.format(models_are_equal))
 
 

examples/probe_store.py

@@ -22,7 +22,7 @@ def _generate_probe_training_summary():
     dummy_input = imagenet_input()
     dummy_target = imagenet_target(dummy_input)
     loss_func = nn.CrossEntropyLoss()
-    model = models.alexnet(pretrained=True)
+    model = models.googlenet(pretrained=True)
     optimizer = torch.optim.SGD(model.parameters(), 1e-3)
     summary = probe_training(model, dummy_input, optimizer, loss_func, dummy_target)
     return summary

generate-archives-docker.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+CONTAINER_NAME=mmlib-python
+
+docker run --rm --name $CONTAINER_NAME -it -d python:3.8
+docker cp ../mmlib $CONTAINER_NAME:/
+docker exec $CONTAINER_NAME /mmlib/generate-archives.sh
+docker cp $CONTAINER_NAME:/mmlib/dist ./
+docker kill $CONTAINER_NAME

generate-archives.sh

@@ -11,6 +11,6 @@ python3 -m pip install --upgrade pip
 
 # install requirements
 python3 -m pip install --upgrade setuptools wheel
-python3 -m pip install -r requirements-tests.txt
+python3 -m pip install -r requirements.txt
 
 python3 setup.py sdist bdist_wheel

mmlib/model_equals.py mmlib/equal.py

@@ -1,9 +1,12 @@
+from typing import Callable
+
 import torch
 
-from mmlib.helper import _get_device
+from util.helper import get_device
 
 
-def blackbox_equals(m1, m2, produce_input, device: torch.device = None):
+def blackbox_model_equal(m1: torch.nn.Module, m2: torch.nn.Module, produce_input: Callable[[], torch.tensor],
+                         device: torch.device = None) -> bool:
     """
     Compares two models in a blackbox manner meaning if the models are equal is determined only by comparing inputs and
     outputs.
@@ -14,7 +17,10 @@ def blackbox_equals(m1, m2, produce_input, device: torch.device = None):
     :return: Returns if the two given models are equal.
     """
 
-    device = _get_device(device)
+    assert isinstance(m1, torch.nn.Module)
+    assert isinstance(m2, torch.nn.Module)
+
+    device = get_device(device)
 
     inp = produce_input()
 
@@ -31,7 +37,7 @@ def blackbox_equals(m1, m2, produce_input, device: torch.device = None):
     return torch.equal(out1, out2)
 
 
-def whitebox_equals(m1, m2, device: torch.device = None):
+def whitebox_model_equal(m1: torch.nn.Module, m2: torch.nn.Module, device: torch.device = None) -> bool:
     """
     Compares two models in a whitebox manner meaning we compare the model weights.
     :param m1: The first model to compare.
@@ -40,15 +46,18 @@ def whitebox_equals(m1, m2, device: torch.device = None):
     :return: Returns if the two given models are equal.
     """
 
-    device = _get_device(device)
+    assert isinstance(m1, torch.nn.Module)
+    assert isinstance(m2, torch.nn.Module)
+
+    device = get_device(device)
 
     state1 = m1.state_dict()
     state2 = m2.state_dict()
 
-    return state_dict_equals(state1, state2, device)
+    return state_dict_equal(state1, state2, device)
 
 
-def state_dict_equals(d1, d2, device: torch.device = None):
+def state_dict_equal(d1: dict, d2: dict, device: torch.device = None) -> bool:
     """
     Compares two given state dicts.
     :param d1: The first state dict.
@@ -57,7 +66,7 @@ def state_dict_equals(d1, d2, device: torch.device = None):
     :return: Returns if the given state dicts are equal.
     """
 
-    device = _get_device(device)
+    device = get_device(device)
 
     for item1, item2 in zip(d1.items(), d2.items()):
         layer_name1, weight_tensor1 = item1
@@ -72,7 +81,8 @@ def state_dict_equals(d1, d2, device: torch.device = None):
     return True
 
 
-def equals(m1, m2, produce_input, device: torch.device = None):
+def model_equal(m1: torch.nn.Module, m2: torch.nn.Module, produce_input: Callable[[], torch.tensor],
+                device: torch.device = None) -> bool:
     """
     An equals method to compare two given models by making use of whitebox and blackbox equals.
     :param m1: The first model to compare.
@@ -81,8 +91,18 @@ def equals(m1, m2, produce_input, device: torch.device = None):
     :param device: The device to execute on
     :return: Returns if the two given models are equal.
     """
-    device = _get_device(device)
+    device = get_device(device)
 
     # whitebox and blackbox check should be redundant,
     # but this way we have an extra safety net in case we forgot a special case
-    return whitebox_equals(m1, m2, device) and blackbox_equals(m1, m2, produce_input, device)
+    return whitebox_model_equal(m1, m2, device) and blackbox_model_equal(m1, m2, produce_input, device)
+
+
+def tensor_equal(tensor1: torch.tensor, tensor2: torch.tensor):
+    """
+    Compares to given Pytorch tensors.
+    :param tensor1: The first tensor to be compared.
+    :param tensor2: The second tensor to be compared.
+    :return: Returns if the two given tensors are equal.
+    """
+    return torch.equal(tensor1, tensor2)

mmlib/helper.py

@@ -3,13 +3,7 @@
 import torch
 
 
-def _get_device(device):
-    if device is None:
-        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-    return device
-
-
-def imagenet_input(batch_size=10):
+def imagenet_input(batch_size: int = 10) -> torch.tensor:
     """
     Generates a batch of dummy imputes for models processing imagenet data.
     :param batch_size: The size of the batch.
@@ -21,7 +15,7 @@ def imagenet_input(batch_size=10):
     return torch.stack(batch)
 
 
-def imagenet_target(dummy_input):
+def imagenet_target(dummy_input: torch.tensor) -> torch.tensor:
     """
     Creates a batch of random labels for imagenet data based on a given input data.
     :param dummy_input: The input to a potential model for the the target values should be produced.

mmlib/log.py

@@ -0,0 +1,2 @@
+def use_model(model_id):
+    print('use model with model_id: {}'.format(str(model_id)))

mmlib/probe.py

@@ -1,11 +1,11 @@
+import warnings
 from enum import Enum
 
 import torch
 import torch.nn as nn
 from colorama import Fore, Style
 
-from mmlib.helper import _get_device
-from mmlib.util import _print_info, _print_warning
+from util.helper import print_info, get_device
 
 
 class ProbeInfo(Enum):
@@ -41,7 +41,10 @@ class ProbeSummary:
     DIFF = 'diff'
     SAME = 'same'
 
-    def __init__(self, summary_path=None):
+    def __init__(self, summary_path: str = None):
+        """
+        :param summary_path: Path to load a summary from
+        """
         if summary_path:
             self.load(summary_path)
         else:
@@ -105,14 +108,14 @@ def compare_to(self, other_summary, common: [ProbeInfo], compare: [ProbeInfo]):
         for layer_key, layer_info in self.summary.items():
             self._print_compare_layer(common, compare, layer_info, other_summary)
 
-    def save(self, path):
+    def save(self, path: str):
         """
         Saves an object to a disk file.
         :param path: The path to store to.
         """
         torch.save(self.summary, path)
 
-    def load(self, path):
+    def load(self, path: str):
         """
         Loads an object saved with :func:`mmlib.probe.save` from a file.
         :param path: The path to load from.
@@ -216,7 +219,8 @@ def _compare_values(self, v1, v2):
             return v1 == v2
 
 
-def probe_inference(model, inp, device: torch.device = None, forward_indices=None):
+def probe_inference(model: torch.nn.Module, inp: torch.tensor, device: torch.device = None,
+                    forward_indices: [int] = None) -> ProbeSummary:
     """
     Probes the inference of a given model.
     :param model: The model to probe.
@@ -233,7 +237,8 @@ def probe_inference(model, inp, device: torch.device = None, forward_indices=Non
     return _probe_reproducibility(model, inp, ProbeMode.INFERENCE, device, forward_indices=forward_indices)
 
 
-def probe_training(model, inp, optimizer, loss_func, target, device: torch.device = None, forward_indices=None):
+def probe_training(model: torch.nn.Module, inp: torch.tensor, optimizer: torch.optim.Optimizer, loss_func,
+                   target: torch.tensor, device: torch.device = None, forward_indices: [int] = None) -> ProbeSummary:
     """
     Probes the training of a given model.
     :param model: The model to probe.
@@ -259,7 +264,7 @@ def _probe_reproducibility(model, inp, mode, device, optimizer=None, loss_func=N
 
     _forward_indices_warning(forward_indices)
 
-    device = _get_device(device)
+    device = get_device(device)
 
     def register_forward_hook(module, ):
 
@@ -362,22 +367,22 @@ def _shape_list(tensor_tuple):
 
 def _forward_indices_warning(forward_indices):
     if forward_indices is not None:
-        _print_warning("You set the forward_indices argument. "
-                       "This means not all layers will be included in the summary.")
+        print_info("You set the forward_indices argument. "
+                   "This means not all layers will be included in the summary.")
     else:
-        _print_warning("You did not set the forward_indices argument. "
-                       "Every layer will be included in the summary. This might lead to very high memory consumption.")
+        warnings.warn("You did not set the forward_indices argument."
+                      "Every layer will be included in the summary. This might lead to very high memory consumption.")
 
 
 def _hashwarning(fields: [ProbeInfo]):
     # If we print tensors or shapes it is likely that they are to long. In this case we print a hash instead.
     # Warn the user that for example for long tensors same hash values do not guarantee the same values.
     if any('shape' in x.value or 'tensor' in x.value for x in fields):
-        _print_warning("Same hashes don\'t have to mean that values are exactly the same (especially for tensors)."
-                       " Hashes should be seen as an indicator.")
+        print_info("Same hashes don\'t have to mean that values are exactly the same (especially for tensors)."
+                   " Hashes should be seen as an indicator.")
 
 
 def _inference_info():
-    _print_info("You are probing in inference mode so the model will be in eval mode."
-                "\nSince layers like dropout are switched off in this mode you won't find factors that produce "
-                "non-reproducibility by these kind of layers.")
+    print_info("You are probing in inference mode so the model will be in eval mode."
+               "\nSince layers like dropout are switched off in this mode you won't find factors that produce "
+               "non-reproducibility by these kind of layers.")