Update new scheme for garnet support (example data and config files) (#5)

* update for new example scheme (data and config)

* new file to probe data and configuration example availability

Author: Duchstf

README.md

@@ -1,3 +1,7 @@
 # Contribution guidelines
 
-- When you add a `keras` model, please save the architecture and weights in `.json` and `.h5` files 
+- When you add a `keras` model, please save the architecture and weights in `<model-name>.json` and `<model-name>_weights.h5` files.
+
+- If you have example **data** you want to add, please save them in `data/` with names `<model-name>_input.*` and `<model-name>_output.*`. Where `*` can either be `npy` or `dat`.
+
+- If you have example **configuration** you want to add, please save them in `config-files/` with names `<model-name>_config.yml`

available_data_config.json

@@ -0,0 +1 @@
+{"KERAS_dense_16x500x500x500x500x500x5.json": {"example_data": false, "example_config": false}, "KERAS_3layer.json": {"example_data": false, "example_config": false}, "KERAS_3layer_batch_norm.json": {"example_data": false, "example_config": false}, "KERAS_dense_16x100x100x100x100x100x5.json": {"example_data": false, "example_config": false}, "garnet_3layer.json": {"example_data": false, "example_config": true}, "jetTagger_Conv2D_Small_NoBatchNorm.json": {"example_data": false, "example_config": false}, "KERAS_dense_16x200x200x200x200x200x5.json": {"example_data": false, "example_config": false}, "jetTagger_Conv2D_Small.json": {"example_data": false, "example_config": false}, "KERAS_3layer_binarydense_relu_max.json": {"example_data": false, "example_config": false}, "KERAS_conv1d_small.json": {"example_data": false, "example_config": false}, "KERAS_3layer_ternary_small.json": {"example_data": false, "example_config": false}, "KERAS_1layer.json": {"example_data": false, "example_config": false}, "KERAS_conv1d.json": {"example_data": false, "example_config": false}, "qkeras_3layer.json": {"example_data": false, "example_config": false}, "KERAS_3layer_binary_smaller.json": {"example_data": false, "example_config": false}, "keras_bnn.json": {"example_data": false, "example_config": false}, "KERAS_conv2d_model.json": {"example_data": false, "example_config": false}, "garnet_1layer.json": {"example_data": true, "example_config": true}, "two_layer_model.pt": {"example_data": false, "example_config": false}, "three_layer_model.pt": {"example_data": false, "example_config": false}, "two_layer_pytorch.onnx": {"example_data": false, "example_config": false}, "three_layer_pytorch.onnx": {"example_data": false, "example_config": false}, "three_layer_keras.onnx": {"example_data": false, "example_config": false}, "conv2d_small_mp_keras.onnx": {"example_data": false, "example_config": false}, "two_layer_keras.onnx": {"example_data": false, "example_config": false}, "conv1d_small_keras.onnx": {"example_data": false, "example_config": false}, "three_layer_bn_pytorch.onnx": {"example_data": false, "example_config": false}, "dense_big_keras.onnx": {"example_data": false, "example_config": false}, "conv2d_small_keras.onnx": {"example_data": false, "example_config": false}, "three_layer_bn_keras.onnx": {"example_data": false, "example_config": false}, "3layer.pb": {"example_data": false, "example_config": false}}

available_models.json

@@ -1 +1 @@
-{"onnx": ["dense_big_keras.onnx", "conv2d_small_mp_keras.onnx", "two_layer_pytorch.onnx", "two_layer_keras.onnx", "three_layer_bn_keras.onnx", "conv1d_small_keras.onnx", "three_layer_pytorch.onnx", "three_layer_bn_pytorch.onnx", "three_layer_keras.onnx", "conv2d_small_keras.onnx"], "pytorch": ["two_layer_model.pt", "three_layer_model.pt"], "tensorflow": ["3layer.pb"], "keras": ["KERAS_3layer.json", "KERAS_dense_16x500x500x500x500x500x5.json", "KERAS_conv1d_small.json", "jetTagger_Conv2D_Small.json", "KERAS_3layer_ternary_small.json", "qkeras_3layer.json", "KERAS_dense_16x200x200x200x200x200x5.json", "keras_bnn.json", "garnet_1layer.json", "jetTagger_Conv2D_Small_NoBatchNorm.json", "garnet_3layer.json", "KERAS_3layer_binary_smaller.json", "KERAS_3layer_binarydense_relu_max.json", "KERAS_conv2d_model.json", "KERAS_dense_16x100x100x100x100x100x5.json", "KERAS_3layer_batch_norm.json", "KERAS_1layer.json", "KERAS_conv1d.json"]}
+{"keras": ["KERAS_dense_16x500x500x500x500x500x5.json", "KERAS_3layer.json", "KERAS_3layer_batch_norm.json", "KERAS_dense_16x100x100x100x100x100x5.json", "garnet_3layer.json", "jetTagger_Conv2D_Small_NoBatchNorm.json", "KERAS_dense_16x200x200x200x200x200x5.json", "jetTagger_Conv2D_Small.json", "KERAS_3layer_binarydense_relu_max.json", "KERAS_conv1d_small.json", "KERAS_3layer_ternary_small.json", "KERAS_1layer.json", "KERAS_conv1d.json", "qkeras_3layer.json", "KERAS_3layer_binary_smaller.json", "keras_bnn.json", "KERAS_conv2d_model.json", "garnet_1layer.json"], "pytorch": ["two_layer_model.pt", "three_layer_model.pt"], "onnx": ["two_layer_pytorch.onnx", "three_layer_pytorch.onnx", "three_layer_keras.onnx", "conv2d_small_mp_keras.onnx", "two_layer_keras.onnx", "conv1d_small_keras.onnx", "three_layer_bn_pytorch.onnx", "dense_big_keras.onnx", "conv2d_small_keras.onnx", "three_layer_bn_keras.onnx"], "tensorflow": ["3layer.pb"]}

scan.py

@@ -1,43 +1,83 @@
 import os
 import json
 
+
+def scan_data_config(model_name):
+    """
+    Take a model name and return a dictionary of whether the associated data and configuration files 
+    are available for that model or not.
+    """
+
+    available_data_config = {
+        'example_data': False,
+        'example_config': False}
+
+    #Check data availability
+    for filename in os.listdir("./data/"):
+        if filename.startswith(model_name):
+            available_data_config['example_data'] = True
+    
+    #Check config file availability
+    for filename in os.listdir("./config-files/"):
+        if filename.startswith(model_name):
+            available_data_config['example_config'] = True
+    
+    return available_data_config
+
+
 def scan_subdir(model_type):
-    """Scan a directory and return a list of example models."""
+    """Scan a directory and return a list of example models, and a dictionary indicating
+    whether each model has associated example data and example config file or not."""
 
     #Create the path to the sub directory
     dir_path = model_type + "/"
 
     #List of available models
     available_models = []
+
+    #Dictionary to save data's and config's availability
+    data_config_avai = {}
 
     for filename in os.listdir(dir_path):
 
         if model_type == 'keras':
             if filename.endswith('.json'):
                 available_models.append(filename)
+                data_config_avai[filename] = scan_data_config(filename[:-5])
         elif model_type == 'pytorch':
             if filename.endswith('.pt'):
                 available_models.append(filename)
+                data_config_avai[filename] = scan_data_config(filename[:-3])
         elif model_type == 'tensorflow':
             if filename.endswith('.pb'):
                 available_models.append(filename)
+                data_config_avai[filename] = scan_data_config(filename[:-3])
         elif model_type == 'onnx':
             if filename.endswith('.onnx'):
                 available_models.append(filename)
+                data_config_avai[filename] = scan_data_config(filename[:-5])
 
-    return available_models
+    return available_models, data_config_avai
 
 def scan():
     """Scan all directories that contain example models and write to a file."""
 
     available_types = ['keras', 'pytorch', 'onnx', 'tensorflow']
 
+    #Dictionary to save available model list
     model_dict = {}
+
+    #Dictionary to check for data and configuration availability
+    data_config_avai = {}
 
     for model_type in available_types:
-        model_dict[model_type] = scan_subdir(model_type)
+        model_dict[model_type], temp_data_config_avai = scan_subdir(model_type)
+
+        #Upate the availability of data and configuration for new scanned model
+        data_config_avai.update(temp_data_config_avai)
 
     # Serialize data into file:
     json.dump(model_dict, open("available_models.json", 'w' ))
-    
+    json.dump(data_config_avai, open("available_data_config.json", 'w' ))
+
 scan()