add save/load fts; fix logger bug; add inference mode

Author: Michael Klear

demo_data/demo.ipynb

@@ -5,11 +5,25 @@
 import numpy as np
 import torch
 import tqdm
+import dill
+import json
 
 from .dataframe import EncoderDataFrame
 from .logging import BasicLogger, IpynbLogger, TensorboardXLogger
 from .scalers import StandardScaler, NullScaler, GaussRankScaler
 
+
+
+
+def load_model(path):
+    """
+    Loads serialized model from input path.
+    """
+    with open(path, 'rb') as f:
+        loaded_serialized_model = f.read()
+        loaded_model = dill.loads(loaded_serialized_model)
+    return loaded_model
+
 def ohe(input_vector, dim, device="cpu"):
     """Does one-hot encoding of input vector."""
     batch_size = len(input_vector)
@@ -54,6 +68,7 @@ def transform(self, df):
         return df
 
 
+
 class CompleteLayer(torch.nn.Module):
     """
     Impliments a layer with linear transformation
@@ -854,6 +869,80 @@ def get_deep_stack_features(self, df):
         result = torch.cat(result, dim=0)
         return result
 
+    def _deserialize_json(self, data):
+        """
+        encodes json data into appropriate features
+        for inference.
+        "data" should be a string.
+        """
+        data = json.loads(data)
+        return data
+        row = pd.DataFrame()
+        for item in data:
+            row[item] = [data[item]]
+        return row
+
+    
+    def compute_targets_dict(self, data):
+        numeric = []
+        for num_name in self.num_names:
+            raw_value = data[num_name]
+            trans_value = self.numeric_fts[num_name]['scaler'].transform(np.array([raw_value]))
+            numeric.append(trans_value)
+        num = torch.tensor(numeric).reshape(1, -1).float().to(self.device)
+
+        binary = []
+        for bin_name in self.bin_names:
+            value = data[bin_name]
+            code = self.binary_fts[bin_name][value]
+            binary.append(int(code))
+        bin = torch.tensor(binary).reshape(1, -1).float().to(self.device)
+        codes = []
+        for ft in self.categorical_fts:
+            category = data[ft]
+            code = self.categorical_fts[ft]['cats'].index(category)
+            code = torch.tensor(code).to(self.device)
+            codes.append(code)
+        return num, bin, codes
+
+    def encode_input_dict(self, data):
+        """
+        Handles raw df inputs.
+        Passes categories through embedding layers.
+        """
+        num, bin, codes = self.compute_targets_dict(data)
+        embeddings = []
+        for i, ft in enumerate(self.categorical_fts):
+            feature = self.categorical_fts[ft]
+            emb = feature['embedding'](codes[i]).reshape(1, -1)
+            embeddings.append(emb)
+        return [num], [bin], embeddings
+
+    def get_deep_stack_features_json(self, data):
+        """
+        gets "deep stack" features for a single record;
+        intended for executing "inference" logic for a
+        network request.
+        data can either be a json string or a dict.
+        """
+        if isinstance(data, str):
+            data = self._deserialize_json(data)
+
+        self.eval()
+
+        with torch.no_grad():
+            this_batch = []
+            num, bin, embeddings = self.encode_input_dict(data)
+            x = torch.cat(num + bin + embeddings, dim=1)
+            for layer in self.encoder:
+                x = layer(x)
+                this_batch.append(x)
+            for layer in self.decoder:
+                x = layer(x)
+                this_batch.append(x)
+            z = torch.cat(this_batch, dim=1)
+        return z
+
     def get_anomaly_score(self, df):
         """
         Returns a per-row loss of the input dataframe.
@@ -957,3 +1046,11 @@ def df_predict(self, df):
             output_df = self.decode_to_df(x, df=df)
 
         return output_df
+
+    def save(self, path):
+        """
+        Saves serialized model to input path.
+        """
+        with open(path, 'wb') as f:
+            serialized_model = dill.dumps(self)
+            f.write(serialized_model)

demo_data/saving_and_loading.ipynb

@@ -49,6 +49,9 @@ def end_epoch(self):
                 self.id_val_fts[ft][1].append(mean)
                 #reset id_val_fts log
                 self.id_val_fts[ft][0] = []
+    
+    def show_embeddings(self, categories):
+        pass
 
 class IpynbLogger(BasicLogger):
     """Plots Logging Data in jupyter notebook"""

dfencoder/autoencoder.py

@@ -15,7 +15,8 @@
     'scikit-learn',
     'tensorboardX',
     'matplotlib', 
-    'wheel'
+    'wheel',
+    'dill'
 ]
 version = '0.0.36'
 

dfencoder/logging.py

@@ -2,6 +2,7 @@
 import time
 import os
 import shutil
+import json
 from collections import OrderedDict
 
 import pandas as pd
@@ -22,6 +23,37 @@ def tearDown(self):
         t = time.time() - self.startTime
         print("%s: %.3f seconds" % (self.id(), t))
 
+class ModelBuilder(object):
+
+    def __init__(self):
+        self.model = None
+        self.out_df = None
+
+    def build_model(self):
+        if self.model is None:
+            encoder = AutoEncoder(
+                encoder_layers=[32, 32],
+                decoder_layers=[32, 32],
+                encoder_dropout=.5,
+                decoder_dropout=[.2, None],
+                activation='tanh',
+                swap_p=.2,
+                batch_size=123,
+                optimizer='sgd',
+                lr_decay=.95
+            )
+            encoder.build_model(df)
+            out_df = encoder.prepare_df(df)
+            assert not out_df.isna().any().any()
+            layers_count = 0
+            for prm in encoder.parameters():
+                layers_count += 1
+            assert layers_count == 33
+            self.model, self.out_df = encoder, out_df
+            return encoder, out_df
+        else:
+            return self.model, self.out_df
+
 class TestCompleteLayer(TimedCase):
     def test_init(self):
         layer = CompleteLayer(12, 5, activation='sigmoid', dropout=.2)
@@ -165,6 +197,17 @@ def test_fit(self):
         assert data.shape == sample.shape
         return encoder
 
+    def test_inference(self):
+        record = df.sample()
+        js = record.iloc[0].to_json()
+        output = model._deserialize_json(js)
+        z_json = model.get_deep_stack_features_json(js)
+        dct = json.loads(js)
+        z_dict = model.get_deep_stack_features_json(dct)
+        z = model.get_deep_stack_features(record)
+        assert (z_json == z).all()
+        assert (z_json == z_dict).all()
+
     def test_get_representation(self):
         encoder = AutoEncoder()
         sample = df.sample(1025)
@@ -314,6 +357,8 @@ def test_null_indicator(self):
 if __name__ == '__main__':
     os.mkdir('_testlog')
     df = pd.read_csv('adult.csv')
+    b = ModelBuilder()
+    model, _ = b.build_model()
     unittest.main(exit=False)
     shutil.rmtree('_testlog')
     quit()