diff --git a/configs/config_ConvNet.py b/configs/config_ConvNet.py index 4b23a05..5a474f7 100644 --- a/configs/config_ConvNet.py +++ b/configs/config_ConvNet.py @@ -21,25 +21,24 @@ def load_config(): config = ConfigFlags().return_flags() config.net = 'ConvNet' - config.training_mode = True + config.training_mode = False config.data_set = 'MNIST' config.image_size = [28, 28, 1] - config.lr = 0.001 config.lr_decay = 0.1 - config.ref_steps = 10 - config.ref_patience = 1 + config.ref_steps = 3 + config.ref_patience = 3 config.batch_size = 32 - config.num_epochs = 1000 + config.num_epochs = 3 config.loss = 'softmax' - config.optimizer = 'adam' + config.optimizer = 'sgd' config.gradcam_record = True config.gradcam_layers = 6 config.gradcam_layers_max = 6 config.mi_record = False config.gpu_load = 0.8 config.num_classes = 10 - config.class_labels = [i for i in range(9)] + config.class_labels = [str(i) for i in range(10)] config.num_filters = 16 config.upconv = 'upconv' config.nonlin = 'relu' @@ -53,10 +52,11 @@ def load_config(): config.normalize = True config.zero_center = True config.dropout = 0.4 - config.chpnt2load = '' + # config.chpnt2load = r'E:\SCCH_PROJECTS\DL_implementaitons\00_templates\training_engine_git\experiments\ckpnt_logs\MNIST\1565681155.1648371\1565681155.1648371_split_0' + config.chpnt2load = r'experiments/ckpnt_logs/MNIST/1565680688.729741/1565680688.729741_split_0' config.multi_task = False config.cross_val = 1 - config.framework = 'tensorflow' + config.framework = 'pytorch' config.experiment_path = None diff --git a/configs/config_ONNXtest.py b/configs/config_ONNXtest.py new file mode 100644 index 0000000..0ccec1a --- /dev/null +++ b/configs/config_ONNXtest.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 5/9/2019 3:49 PM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 3/21/2019 8:30 AM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + + +from configs.config import ConfigFlags + + +def load_config(): + config = ConfigFlags().return_flags() + + config.net = r'E:\o156.onnx' + + config.autotune = False + config.training_mode = True + + config.data_set = 'CIFAR10' + config.image_size = [32, 32, 3] + config.data_folder = None + config.data_file = None + + + config.lr = 0.001 + config.lr_decay = 0.1 + config.ref_steps = 10 + config.ref_patience = 10 + config.batch_size = 128 + config.num_epochs = 2 + config.loss = 'softmax' + config.optimizer = 'sgd' + config.gradcam_record = True + config.gradcam_layers = 6 + config.gradcam_layers_max = 6 + config.mi_record = False + config.gpu_load = 0.8 + config.num_classes = 10 + config.class_labels = [i for i in range(9)] + config.num_filters = 64 + config.upconv = 'upconv' + config.nonlin = 'relu' + config.task_type = 'classification' + config.accuracy = 'percent' + config.augmentation = {'flip_hor': False, + 'flip_vert': False} + config.data_split = 0.7 + config.long_summary = True + config.trainable_layers = 'all' + config.normalize = True + config.zero_center = True + config.dropout = 0.4 + config.chpnt2load = '' + config.multi_task = False + config.cross_val = 1 + config.framework = 'tensorflow' + config.experiment_path = None + + + return config \ No newline at end of file diff --git a/configs/config_UNet.py b/configs/config_UNet.py new file mode 100644 index 0000000..2469e02 --- /dev/null +++ b/configs/config_UNet.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 01/08/2018 16:29 $ +# by : shepeleva $ +# SVN : $ +# + +# --- imports ----------------------------------------------------------------- + +from configs.config import ConfigFlags + + +def load_config(): + config = ConfigFlags().return_flags() + + config.autotune = False + + config.net = 'UNet' + config.training_mode = True + config.data_set = 'rubble_sample' + config.data_dir = r'F:\Datasets\Rubblemaster_segmentation' + config.image_size = [1024, 1024, 1] + + config.lr = 0.001 + config.lr_decay = 0.1 + config.ref_steps = 3 + config.ref_patience = 3 + config.batch_size = 2 + config.num_epochs = 100 + config.loss = 'dice_jaccard' + config.optimizer = 'sgd' + config.gradcam_record = True + config.gradcam_layers = 6 + config.gradcam_layers_max = 6 + config.mi_record = False + config.gpu_load = 0.8 + config.num_classes = 1 + config.class_labels = [i for i in range(1)] + config.num_filters = 8 + config.upconv = 'upconv' + config.nonlin = 'relu' + config.task_type = 'segmentation' + config.accuracy = 'IoU' + config.augmentation = {'flip_hor': False, + 'flip_vert': False} + config.data_split = 0.7 + config.long_summary = True + config.trainable_layers = 'all' + config.normalize = True + config.zero_center = True + config.dropout = 0.4 + config.chpnt2load = '' + config.multi_task = False + config.cross_val = 1 + config.framework = 'pytorch' + config.experiment_path = None + + + return config \ No newline at end of file diff --git a/configs/config_VGG19.py b/configs/config_VGG19.py new file mode 100644 index 0000000..dd377ef --- /dev/null +++ b/configs/config_VGG19.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 3/21/2019 8:30 AM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + + +from configs.config import ConfigFlags + + +def load_config(): + config = ConfigFlags().return_flags() + + config.net = 'VGG19' + + config.autotune = False + config.training_mode = True + + config.data_set = 'CIFAR10' + config.image_size = [32, 32, 3] + config.data_folder = None + config.data_file = None + + + config.lr = 0.001 + config.lr_decay = 0.1 + config.ref_steps = 10 + config.ref_patience = 10 + config.batch_size = 128 + config.num_epochs = 2 + config.loss = 'softmax' + config.optimizer = 'sgd' + config.gradcam_record = True + config.gradcam_layers = 6 + config.gradcam_layers_max = 6 + config.mi_record = False + config.gpu_load = 0.8 + config.num_classes = 10 + config.class_labels = [i for i in range(9)] + config.num_filters = 64 + config.upconv = 'upconv' + config.nonlin = 'relu' + config.task_type = 'classification' + config.accuracy = 'percent' + config.augmentation = {'flip_hor': False, + 'flip_vert': False} + config.data_split = 0.7 + config.long_summary = True + config.trainable_layers = 'all' + config.normalize = True + config.zero_center = True + config.dropout = 0.4 + config.chpnt2load = '' + config.multi_task = False + config.cross_val = 1 + config.framework = 'tensorflow' + config.experiment_path = None + + + return config \ No newline at end of file diff --git a/main.py b/main.py index efcda3f..47f8272 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,6 @@ EXPERIMENT_ID = 'ConvNet' - def run(experiment_id): config = importlib.import_module('configs.config_' + experiment_id) args = config.load_config() @@ -32,7 +31,7 @@ def run(experiment_id): else: inference = InferenceRunner(experiment_id=experiment_id) inference.start_inference() - raise ValueError('Inference not implemented yet') + # raise ValueError('Inference not implemented yet') if __name__ == '__main__': diff --git a/network/InferenceRunner.py b/network/InferenceRunner.py index 1e54d7f..597b7e9 100644 --- a/network/InferenceRunner.py +++ b/network/InferenceRunner.py @@ -15,10 +15,9 @@ # --- imports ----------------------------------------------------------------- import os -import time +import torch from tqdm import tqdm -import h5py import numpy as np import tensorflow as tf import matplotlib.pyplot as plt @@ -27,9 +26,9 @@ from sklearn.metrics import classification_report, confusion_matrix, accuracy_score from network.NetRunner import NetRunner -from utils.BatchIterator import BatchIterator -from utils.gradcam.GradCAM import grad_cam_plus_plus as gradcam -from utils.gradcam.GradCAM import save_gradcam_data as save_gradcam +# from utils.BatchIterator import BatchIterator +# from utils.gradcam.GradCAM import grad_cam_plus_plus as gradcam +# from utils.gradcam.GradCAM import save_gradcam_data as save_gradcam class InferenceRunner(NetRunner): def __init__(self, args=None, experiment_id=None): @@ -41,11 +40,14 @@ def start_inference(self): def _initialize_inference(self): if self.framework is 'tensorflow': self._run_tensorflow_pipeline() + self._result_eval() elif self.framework is 'pytorch': self._run_pytorch_pipeline() + self._result_eval() else: raise ValueError('Framework is not supported') + def _run_tensorflow_pipeline(self): tf.reset_default_graph() if self.chpnt2load: @@ -55,8 +57,6 @@ def _run_tensorflow_pipeline(self): raise ValueError('Mo such checkpoint is found ' + self.chpnt2load) else: raise ValueError("No checkpoint provided") - # model_file_path = os.path.join('experiments', 'ckpnt_logs', self.data_set, self.timestamp, - # self.timestamp + '_split_' + str(split)) saver = tf.train.import_meta_graph(latest_ckpt + '.meta') graph = tf.get_default_graph() @@ -68,162 +68,123 @@ def _run_tensorflow_pipeline(self): os.environ['CUDA_VISIBLE_DEVICES'] = '-1' config = tf.ConfigProto(device_count={'CPU': 1}) - # h5_file = h5py.File(self.h5_data_file, 'r') - with tf.Session(graph=graph, config=config) as sess: saver.restore(sess, latest_ckpt) X = tf.get_collection("inputs_train")[0] pred = tf.get_collection("outputs_train")[0] mode = tf.get_collection_ref("mode_train")[0] - y_true = tf.get_collection("y_true") - if self.gradcam_record: - y_true = tf.get_collection("y_true")[0] - gradcam_in_data = tf.get_collection("gradcam_in_data")[0] - gradcam_gb_grads = tf.get_collection("gradcam_gb_grads")[0] - gradcam_conv_outs = [] - gradcam_first_derivatives = [] - gradcam_second_derivatives = [] - gradcam_third_derivatives = [] - for i in range(self.gradcam_layers): - gradcam_conv_outs.append(tf.get_collection("gradcam_conv_outs_{}".format(i))[0]) - gradcam_first_derivatives.append(tf.get_collection("gradcam_first_derivatives_{}".format(i))[0]) - gradcam_second_derivatives.append(tf.get_collection("gradcam_second_derivatives_{}".format(i))[0]) - gradcam_third_derivatives.append(tf.get_collection("gradcam_third_derivatives_{}".format(i))[0]) - gradcam_op = [gradcam_in_data, gradcam_gb_grads, gradcam_conv_outs, gradcam_first_derivatives, - gradcam_second_derivatives, gradcam_third_derivatives] - pred_result = [] - gradcam_results = [] - - # test_generator = BatchIterator(h5_file, 'test', self.img_size, self.num_classes, 1, self.task_type) - # test_lim = test_generator.get_max_lim() + + self.pred_result = [] count = 0 - for i in tqdm(len(self.inference_X), total=len(self.inference_X), unit=' steps', desc='Inference'): - start_time = time.time() - ff = list(i) + for i in tqdm(range(0, len(self.inference_X)), total=len(self.inference_X), unit=' steps', + desc='Inference'): if self.task_type == 'classification': pred_result_ = sess.run(pred, feed_dict={ - X: [np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f - in ff], mode: False}) - pred_result.extend(pred_result_) - # Calculate Grad-CAM parameters - if self.gradcam_record: - gradcam_params = sess.run(gradcam_op, feed_dict={ - X: [np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f - in ff], mode: False, y_true: pred_result_}) - pred_result.extend(pred_result_) - gradcam_results_ = [] - # Run Grad-CAM - for cl in range(self.gradcam_layers): - gradcam_results_.append((gradcam(gradcam_params, cl), gradcam_params[0])) - count = count + 1 - # Save Grad-CAM results - save_gradcam(gradcam_results_, self.data_dir, count, self.gradcam_layers_max) - # pred_result = sess.run(pred, feed_dict={X: [np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f - # in ff], mode: False}) + X: np.reshape(self.inference_X[i], + [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + self.pred_result.extend(pred_result_) + elif self.task_type == 'segmentation': - # if self.y: - for fr in ff: - # <<<<<<< .mine - # mask_pred = sess.run(pred, feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) - # mask = tf.nn.sigmoid(mask_pred).eval() - # m_r = np.argmax(mask, axis=3) - # f, (ax1, ax2) = plt.subplots(1, 2) - # # f_r = np.argmax(fr[1], axis=2) - # ax2.imshow(fr[0]) - # ax1.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) - # # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) - # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file - # plt.close(f) - # plt.show() - - # mask_pred = sess.run(pred, feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) - # ||||||| .r165 - mask_pred = sess.run(pred, feed_dict={ - X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), - mode: False}) - # ======= - # mask_pred, gradcam_params_ = sess.run([pred, gradcam_op], feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) - # >>>>>>> .r281 - # mask = tf.nn.sigmoid(mask_pred).eval() - # m_r = np.argmax(mask, axis=3) - # f, (ax1, ax2, ax3) = plt.subplots(1, 3) - # f_r = np.argmax(fr[1], axis=2) - # ax3.imshow(fr[0]) - # ax3.set(xlabel='original mask') - # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) - # ax2.set(xlabel='generated segmentation') - # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) - # ax1.set(xlabel='original image') - # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file - # plt.close(f) - - # mask = tf.nn.sigmoid(mask_pred).eval() - # m_r = np.argmax(mask, axis=3) - # f, (ax1, ax2, ax3) = plt.subplots(1, 3) - # # f_r = np.argmax(fr[1], axis=2) - # ax3.imshow(fr[0]) - # ax3.set(xlabel='original mask') - # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) - # # ax2.set(xlabel='generated segmentation') - # # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) - # ax1.set(xlabel='original image') - # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file - # plt.close(f) - - mask = tf.nn.sigmoid(mask_pred).eval() - m_r = np.argmax(mask, axis=3) - f, (ax1, ax2) = plt.subplots(1, 2) - # f_r = np.argmax(fr[1], axis=2) - - ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) - ax2.set(xlabel='generated segmentation') - ax1.imshow(np.reshape(fr[0], [self.img_size[0], self.img_size[1]])) - ax1.set(xlabel='original image') - f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png'), figsize=(24, 18), - dpi=600) # save the figure to file - plt.close(f) - - count = count + 1 + mask_pred = sess.run(pred, feed_dict={ + X: np.reshape(self.inference_X[i], + [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + + mask = tf.nn.sigmoid(mask_pred).eval() + m_r = np.argmax(mask, axis=3) + f, (ax1, ax2) = plt.subplots(1, 2) + + ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + ax2.set(xlabel='generated segmentation') + ax1.imshow(np.reshape(self.inference_X[i], [self.img_size[0], self.img_size[1]])) + ax1.set(xlabel='original image') + f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png'), figsize=(24, 18), + dpi=600) # save the figure to file + plt.close(f) + + count = count + 1 + else: + raise ValueError('No inference for this task type implemented') - if self.task_type == 'classification': - if self.loss_type == 'mse': - m = 0 - y_pred = np.array(pred_result) - y_true = np.array(h5_file['lbl_test'][:]) - for i in range(0, len(y_pred)): - mse_test = np.sqrt(np.sum(((y_true[i] - y_pred[i]) ** 2) / 2)) - # mse_test = sess.run(mse_loss(y_true[i], y_pred[i])) - m += mse_test - print("Prediction: {} GT: {} Score {}%".format(str(y_pred[i]), str(y_true[i]), - int(mse_test * 100))) - - # mse_test = np.mean(np.sqrt(np.sum(((y_true - y_pred) ** 2)/2))) - print("Overall: {}%".format(int(m / len(y_pred) * 100))) - else: - y_pred = np.argmax(np.array(pred_result).squeeze(), axis=1) - # one hot to label encoding - 1 = FRAUD, 0 = OK - # y_train = [1 if np.all(yt == np.array([0, 1])) else 0 for yt in y_train] - y_true = np.array(h5_file['lbl_test'][:]) - if len(y_true.shape) != 1: - y_true = np.array([np.argmax(yt) for yt in y_true]) + def _run_pytorch_pipeline(self): + cuda_en = torch.cuda.is_available() + if self.gpu_load != 0 and cuda_en: + device = torch.device('cuda') + print("CUDA detecvted") + else: + device = torch.device('cpu') + print("CPU detecvted") - self.plot_confusion_matrix(y_true, y_pred, self.class_labels, - self.data_dir + '/CNN_confusion_matrix_{}.png'.format(self.data_set)) - class_report = classification_report(y_true, y_pred, labels=[i for i in range(0, self.num_classes)], - target_names=self.class_labels) - with open(self.data_dir + "/classification_report_{}.txt".format(self.data_set), 'w') as f: - f.write('RF accuracy score on test set is {:.2f}\n\n'.format( - accuracy_score(y_true, y_pred) * 100)) - f.write(class_report) - print(class_report) + model = torch.load(os.path.join(self.chpnt2load, 'model.pth')) + model.to(device) + model.eval() + # return NotImplementedError + self.pred_result = [] + count = 0 + for i in tqdm(range(0, len(self.inference_X)), total=len(self.inference_X), unit=' steps', + desc='Inference'): + if self.task_type == 'classification': + pred_result_ = model(torch.from_numpy(np.reshape(self.inference_X[i], + [1, self.img_size[2], self.img_size[0], self.img_size[1]])).to(device, dtype=torch.float)).to(device) + self.pred_result.extend(pred_result_.cpu().detach().numpy()) + + elif self.task_type == 'segmentation': + mask_pred = model(torch.from_numpy(np.reshape(self.inference_X[i], + [1, self.img_size[2], self.img_size[0], self.img_size[1]])).to(device, dtype=torch.float)).to(device) + + # mask = tf.nn.sigmoid(mask_pred).eval() + # m_r = np.argmax(mask, axis=3) + # f, (ax1, ax2) = plt.subplots(1, 2) + # + # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + # ax2.set(xlabel='generated segmentation') + # ax1.imshow(np.reshape(self.inference_X[i], [self.img_size[0], self.img_size[1]])) + # ax1.set(xlabel='original image') + # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png'), figsize=(24, 18), + # dpi=600) # save the figure to file + # plt.close(f) + # + # count = count + 1 + else: + raise ValueError('No inference for this task type implemented') + + + def _result_eval(self): + if self.task_type == 'classification': + + if self.loss_type == 'mse': + m = 0 + y_pred = np.array(self.pred_result) + y_true = np.array(self.inference_y) + for i in range(0, len(y_pred)): + mse_test = np.sqrt(np.sum(((y_true[i] - y_pred[i]) ** 2) / 2)) + # mse_test = sess.run(mse_loss(y_true[i], y_pred[i])) + m += mse_test + print("Prediction: {} GT: {} Score {}%".format(str(y_pred[i]), str(y_true[i]), + int(mse_test * 100))) + + # mse_test = np.mean(np.sqrt(np.sum(((y_true - y_pred) ** 2)/2))) + print("Overall: {}%".format(int(m / len(y_pred) * 100))) + else: + y_pred = np.argmax(np.array(self.pred_result).squeeze(), axis=1) + y_true = np.array(self.inference_y) + if len(y_true.shape) != 1: + y_true = np.array([np.argmax(yt) for yt in y_true]) + + self.plot_confusion_matrix(y_true, y_pred, self.class_labels, + self.data_dir + '/CNN_confusion_matrix_{}.png'.format(self.data_set)) + class_report = classification_report(y_true, y_pred, labels=[i for i in range(0, self.num_classes)], + target_names=self.class_labels) + with open(self.data_dir + "/classification_report_{}.txt".format(self.data_set), 'w') as f: + f.write('RF accuracy score on test set is {:.2f}\n\n'.format( + accuracy_score(y_true, y_pred) * 100)) + f.write(class_report) + print(class_report) - def _run_pytorch_pipeline(self): - return NotImplementedError def plot_confusion_matrix(self, y_true, y_pred, classes, save_path, normalize=False, cmap=plt.cm.Blues): """ @@ -271,4 +232,185 @@ def plot_confusion_matrix(self, y_true, y_pred, classes, save_path, normalize=Fa fig.savefig(save_path) plt.close(fig) - return cm \ No newline at end of file + return cm + + + # def _run_tensorflow_pipeline_dep(self): + # tf.reset_default_graph() + # if self.chpnt2load: + # try: + # latest_ckpt = tf.train.latest_checkpoint(self.chpnt2load) + # except: + # raise ValueError('Mo such checkpoint is found ' + self.chpnt2load) + # else: + # raise ValueError("No checkpoint provided") + # # model_file_path = os.path.join('experiments', 'ckpnt_logs', self.data_set, self.timestamp, + # # self.timestamp + '_split_' + str(split)) + # + # saver = tf.train.import_meta_graph(latest_ckpt + '.meta') + # graph = tf.get_default_graph() + # if self.gpu_load != 0: + # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=self.gpu_load) + # config = tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False) + # config.gpu_options.allow_growth = True + # else: + # os.environ['CUDA_VISIBLE_DEVICES'] = '-1' + # config = tf.ConfigProto(device_count={'CPU': 1}) + # + # # h5_file = h5py.File(self.h5_data_file, 'r') + # + # with tf.Session(graph=graph, config=config) as sess: + # saver.restore(sess, latest_ckpt) + # + # X = tf.get_collection("inputs_train")[0] + # pred = tf.get_collection("outputs_train")[0] + # mode = tf.get_collection_ref("mode_train")[0] + # y_true = tf.get_collection("y_true") + # # if self.gradcam_record: + # # y_true = tf.get_collection("y_true")[0] + # # gradcam_in_data = tf.get_collection("gradcam_in_data")[0] + # # gradcam_gb_grads = tf.get_collection("gradcam_gb_grads")[0] + # # gradcam_conv_outs = [] + # # gradcam_first_derivatives = [] + # # gradcam_second_derivatives = [] + # # gradcam_third_derivatives = [] + # # for i in range(self.gradcam_layers): + # # gradcam_conv_outs.append(tf.get_collection("gradcam_conv_outs_{}".format(i))[0]) + # # gradcam_first_derivatives.append(tf.get_collection("gradcam_first_derivatives_{}".format(i))[0]) + # # gradcam_second_derivatives.append(tf.get_collection("gradcam_second_derivatives_{}".format(i))[0]) + # # gradcam_third_derivatives.append(tf.get_collection("gradcam_third_derivatives_{}".format(i))[0]) + # # gradcam_op = [gradcam_in_data, gradcam_gb_grads, gradcam_conv_outs, gradcam_first_derivatives, + # # gradcam_second_derivatives, gradcam_third_derivatives] + # self.pred_result = [] + # # gradcam_results = [] + # + # # test_generator = BatchIterator(h5_file, 'test', self.img_size, self.num_classes, 1, self.task_type) + # # test_lim = test_generator.get_max_lim() + # + # count = 0 + # for i in tqdm(range(0, len(self.inference_X)), total=len(self.inference_X), unit=' steps', desc='Inference'): + # start_time = time.time() + # # ff = list(i) + # if self.task_type == 'classification': + # pred_result_ = sess.run(pred, feed_dict={ + # X: np.reshape(self.inference_X[i], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + # self.pred_result.extend(pred_result_) + # # Calculate Grad-CAM parameters + # # if self.gradcam_record: + # # gradcam_params = sess.run(gradcam_op, feed_dict={ + # # X: np.reshape(self.inference_X[i], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False, y_true: pred_result_}) + # # # pred_result.extend(pred_result_) + # # gradcam_results_ = [] + # # # Run Grad-CAM + # # for cl in range(self.gradcam_layers): + # # gradcam_results_.append((gradcam(gradcam_params, cl), gradcam_params[0])) + # # count = count + 1 + # # # Save Grad-CAM results + # # save_gradcam(gradcam_results_, self.data_dir, count, self.gradcam_layers_max) + # # pred_result = sess.run(pred, feed_dict={X: [np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f + # # in ff], mode: False}) + # + # + # + # + # elif self.task_type == 'segmentation': + # # if self.y: + # mask_pred = sess.run(pred, feed_dict={ + # X: np.reshape(self.inference_X[i], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + # + # # + # # for fr in ff: + # # <<<<<<< .mine + # # mask_pred = sess.run(pred, feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + # # mask = tf.nn.sigmoid(mask_pred).eval() + # # m_r = np.argmax(mask, axis=3) + # # f, (ax1, ax2) = plt.subplots(1, 2) + # # # f_r = np.argmax(fr[1], axis=2) + # # ax2.imshow(fr[0]) + # # ax1.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + # # # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) + # # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file + # # plt.close(f) + # # plt.show() + # # + # # mask_pred = sess.run(pred, feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + # # ||||||| .r165 + # # mask_pred = sess.run(pred, feed_dict={ + # # X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), + # # mode: False}) + # # ======= + # # mask_pred, gradcam_params_ = sess.run([pred, gradcam_op], feed_dict={X: np.reshape(fr[0], [1, self.img_size[0], self.img_size[1], self.img_size[2]]), mode: False}) + # # >>>>>>> .r281 + # # mask = tf.nn.sigmoid(mask_pred).eval() + # # m_r = np.argmax(mask, axis=3) + # # f, (ax1, ax2, ax3) = plt.subplots(1, 3) + # # f_r = np.argmax(fr[1], axis=2) + # # ax3.imshow(fr[0]) + # # ax3.set(xlabel='original mask') + # # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + # # ax2.set(xlabel='generated segmentation') + # # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) + # # ax1.set(xlabel='original image') + # # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file + # # plt.close(f) + # # + # # mask = tf.nn.sigmoid(mask_pred).eval() + # # m_r = np.argmax(mask, axis=3) + # # f, (ax1, ax2, ax3) = plt.subplots(1, 3) + # # # f_r = np.argmax(fr[1], axis=2) + # # ax3.imshow(fr[0]) + # # ax3.set(xlabel='original mask') + # # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + # # # ax2.set(xlabel='generated segmentation') + # # # ax1.imshow(np.reshape(f_r, [self.img_size[0], self.img_size[1]])) + # # ax1.set(xlabel='original image') + # # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png')) # save the figure to file + # # plt.close(f) + # + # mask = tf.nn.sigmoid(mask_pred).eval() + # m_r = np.argmax(mask, axis=3) + # f, (ax1, ax2) = plt.subplots(1, 2) + # # f_r = np.argmax(fr[1], axis=2) + # + # ax2.imshow(np.reshape(m_r, [self.img_size[0], self.img_size[1]])) + # ax2.set(xlabel='generated segmentation') + # ax1.imshow(np.reshape(self.inference_X[i], [self.img_size[0], self.img_size[1]])) + # ax1.set(xlabel='original image') + # f.savefig(os.path.join(self.data_dir, 'results', str(count) + '_.png'), figsize=(24, 18), + # dpi=600) # save the figure to file + # plt.close(f) + # + # count = count + 1 + # + # if self.task_type == 'classification': + # + # if self.loss_type == 'mse': + # m = 0 + # y_pred = np.array(self.pred_result) + # y_true = np.array(self.inference_y) + # for i in range(0, len(y_pred)): + # mse_test = np.sqrt(np.sum(((y_true[i] - y_pred[i]) ** 2) / 2)) + # # mse_test = sess.run(mse_loss(y_true[i], y_pred[i])) + # m += mse_test + # print("Prediction: {} GT: {} Score {}%".format(str(y_pred[i]), str(y_true[i]), + # int(mse_test * 100))) + # + # # mse_test = np.mean(np.sqrt(np.sum(((y_true - y_pred) ** 2)/2))) + # print("Overall: {}%".format(int(m / len(y_pred) * 100))) + # else: + # y_pred = np.argmax(np.array(self.pred_result).squeeze(), axis=1) + # # one hot to label encoding - 1 = FRAUD, 0 = OK + # # y_train = [1 if np.all(yt == np.array([0, 1])) else 0 for yt in y_train] + # y_true = np.array(self.inference_y) + # if len(y_true.shape) != 1: + # y_true = np.array([np.argmax(yt) for yt in y_true]) + # + # self.plot_confusion_matrix(y_true, y_pred, self.class_labels, + # self.data_dir + '/CNN_confusion_matrix_{}.png'.format(self.data_set)) + # class_report = classification_report(y_true, y_pred, labels=[i for i in range(0, self.num_classes)], + # target_names=self.class_labels) + # with open(self.data_dir + "/classification_report_{}.txt".format(self.data_set), 'w') as f: + # f.write('RF accuracy score on test set is {:.2f}\n\n'.format( + # accuracy_score(y_true, y_pred) * 100)) + # f.write(class_report) + # print(class_report) diff --git a/network/NetRunner.py b/network/NetRunner.py index b54463b..3c54a37 100644 --- a/network/NetRunner.py +++ b/network/NetRunner.py @@ -15,9 +15,16 @@ # --- imports ----------------------------------------------------------------- import importlib import tensorflow as tf +import torch +import tf2onnx + +import onnx +from onnx_tf.backend import prepare + +from utils.VisdomLogger import * from utils.DataParser import DataParser -from network.wrappers import ConvNet, UNet, FakeDetection +from network.wrappers import ConvNet, VGG19, UNet class NetRunner: @@ -30,6 +37,7 @@ def __init__(self, args=None, experiment_id=None): self.X_valid = None self.y_valid = None self.num_classes = None + self._parse_config(args, experiment_id) def _parse_config(self, args, experiment_id): @@ -39,14 +47,12 @@ def _parse_config(self, args, experiment_id): :param experiment_id: :return: """ - if not args: if experiment_id: config = importlib.import_module('configs.config_' + experiment_id) args = config.load_config() else: - config = importlib.import_module('configs.config') - args = config.ConfigFlags().return_flags() + raise ValueError('No arguments or configuration data given') # Mandatory parameters for all architectures self.network_type = args.net self.is_training = args.training_mode @@ -242,13 +248,21 @@ def build_tensorflow_pipeline(self): self.graph_op = tf.global_variables_initializer() def build_pytorch_pipeline(self): + cuda_en = torch.cuda.is_available() + if self.gpu_load != 0 and cuda_en: + self.device = torch.device('cuda') + print("CUDA detecvted") + else: + self.device = torch.device('cpu') + print("CPU detecvted") + + self.learning_rate = self.lr - self.network = self._pick_model() + self.network = self._pick_model().to(self.device) self.train_op = self.network.return_optimizer(net_param=self.network.parameters()) - return NotImplementedError - + self.graph_op = VisdomPlotter(env_name='Training_Procedure') def _pick_model(self): """ @@ -256,30 +270,40 @@ def _pick_model(self): :return: """ if self.framework == "tensorflow": - if self.network_type == 'ConvNet': - return ConvNet.ConvNet(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, - training=self.is_training, num_filters=self.num_filters, nonlin=self.nonlin, num_classes=self.num_classes, - trainable_layers=self.trainable_layers) - elif self.network_type == 'UNet': - return UNet.UNet(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, training=self.training_mode, - trainable_layers=self.trainable_layers, num_classes=self.num_classes) - # elif self.network_type == 'VGG16': - # # return VGG16.VGG16(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, - # # training=self.training_mode, framework=self.framework, num_classes=self.num_classes, trainable_layers=self.trainable_layers) - elif self.network_type == 'FakeDetection': - return FakeDetection.FakeDetection(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, training=self.training_mode, - trainable_layers=self.trainable_layers, num_classes=self.num_classes) + if self.network_type.endswith('.onnx'): + print('onnx!') + return 'onxx' else: - raise ValueError('Architecture does not exist') + if self.network_type == 'ConvNet': + return ConvNet.ConvNet(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, + training=self.is_training, num_filters=self.num_filters, nonlin=self.nonlin, num_classes=self.num_classes, + trainable_layers=self.trainable_layers) + elif self.network_type == 'UNet': + return UNet.UNet(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, training=self.training_mode, + trainable_layers=self.trainable_layers, num_classes=self.num_classes) + elif self.network_type == 'VGG19': + return VGG19.VGG19_tf(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, + training=self.training_mode, framework=self.framework, num_classes=self.num_classes, trainable_layers=self.trainable_layers) + + else: + raise ValueError('Architecture does not exist') elif self.framework == "pytorch": if self.network_type == 'ConvNet': return ConvNet.ConvNet_pt(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, training=self.is_training, num_filters=self.num_filters, nonlin=self.nonlin, num_classes=self.num_classes, trainable_layers=self.trainable_layers) + elif self.network_type == 'VGG19': + return VGG19.VGG19_pt(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, + training=self.is_training, num_filters=self.num_filters, nonlin=self.nonlin, num_classes=self.num_classes, + trainable_layers=self.trainable_layers) + elif self.network_type == 'UNet': + return UNet.UNet_pt(self.network_type, self.loss_type, self.accuracy_type, self.learning_rate, framework=self.framework, + training=self.is_training, num_filters=self.num_filters, nonlin=self.nonlin, num_classes=self.num_classes, + trainable_layers=self.trainable_layers) + else: raise ValueError('Architecture does not exist') - def _initialize_short_summary(self): """ Tensorboard scope initialization diff --git a/network/TrainRunner.py b/network/TrainRunner.py index b16957e..eb9cddd 100644 --- a/network/TrainRunner.py +++ b/network/TrainRunner.py @@ -19,6 +19,7 @@ import time import h5py import json +import tf2onnx import numpy as np import tensorflow as tf from tqdm import tqdm @@ -46,19 +47,22 @@ def start_training(self): Start Neural Network training :return: """ - # os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # training initialisation # with tf.device('/cpu:0'): # self._initialize_data() - self._initialize_training() + validation_scores = self._initialize_training() + return validation_scores def _initialize_training(self): if self.framework == 'tensorflow': + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' self.build_tensorflow_pipeline() - self._run_tensorflow_pipeline() + valid_loss_final = self._run_tensorflow_pipeline() + return valid_loss_final elif self.framework == 'pytorch': self.build_pytorch_pipeline() - self._run_pytorch_pipeline() + valid_loss_final =self._run_pytorch_pipeline() + return valid_loss_final else: raise ValueError('Framework is not supported') @@ -67,11 +71,7 @@ def _run_tensorflow_pipeline(self): if self.gpu_load != 0: gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=self.gpu_load) - # config = tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True, - # device_count={"CPU": 4}, - # inter_op_parallelism_threads=4, - # intra_op_parallelism_threads=5,) - config = tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True) + config = tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False) config.gpu_options.allow_growth = True else: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' @@ -153,7 +153,7 @@ def _run_tensorflow_pipeline(self): epoch_duration_valid = 0 # step_cout_train = 1 - for i in tqdm(train_generator, total=train_lim, unit=' steps', desc='Epoch {:d} train'.format(epoch)): + for i in tqdm(train_generator, total=train_lim, unit=' steps', desc='Epoch {:d} train'.format(epoch), disable=True): with tf.device('/cpu:0'): total_recall_counter_train += 1 start_time = time.time() @@ -198,7 +198,7 @@ def _run_tensorflow_pipeline(self): # step_cout_train += 1 # step_cout_valid = 1 - for i in tqdm(valid_generator, total=valid_lim, unit=' steps', desc='Epoch {:d} valid'.format(epoch)): + for i in tqdm(valid_generator, total=valid_lim, unit=' steps', desc='Epoch {:d} valid'.format(epoch), disable=True): with tf.device('/cpu:0'): total_recall_counter_valid += 1 @@ -291,6 +291,14 @@ def _run_tensorflow_pipeline(self): os.makedirs(model_file_path) saver.save(sess, "{}/model.ckpt".format(model_file_path), global_step=epoch) + # tf.train.write_graph(sess.graph.as_graph_def(), model_file_path, 'tensorflowModel.pbtxt', + # as_text=True) + # + # onnx_graph = tf2onnx.tfonnx.process_tf_graph(sess.graph, input_names=["Input_train:0"], + # output_names=["output:0"]) + # model_proto = onnx_graph.make_model("test") + # with open("{}/model.onnx".format(model_file_path), "wb") as f: + # f.write(model_proto.SerializeToString()) # onnx_graph = tf2onnx.tfonnx.process_tf_graph(sess.graph, output_names=["output:0"]) # model_proto = onnx_graph.make_model("test") @@ -312,8 +320,22 @@ def _run_tensorflow_pipeline(self): ref_iter_count += 1 if ref_iter_count == self.ref_steps: print('\nEarly stopping') + model_file_path = os.path.join(self.ckpnt_path, + self.timestamp + '_split_' + str(split)) + if not os.path.exists(model_file_path): + os.makedirs(model_file_path) + + saver.save(sess, "{}/model.ckpt".format(model_file_path), global_step=epoch) + + # onnx_graph = tf2onnx.tfonnx.process_tf_graph(sess.graph, + # input_names=["Input_train:0"], + # output_names=["output:0"]) + # model_proto = onnx_graph.make_model("test") + # with open("{}/model.onnx".format(model_file_path), "wb") as f: + # f.write(model_proto.SerializeToString()) # sess.close() - break + return prev_loss + # break gc.collect() coord.request_stop() coord.join(enqueue_threads) @@ -379,8 +401,8 @@ def _run_pytorch_pipeline(self): epoch_duration_train = 0 epoch_duration_valid = 0 - print("image dimentions") - print([self.img_size[0], self.img_size[1], self.img_size[2]]) + # print("image dimentions") + # print([self.img_size[0], self.img_size[1], self.img_size[2]]) for i in tqdm(train_generator, total=train_lim, unit=' steps', desc='Epoch {:d} train'.format(epoch)): total_recall_counter_train += 1 @@ -388,14 +410,14 @@ def _run_pytorch_pipeline(self): ff = list(i) outputs = self.network(torch.from_numpy(np.array([np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f - in ff]).transpose((0,3,1,2))).float()) - loss = self.network.return_loss(outputs, torch.from_numpy(np.array([f[1] for f in ff])).long()) + in ff]).transpose((0,3,1,2))).float().to(self.device)) + loss = self.network.return_loss(outputs, torch.from_numpy(np.array([f[1] for f in ff])).long().to(self.device)).to(self.device) self.train_op.zero_grad() loss.backward() self.train_op.step() - train_step_accuracy = int(self.network.return_accuracy(outputs, torch.from_numpy(np.array([f[1] for f in ff])), self.batch_size, - self.task_type)) + train_step_accuracy = int(self.network.return_accuracy(outputs, torch.from_numpy(np.array([f[1] for f in ff])).to(self.device), self.batch_size, + self.task_type).to(self.device)) epoch_loss_train += loss.item() epoch_accur_train += (100*train_step_accuracy)/self.batch_size @@ -406,11 +428,11 @@ def _run_pytorch_pipeline(self): ff = list(i) outputs = self.network(torch.from_numpy(np.array([np.reshape(f[0], [self.img_size[0], self.img_size[1], self.img_size[2]]) for f - in ff]).transpose((0,3,1,2))).float()) - loss = self.network.return_loss(outputs, torch.from_numpy(np.array([f[1] for f in ff])).long()) + in ff]).transpose((0,3,1,2))).float().to(self.device)) + loss = self.network.return_loss(outputs, torch.from_numpy(np.array([f[1] for f in ff])).long().to(self.device)).to(self.device) - valid_step_accuracy = int(self.network.return_accuracy(outputs, torch.from_numpy(np.array([f[1] for f in ff])), 1, - self.task_type)) + valid_step_accuracy = int(self.network.return_accuracy(outputs, torch.from_numpy(np.array([f[1] for f in ff])).to(self.device), 1, + self.task_type).to(self.device)) epoch_loss_valid += loss.item() epoch_accur_valid += (100*valid_step_accuracy)/self.batch_size @@ -458,7 +480,11 @@ def _run_pytorch_pipeline(self): # print("saving figure") # # print(figure) + self.graph_op.plot('loss', 'train', 'Class Loss', epoch, train_aver_loss) + self.graph_op.plot('loss', 'valid', 'Class Loss', epoch, valid_aver_loss) + self.graph_op.plot('acc', 'train', 'Class Accuracy', epoch, epoch_accur_train) + self.graph_op.plot('acc', 'valid', 'Class Accuracy', epoch, epoch_accur_valid) # figure_file = os.path.join(loss_plot_path,'train_acur_plot.jpg') # figure.savefig(figure_file) @@ -470,11 +496,11 @@ def _run_pytorch_pipeline(self): epoch_accur_valid, epoch_duration_valid)) if prev_loss > valid_aver_loss: - # if not os.path.isdir(self.checkpoint_dir): - # os.mkdir(self.checkpoint_dir) + if not os.path.isdir(self.checkpoint_dir): + os.mkdir(self.checkpoint_dir) - # if not os.path.isdir(os.path.join(self.tr_path, self.data_set, self.timestamp)): - # os.mkdir(os.path.join(self.tr_path, self.data_set, self.timestamp)) + if not os.path.isdir(self.tr_path): + os.mkdir(self.tr_path) model_file_path = os.path.join(self.ckpnt_path, self.timestamp + '_split_' + str(split)) @@ -493,29 +519,24 @@ def _run_pytorch_pipeline(self): ref_iter_count += 1 if ref_iter_count == self.ref_steps+1: print('\nEarly stopping\n Saving last best model:') - final_network = torch.load(model_file_path) + final_network = torch.load(os.path.join(model_file_path, 'model.pth')) # pprint.pprint(final_network.state_dict()) print("Importing model to ONNX") - # model_file_path_onnx = create_ckpt_data_onnx(self.ckpnt_path, self.network_type, event_time) - - - - dummy_input = Variable(torch.randn(1, *[self.img_size[2], self.img_size[0], self.img_size[1]])) - torch_onnx.export(final_network, dummy_input, model_file_path, verbose=False) + dummy_input = Variable(torch.randn(1, *[self.img_size[2], self.img_size[0], self.img_size[1]]).to(self.device)) + torch_onnx.export(final_network, dummy_input, os.path.join(model_file_path, 'model.onnx'), verbose=False) print('Done') - status = "FINISHED: Early stopping" - return status + + return prev_loss gc.collect() print("Finalizing training") - final_network = torch.load(model_file_path) + final_network = torch.load(os.path.join(model_file_path, 'model.pth')) # pprint.pprint(final_network.state_dict()) print("Importing model to ONNX") # model_file_path_onnx = create_ckpt_data_onnx(self.ckpnt_path, self.network_type, event_time) - dummy_input = Variable(torch.randn(1, *[self.img_size[2], self.img_size[0], self.img_size[1]])) - torch_onnx.export(final_network, dummy_input, model_file_path, verbose=False) + dummy_input = Variable(torch.randn(1, *[self.img_size[2], self.img_size[0], self.img_size[1]]).to(self.device)) + torch_onnx.export(final_network, dummy_input, os.path.join(model_file_path, 'model.onnx'), verbose=False) print('Done') - status = 'FINISHED' h5_file.close() - return status \ No newline at end of file + return prev_loss \ No newline at end of file diff --git a/network/wrappers/ConvNet.py b/network/wrappers/ConvNet.py index 66a8bcb..12f559a 100644 --- a/network/wrappers/ConvNet.py +++ b/network/wrappers/ConvNet.py @@ -101,21 +101,22 @@ def __init__(self, network_type, loss, accuracy, lr, framework, training, train nn.Module.__init__(self) self.conv_1_1 = self._conv_bn_layer_pt(1, self.num_filters, filter_size=5, stride=1, is_training=True, - nonlin_f=self.nonlin_f, padding='same', name_postfix='1_1') + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_1 = nn.MaxPool2d(kernel_size=2, stride=2) self.drop_1 = nn.Dropout2d(p=self.dropout) - self.conv_2_1 = self._conv_bn_layer_pt(self.num_filters, self.num_filters*2, filter_size=5, stride=1, is_training=True, - nonlin_f=self.nonlin_f, padding='same', name_postfix='1_1') + self.conv_2_1 = self._conv_bn_layer_pt(self.num_filters, self.num_filters * 2, filter_size=5, stride=1, + is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') self.pool_2 = nn.MaxPool2d(kernel_size=2, stride=2) self.drop_2 = nn.Dropout2d(p=self.dropout) - # self.fc_1 = nn.Linear(1024, 512) # self.drop_4 = nn.Dropout2d(p=self.dropout) - self.fc_2 = nn.Linear(512, 128) - self.drop_5 = nn.Dropout2d(p=self.dropout) + self.fc_2 = nn.Linear(800, 128) + self.drop_5 = nn.Dropout(p=self.dropout) self.out = nn.Linear(128, self.num_classes) @@ -139,4 +140,4 @@ def forward(self, X): x = self.drop_5(x) x = self.out(x) - return x + return x \ No newline at end of file diff --git a/network/wrappers/NetworkBase.py b/network/wrappers/NetworkBase.py index 606f943..491997a 100644 --- a/network/wrappers/NetworkBase.py +++ b/network/wrappers/NetworkBase.py @@ -224,6 +224,8 @@ def _pick_loss_func(key, framework): return mse_pt if key == 'mse_loss': return mse_loss_pt + elif key == 'dice_jaccard': + return dice_loss else: raise ValueError('Unexpected metric function %s' % key) @@ -247,10 +249,6 @@ def _pick_accuracy_function(key, framework): elif framework == "pytorch": if key == 'IoU': return IoU_pt - elif key == 'dice_sorensen': - return dice_sorensen_pt - elif key == 'dice_jaccard': - return dice_jaccard_pt elif key == 'mse': return mse_pt elif key == 'hinge': @@ -304,21 +302,21 @@ def _conv_bn_layer_tf(input_layer, n_filters, filter_scale=1, filter_size=3, is_ """ with tf.name_scope(name + name_postfix): conv = tf.layers.conv2d(input_layer, filters=filter_scale * n_filters, kernel_size=filter_size, - activation=None, padding=padding, kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=0.1), + activation=None, padding=padding, name='conv_' + name_postfix) batch_norm = tf.layers.batch_normalization(conv, training=is_training, fused=False, name='batch_' + name_postfix) # weights, biases = NetworkBase.weights_and_biases(tf.shape(batch_norm), tf.shape(batch_norm)[-1]) # nonlin = nonlin_f(tf.matmul(batch_norm, weights) + biases, name='activation_' + name_postfix) - nonlin = nonlin_f(batch_norm, name='activation_' + name_postfix) + nonlin = nonlin_f(conv, name='activation_' + name_postfix) return conv, batch_norm, nonlin @staticmethod def _conv_bn_layer_pt(n_in, n_out, filter_size=3, stride=1, is_training=True, nonlin_f=None, - padding='same', name_postfix='1_1'): + padding=1, name_postfix='1_1'): m = nn.Sequential( - nn.Conv2d(n_in, n_out, filter_size, stride), - nn.BatchNorm2d(n_out), + nn.Conv2d(n_in, n_out, filter_size, stride, padding), + # nn.BatchNorm2d(n_out), nonlin_f() ) return m diff --git a/network/wrappers/UNet.py b/network/wrappers/UNet.py index b99d50a..d1d857a 100644 --- a/network/wrappers/UNet.py +++ b/network/wrappers/UNet.py @@ -14,7 +14,11 @@ # --- imports ----------------------------------------------------------------- +import torch +import torch.nn as nn import tensorflow as tf +import torch.nn.functional as F + from network.wrappers.NetworkBase import NetworkBase @@ -127,3 +131,64 @@ def build_net_tf(self, X): self.nets.append(output) return output + + + +class UNet_pt(NetworkBase, nn.Module): + def __init__(self, network_type, loss, accuracy, lr, framework, training, trainable_layers=None, num_filters=16, + optimizer='adam', nonlin='elu', num_classes=2, dropout=0.25): + NetworkBase.__init__(self, network_type=network_type, loss=loss, accuracy=accuracy, framework=framework, lr=lr, training=training, + trainable_layers=trainable_layers, num_filters=num_filters, optimizer=optimizer, nonlin=nonlin, + num_classes=num_classes, dropout=dropout) + nn.Module.__init__(self) + + self.dconv_down1 = double_conv(1, 16) + self.dconv_down2 = double_conv(16, 32) + self.dconv_down3 = double_conv(32, 64) + self.dconv_down4 = double_conv(64, 128) + + self.maxpool = nn.MaxPool2d(2) + self.upsample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True) + + self.dconv_up3 = double_conv(64 + 128, 64) + self.dconv_up2 = double_conv(32 + 64, 32) + self.dconv_up1 = double_conv(32 + 16, 16) + + self.conv_last = nn.Conv2d(16, num_classes, 1) + + def forward(self, x): + conv1 = self.dconv_down1(x) + x = self.maxpool(conv1) + + conv2 = self.dconv_down2(x) + x = self.maxpool(conv2) + + conv3 = self.dconv_down3(x) + x = self.maxpool(conv3) + + x = self.dconv_down4(x) + + x = self.upsample(x) + x = torch.cat([x, conv3], dim=1) + + x = self.dconv_up3(x) + x = self.upsample(x) + x = torch.cat([x, conv2], dim=1) + + x = self.dconv_up2(x) + x = self.upsample(x) + x = torch.cat([x, conv1], dim=1) + + x = self.dconv_up1(x) + + out = self.conv_last(x) + + return out + +def double_conv(in_channels, out_channels): + return nn.Sequential( + nn.Conv2d(in_channels, out_channels, 3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels, 3, padding=1), + nn.ReLU(inplace=True) + ) \ No newline at end of file diff --git a/network/wrappers/VGG19.py b/network/wrappers/VGG19.py new file mode 100644 index 0000000..8882a5a --- /dev/null +++ b/network/wrappers/VGG19.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 3/21/2019 8:32 AM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + +import torch.nn as nn +import tensorflow as tf +import torch.nn.functional as F + +from network.wrappers.NetworkBase import NetworkBase + + +class VGG19_tf(NetworkBase): + def __init__(self, network_type, loss, accuracy, lr, framework, training, trainable_layers=None, num_filters=16, + optimizer='adam', nonlin='elu', num_classes=2, dropout=0.25): + """ + Convolutional Neural Network constructor + :param loss: used loss function + :param lr: learning rate + :param training: is training True/False + :param num_filters: number of filters + :param optimizer: used optimizer + :param nonlin: used nonliniearity + :param num_classes: number of classes/labels + :param dropout: dropout ratio + """ + super().__init__(network_type=network_type, loss=loss, accuracy=accuracy, framework=framework, lr=lr, + training=training, + trainable_layers=trainable_layers, num_filters=num_filters, optimizer=optimizer, nonlin=nonlin, + num_classes=num_classes, dropout=dropout) + self.weights, self.biases, self.nets = [], [], [] + + def build_net(self, X): + """ + Build the Convolutional Neural Network + :param X: input tensor + :return: network + """ + + with tf.name_scope('s_conv_1'): + conv_1_1, batch_1_1, activ_1_1 = self._conv_bn_layer_tf(X, n_filters=self.num_filters, filter_size=3, + is_training=self.is_training, + nonlin_f=self.nonlin_f, + name_postfix='1_1') + conv_1_2, batch_1_2, activ_1_2 = self._conv_bn_layer_tf(activ_1_1, n_filters=self.num_filters, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='1_2') + pooling_1 = tf.layers.max_pooling2d(activ_1_2, pool_size=2, strides=2, padding='same', name='pooling_1') + self.nets.extend([conv_1_1, conv_1_2]) + + with tf.name_scope('s_conv_2'): + conv_2_1, batch_2_1, activ_2_1 = self._conv_bn_layer_tf(pooling_1, n_filters=self.num_filters, filter_scale=2, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='2_1') + conv_2_2, batch_2_2, activ_2_2 = self._conv_bn_layer_tf(activ_2_1, n_filters=self.num_filters, + filter_scale=2, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='2_2') + pooling_2 = tf.layers.max_pooling2d(activ_2_2, pool_size=2, strides=2, padding='same', name='pooling_2') + self.nets.extend([conv_2_1, conv_2_2]) + + with tf.name_scope('s_conv_3'): + conv_3_1, batch_3_1, activ_3_1 = self._conv_bn_layer_tf(pooling_2, n_filters=self.num_filters, filter_scale=4, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='3_1') + conv_3_2, batch_3_2, activ_3_2 = self._conv_bn_layer_tf(activ_3_1, n_filters=self.num_filters, + filter_scale=4, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='3_2') + conv_3_3, batch_3_3, activ_3_3 = self._conv_bn_layer_tf(conv_3_2, n_filters=self.num_filters, filter_scale=4, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='3_3') + conv_3_4, batch_3_2, activ_3_4 = self._conv_bn_layer_tf(conv_3_3, n_filters=self.num_filters, + filter_scale=4, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='3_4') + pooling_3 = tf.layers.max_pooling2d(conv_3_4, pool_size=3, strides=2, padding='same', name='pooling_3') + + self.nets.extend([conv_3_1, conv_3_2]) + + with tf.name_scope('s_conv_4'): + conv_4_1, batch_4_1, activ_4_1 = self._conv_bn_layer_tf(pooling_3, n_filters=self.num_filters, filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='4_1') + conv_4_2, batch_4_2, activ_4_2 = self._conv_bn_layer_tf(activ_4_1, n_filters=self.num_filters, + filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='4_2') + conv_4_3, batch_4_3, activ_4_3 = self._conv_bn_layer_tf(conv_4_2, n_filters=self.num_filters, filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='4_3') + conv_4_4, batch_4_2, activ_4_4 = self._conv_bn_layer_tf(conv_4_3, n_filters=self.num_filters, + filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='4_4') + pooling_4 = tf.layers.max_pooling2d(conv_4_4, pool_size=3, strides=2, padding='same', name='pooling_4') + + with tf.name_scope('s_conv_5'): + conv_5_1, batch_5_1, activ_5_1 = self._conv_bn_layer_tf(pooling_4, n_filters=self.num_filters, filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='5_1') + conv_5_2, batch_5_2, activ_5_2 = self._conv_bn_layer_tf(activ_5_1, n_filters=self.num_filters, + filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='5_2') + conv_5_3, batch_5_3, activ_5_3 = self._conv_bn_layer_tf(conv_5_2, n_filters=self.num_filters, filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='5_3') + conv_5_4, batch_5_2, activ_5_4 = self._conv_bn_layer_tf(conv_5_3, n_filters=self.num_filters, + filter_scale=8, + filter_size=3, is_training=self.is_training, + nonlin_f=self.nonlin_f, name_postfix='5_4') + pooling_5 = tf.layers.max_pooling2d(conv_5_4, pool_size=3, strides=2, padding='same', name='pooling_5') + + with tf.name_scope('s_outputs'): + flat = tf.layers.flatten(pooling_5, name='flatten') + drop_5 = tf.layers.dropout(flat, rate=self.dropout, name='drop_5') + fc_1 = tf.layers.dense(drop_5, units=512, activation=self.nonlin_f, name='fc_1') + drop_6 = tf.layers.dropout(fc_1, rate=self.dropout, name='drop_6') + fc_2 = tf.layers.dense(drop_6, units=512, activation=self.nonlin_f, name='fc_2') + # drop_6 = tf.layers.dropout(fc_2, rate=self.dropout, name='drop_6') + output_p = tf.layers.dense(fc_2, units=self.num_classes, activation=None, name='output') + return output_p + + +class VGG19_pt(NetworkBase, nn.Module): + def __init__(self, network_type, loss, accuracy, lr, framework, training, trainable_layers=None, num_filters=16, + optimizer='adam', nonlin='elu', num_classes=2, dropout=0.25): + NetworkBase.__init__(self, network_type=network_type, loss=loss, accuracy=accuracy, framework=framework, lr=lr, training=training, + trainable_layers=trainable_layers, num_filters=num_filters, optimizer=optimizer, nonlin=nonlin, + num_classes=num_classes, dropout=dropout) + nn.Module.__init__(self) + + self.conv_1_1 = self._conv_bn_layer_pt(3, self.num_filters, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_1_2 = self._conv_bn_layer_pt(num_filters, self.num_filters, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_1 = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_2_1 = self._conv_bn_layer_pt(num_filters, self.num_filters * 2, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_2_2 = self._conv_bn_layer_pt(num_filters * 2, self.num_filters * 2, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_2 = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_3_1 = self._conv_bn_layer_pt(num_filters * 2, self.num_filters * 4, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_3_2 = self._conv_bn_layer_pt(num_filters * 4, self.num_filters * 4, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_3_3 = self._conv_bn_layer_pt(num_filters * 4, self.num_filters * 4, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_3_4 = self._conv_bn_layer_pt(num_filters * 4, self.num_filters * 4, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_3 = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_4_1 = self._conv_bn_layer_pt(num_filters * 4, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_4_2 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_4_3 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_4_4 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_4 = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_5_1 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_5_2 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_5_3 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.conv_5_4 = self._conv_bn_layer_pt(num_filters * 8, self.num_filters * 8, filter_size=3, stride=1, is_training=True, + nonlin_f=self.nonlin_f, padding=1, name_postfix='1_1') + self.pool_5 = nn.MaxPool2d(kernel_size=2, stride=2) + self.drop_1 = nn.Dropout2d(p=0.5) + self.fc_1 = nn.Linear(512, 512) + self.n_1 = self.nonlin_f() + self.drop_2 = nn.Dropout2d(p=0.5) + self.fc_2 = nn.Linear(512, 512) + self.n_2 = self.nonlin_f() + self.drop_3 = nn.Dropout2d(p=0.5) + self.out = nn.Linear(512, self.num_classes) + + def forward(self, X): + x = self.conv_1_1(X) + x = self.conv_1_2(x) + x = self.pool_1(x) + + x = self.conv_2_1(x) + x = self.conv_2_2(x) + x = self.pool_2(x) + + x = self.conv_3_1(x) + x = self.conv_3_2(x) + x = self.conv_3_3(x) + x = self.conv_3_4(x) + x = self.pool_3(x) + + x = self.conv_4_1(x) + x = self.conv_4_2(x) + x = self.conv_4_3(x) + x = self.conv_4_4(x) + x = self.pool_4(x) + + x = self.conv_5_1(x) + x = self.conv_5_2(x) + x = self.conv_5_3(x) + x = self.conv_5_4(x) + x = self.pool_5(x) + + x = x.view(x.size(0), -1) + x = self.drop_1(x) + x = self.fc_1(x) + x = self.n_1(x) + x = self.drop_2(x) + x = self.fc_2(x) + x = self.n_2(x) + # x = self.drop_3(x) + x = self.out(x) + + return x \ No newline at end of file diff --git a/utils/BatchIterator.py b/utils/BatchIterator.py index 0d43ee8..adc8033 100644 --- a/utils/BatchIterator.py +++ b/utils/BatchIterator.py @@ -259,12 +259,13 @@ def _recode_masks(self, msks): # only if segmentation colors are encoded as class labels or binary if ((len(msks.shape) == 3) and (np.array_equal(msks[:, :, 0], msks[:, :, 1]))) or (len(msks.shape) != 3): new_img = np.zeros([msks.shape[0], msks.shape[1], self.num_lbls]) - img = msks[:, :, 0] + img = msks # binary segmentation - known bag: binary segmentation only - if self.num_lbls == 2: + if self.num_lbls == 1: img = img//255 - new_img[:, :, 0] = img - new_img[:, :, 1] = 1 - img + new_img = img + # new_img[:, :, 0] = img + # new_img[:, :, 1] = 1 - img else: for j in range(0, self.num_lbls): x_ind, y_ind = np.where(img == j) diff --git a/utils/DataParser.py b/utils/DataParser.py index aa54aac..a6e14a5 100644 --- a/utils/DataParser.py +++ b/utils/DataParser.py @@ -168,7 +168,7 @@ def _process_txt(self): if self.is_training: raise ValueError("not sufficient data for given task") else: - if (os.path.splitext(line)[1].lower() in ['.jpg', '.jpeg', '.png']): + if (os.path.splitext(line)[1].lower() in ['.jpg', '.jpeg', '.png', '.bmp']): X_data_ = [line.rstrip().split()[0] for line in file_lines] y_data_ = [] else: @@ -221,7 +221,7 @@ def _process_json(self): for d in file_lines['meta']: lst = [] for o in d['objects']: - lst.append([o['bb'] + one_to_onehot(o['object_class'], self.num_classes)]) + lst.append(o['bb'] + one_to_onehot(o['object_class'], self.num_classes)) y_list.append(lst) ll = max([len(l) for l in y_list]) @@ -229,7 +229,7 @@ def _process_json(self): for l in y_list: m = l for i in range(ll - len(m)): - m.append([[0, 0, 0, 0] + [0 for k in range(10)]]) + m.append([0, 0, 0, 0] + [0 for k in range(10)]) y_data_.append(m) y_data_ = np.array(y_data_) else: @@ -244,7 +244,7 @@ def _process_json(self): return X_data_, y_data_ def _data_folder_parse(self, h5py_file_name, log_file_name): - path_list = [i for i in os.listdir(self.data_folder) if os.path.isdir(i)] + path_list = [i for i in os.listdir(self.data_folder) if os.path.isdir(os.path.join(self.data_folder, i))] if len(path_list) < 2: if self.is_training: raise ValueError("not sufficient data for given task") @@ -316,9 +316,11 @@ def _load_mnist(self, h5py_file_name, log_file_name): # y_data = one_hot_encode(y_data) elif self.framework == 'pytorch': import torchvision.datasets as datasets - mnist_trainset = datasets.MNIST(root=os.path.join(self.data_path, self.data_set), train=True, download=True, transform=None) import struct if self.is_training: + mnist_trainset = datasets.MNIST(root=os.path.join(self.data_path, self.data_set), train=True, + download=True, transform=None) + with open(os.path.join(self.data_path, self.data_set + r'/raw/train-labels-idx1-ubyte'), 'rb') as lbpath: magic, n = struct.unpack('>II', lbpath.read(8)) y_data = np.fromfile(lbpath, dtype=np.uint8).tolist() @@ -327,12 +329,14 @@ def _load_mnist(self, h5py_file_name, log_file_name): magic, num, rows, cols = struct.unpack(">IIII", imgpath.read(16)) X_data = np.fromfile(imgpath, dtype=np.uint8).reshape((-1, 28, 28, 1)) else: - with open(os.path.join(self.data_path, self.data_set + r'/raw/test-labels-idx1-ubyte'), + mnist_trainset = datasets.MNIST(root=os.path.join(self.data_path, self.data_set), train=False, download=True, + transform=None) + with open(os.path.join(self.data_path, self.data_set + r'/raw/t10k-labels-idx1-ubyte'), 'rb') as lbpath: magic, n = struct.unpack('>II', lbpath.read(8)) self.y_data = np.fromfile(lbpath, dtype=np.uint8).tolist() - with open(os.path.join(self.data_path, self.data_set + r'/raw/test-images-idx3-ubyte'), + with open(os.path.join(self.data_path, self.data_set + r'/raw/t10k-images-idx3-ubyte'), 'rb') as imgpath: magic, num, rows, cols = struct.unpack(">IIII", imgpath.read(16)) self.X_data = np.fromfile(imgpath, dtype=np.uint8).reshape((-1, 28, 28, 1)) @@ -380,6 +384,7 @@ def _load_cifar10(self, h5py_file_name, log_file_name): # works ONLY for Windows. I have NO idea why # X_data = cifar_trainset.train_data # y_data = cifar_trainset.train_labels + X_data = [np.asarray(x) for x, _ in cifar_trainset] y_data = [x for _, x in cifar_trainset] [np.asarray(x) for x, _ in cifar_trainset] diff --git a/utils/VisdomLogger.py b/utils/VisdomLogger.py new file mode 100644 index 0000000..1a81932 --- /dev/null +++ b/utils/VisdomLogger.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 3/19/2019 9:33 AM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + +from visdom import Visdom +import numpy as np + +class VisdomPlotter(object): + def __init__(self, env_name='main'): + self.viz = Visdom() + self.env = env_name + self.plots = {} + + def plot(self, var_name, split_name, title_name, x, y): + if var_name not in self.plots: + self.plots[var_name] = self.viz.line(X=np.array([x,x]), Y=np.array([y,y]), env=self.env, opts=dict( + legend=[split_name], + title=title_name, + xlabel='Epochs', + ylabel=var_name + )) + else: + self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update = 'append') \ No newline at end of file diff --git a/utils/metric.py b/utils/metric.py index adc524a..722b4fd 100644 --- a/utils/metric.py +++ b/utils/metric.py @@ -14,6 +14,8 @@ # --- imports ----------------------------------------------------------------- +import torch +import torch.nn.functional as F import torch.nn as nn import tensorflow as tf import numpy as np @@ -254,41 +256,36 @@ def IoU_pt(y_pred, y_true): :return float: IOU score """ - raise NotImplementedError - # ious = [] - # pred = pred.view(-1) - # target = target.view(-1) - # - # # Ignore IoU for background class ("0") - # for cls in range(1, n_classes): # This goes from 1:n_classes-1 -> class "0" is ignored - # pred_inds = pred == cls - # target_inds = target == cls - # intersection = (pred_inds[target_inds]).long().sum().data.cpu()[0] # Cast to long to prevent overflows - # union = pred_inds.long().sum().data.cpu()[0] + target_inds.long().sum().data.cpu()[0] - intersection - # if union == 0: - # ious.append(float('nan')) # If there is no ground truth, do not include in evaluation - # else: - # ious.append(float(intersection) / float(max(union, 1))) - # return np.array(ious) - - - - -def dice_jaccard_pt(y_pred, y_true): - """Returns a (approx) dice score - intesection = y_pred.flatten() * y_true.flatten() - Then, dice = 2 * intersection / (y_pred.sum() + y_true.sum()) - :param y_pred: predicted labels (4-D array): (N, H, W, 1) - :param y_true: groundtruth labels (4-D array): (N, H, W, 1) - :return - float: dice score - """ - raise NotImplementedError - - -def dice_sorensen_pt(y_pred, y_true): - raise NotImplementedError - + smooth = 1. + y_pred_sig = F.sigmoid(y_pred) + num = y_true.size(0) # Number of batches + x = y_pred_sig.view(num, -1).float() # Flatten + y = y_true.view(num, -1).float() + intersection = torch.sum(x * y) + score = (intersection + smooth) / (torch.sum(x) + torch.sum(y) - intersection + smooth) + out = torch.sum(score) + print("iou {}".format(out)) + return out + + + +def dice_loss(y_pred, y_true, eps=1e-7): + return DiceLoss_pt()(y_pred, y_true) + +class DiceLoss_pt(nn.Module): + def __init__(self, weight=None, size_average=True): + super(DiceLoss_pt, self).__init__() + + def forward(self, y_pred, y_true): + smooth = 1. + y_pred_sig = F.sigmoid(y_pred) + num = y_true.size(0) # Number of batches + x = y_pred_sig.view(num, -1).float() # Flatten + y = y_true.view(num, -1).float() + intersection = torch.sum(x * y) + score = (2. * intersection + smooth) / (torch.sum(x) + torch.sum(y) + smooth) + out = 1 - torch.sum(score) / num + return out def softmax_pt(y_pred, y_true, epsilon=1e-10): """ diff --git a/utils/visualizer.py b/utils/visualizer.py new file mode 100644 index 0000000..d353516 --- /dev/null +++ b/utils/visualizer.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------------- +# Copyright (C) Software Competence Center Hagenberg GmbH (SCCH) +# All rights reserved. +# ----------------------------------------------------------------------------- +# This document contains proprietary information belonging to SCCH. +# Passing on and copying of this document, use and communication of its +# contents is not permitted without prior written authorization. +# ----------------------------------------------------------------------------- +# Created on : 3/19/2019 3:44 PM $ +# by : Shepeleva $ +# SVN $ +# + +# --- imports ----------------------------------------------------------------- + +import numpy as np +import os +import sys +import ntpath +import time +from . import util, html +from subprocess import Popen, PIPE +from scipy.misc import imresize + +if sys.version_info[0] == 2: + VisdomExceptionBase = Exception +else: + VisdomExceptionBase = ConnectionError + + +def save_images(webpage, visuals, image_path, aspect_ratio=1.0, width=256): + """Save images to the disk. + + Parameters: + webpage (the HTML class) -- the HTML webpage class that stores these imaegs (see html.py for more details) + visuals (OrderedDict) -- an ordered dictionary that stores (name, images (either tensor or numpy) ) pairs + image_path (str) -- the string is used to create image paths + aspect_ratio (float) -- the aspect ratio of saved images + width (int) -- the images will be resized to width x width + + This function will save images stored in 'visuals' to the HTML file specified by 'webpage'. + """ + image_dir = webpage.get_image_dir() + short_path = ntpath.basename(image_path[0]) + name = os.path.splitext(short_path)[0] + + webpage.add_header(name) + ims, txts, links = [], [], [] + + for label, im_data in visuals.items(): + im = util.tensor2im(im_data) + image_name = '%s_%s.png' % (name, label) + save_path = os.path.join(image_dir, image_name) + h, w, _ = im.shape + if aspect_ratio > 1.0: + im = imresize(im, (h, int(w * aspect_ratio)), interp='bicubic') + if aspect_ratio < 1.0: + im = imresize(im, (int(h / aspect_ratio), w), interp='bicubic') + util.save_image(im, save_path) + + ims.append(image_name) + txts.append(label) + links.append(image_name) + webpage.add_images(ims, txts, links, width=width) + + +class Visualizer(): + """This class includes several functions that can display/save images and print/save logging information. + + It uses a Python library 'visdom' for display, and a Python library 'dominate' (wrapped in 'HTML') for creating HTML files with images. + """ + + def __init__(self, opt): + """Initialize the Visualizer class + + Parameters: + opt -- stores all the experiment flags; needs to be a subclass of BaseOptions + Step 1: Cache the training/test options + Step 2: connect to a visdom server + Step 3: create an HTML object for saveing HTML filters + Step 4: create a logging file to store training losses + """ + self.opt = opt # cache the option + self.display_id = opt.display_id + self.use_html = opt.isTrain and not opt.no_html + self.win_size = opt.display_winsize + self.name = opt.name + self.port = opt.display_port + self.saved = False + if self.display_id > 0: # connect to a visdom server given and + import visdom + self.ncols = opt.display_ncols + self.vis = visdom.Visdom(server=opt.display_server, port=opt.display_port, env=opt.display_env) + if not self.vis.check_connection(): + self.create_visdom_connections() + + if self.use_html: # create an HTML object at /web/; images will be saved under /web/images/ + self.web_dir = os.path.join(opt.checkpoints_dir, opt.name, 'web') + self.img_dir = os.path.join(self.web_dir, 'images') + print('create web directory %s...' % self.web_dir) + util.mkdirs([self.web_dir, self.img_dir]) + # create a logging file to store training losses + self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt') + with open(self.log_name, "a") as log_file: + now = time.strftime("%c") + log_file.write('================ Training Loss (%s) ================\n' % now) + + def reset(self): + """Reset the self.saved status""" + self.saved = False + + def create_visdom_connections(self): + """If the program could not connect to Visdom server, this function will start a new server at port < self.port > """ + cmd = sys.executable + ' -m visdom.server -p %d &>/dev/null &' % self.port + print('\n\nCould not connect to Visdom server. \n Trying to start a server....') + print('Command: %s' % cmd) + Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + + def display_current_results(self, visuals, epoch, save_result): + """Display current results on visdom; save current results to an HTML file. + + Parameters: + visuals (OrderedDict) - - dictionary of images to display or save + epoch (int) - - the current epoch + save_result (bool) - - if save the current results to an HTML file + """ + if self.display_id > 0: # show images in the browser using visdom + ncols = self.ncols + if ncols > 0: # show all the images in one visdom panel + ncols = min(ncols, len(visuals)) + h, w = next(iter(visuals.values())).shape[:2] + table_css = """""" % (w, h) # create a table css + # create a table of images. + title = self.name + label_html = '' + label_html_row = '' + images = [] + idx = 0 + for label, image in visuals.items(): + image_numpy = util.tensor2im(image) + label_html_row += '%s' % label + images.append(image_numpy.transpose([2, 0, 1])) + idx += 1 + if idx % ncols == 0: + label_html += '%s' % label_html_row + label_html_row = '' + white_image = np.ones_like(image_numpy.transpose([2, 0, 1])) * 255 + while idx % ncols != 0: + images.append(white_image) + label_html_row += '' + idx += 1 + if label_html_row != '': + label_html += '%s' % label_html_row + try: + self.vis.images(images, nrow=ncols, win=self.display_id + 1, + padding=2, opts=dict(title=title + ' images')) + label_html = '%s
' % label_html + self.vis.text(table_css + label_html, win=self.display_id + 2, + opts=dict(title=title + ' labels')) + except VisdomExceptionBase: + self.create_visdom_connections() + + else: # show each image in a separate visdom panel; + idx = 1 + try: + for label, image in visuals.items(): + image_numpy = util.tensor2im(image) + self.vis.image(image_numpy.transpose([2, 0, 1]), opts=dict(title=label), + win=self.display_id + idx) + idx += 1 + except VisdomExceptionBase: + self.create_visdom_connections() + + if self.use_html and (save_result or not self.saved): # save images to an HTML file if they haven't been saved. + self.saved = True + # save images to the disk + for label, image in visuals.items(): + image_numpy = util.tensor2im(image) + img_path = os.path.join(self.img_dir, 'epoch%.3d_%s.png' % (epoch, label)) + util.save_image(image_numpy, img_path) + + # update website + webpage = html.HTML(self.web_dir, 'Experiment name = %s' % self.name, refresh=1) + for n in range(epoch, 0, -1): + webpage.add_header('epoch [%d]' % n) + ims, txts, links = [], [], [] + + for label, image_numpy in visuals.items(): + image_numpy = util.tensor2im(image) + img_path = 'epoch%.3d_%s.png' % (n, label) + ims.append(img_path) + txts.append(label) + links.append(img_path) + webpage.add_images(ims, txts, links, width=self.win_size) + webpage.save() + + def plot_current_losses(self, epoch, counter_ratio, losses): + """display the current losses on visdom display: dictionary of error labels and values + + Parameters: + epoch (int) -- current epoch + counter_ratio (float) -- progress (percentage) in the current epoch, between 0 to 1 + losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + """ + if not hasattr(self, 'plot_data'): + self.plot_data = {'X': [], 'Y': [], 'legend': list(losses.keys())} + self.plot_data['X'].append(epoch + counter_ratio) + self.plot_data['Y'].append([losses[k] for k in self.plot_data['legend']]) + try: + self.vis.line( + X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1), + Y=np.array(self.plot_data['Y']), + opts={ + 'title': self.name + ' loss over time', + 'legend': self.plot_data['legend'], + 'xlabel': 'epoch', + 'ylabel': 'loss'}, + win=self.display_id) + except VisdomExceptionBase: + self.create_visdom_connections() + + # losses: same format as |losses| of plot_current_losses + def print_current_losses(self, epoch, iters, losses, t_comp, t_data): + """print current losses on console; also save the losses to the disk + + Parameters: + epoch (int) -- current epoch + iters (int) -- current training iteration during this epoch (reset to 0 at the end of every epoch) + losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + t_comp (float) -- computational time per data point (normalized by batch_size) + t_data (float) -- data loading time per data point (normalized by batch_size) + """ + message = '(epoch: %d, iters: %d, time: %.3f, data: %.3f) ' % (epoch, iters, t_comp, t_data) + for k, v in losses.items(): + message += '%s: %.3f ' % (k, v) + + print(message) # print the message + with open(self.log_name, "a") as log_file: + log_file.write('%s\n' % message) # save the message