From 74faa697cef0cc7749a98905658375ef9fe8ca57 Mon Sep 17 00:00:00 2001 From: scchwagner Date: Thu, 26 Sep 2019 14:20:16 +0200 Subject: [PATCH] [UPDATED] Documentation and code refactoring. --- configs/config.py | 2 ++ configs/config_LeNet.py | 40 +++++++++++++++++++++------------------- configs/config_VGG16.py | 20 ++++++++++---------- network/TrainRunner.py | 32 ++++++++++++++++++-------------- utils/DataParser.py | 41 ++++------------------------------------- utils/metric.py | 3 ++- 6 files changed, 57 insertions(+), 81 deletions(-) diff --git a/configs/config.py b/configs/config.py index 535fafe..472a2a6 100644 --- a/configs/config.py +++ b/configs/config.py @@ -10,6 +10,8 @@ # Created on : 11/16/2017 11:07 $ # by : shepeleva $ # SVN : $ +# Edited on : 9/26/2019 $ +# by : scchwagner $ # # --- imports ----------------------------------------------------------------- diff --git a/configs/config_LeNet.py b/configs/config_LeNet.py index 1e0ea0c..2c3c8d5 100644 --- a/configs/config_LeNet.py +++ b/configs/config_LeNet.py @@ -24,16 +24,16 @@ def load_config(): config.training_mode = True config.data_set = 'MNIST' config.image_size = [32, 32, 1] - config.lr = 0.02419 # changable - config.lr_decay = 0.05931 # changable - config.ref_steps = 2 # changable - config.ref_patience = 1 # changable - config.batch_size = 124 # changable - config.num_epochs = 20 # changable - config.loss = 'sigmoid' # changable - config.optimizer = 'gradient' #changable + config.lr = 0.001 # changable + config.lr_decay = 0.00001 # changable + config.ref_steps = 5 # changable + config.ref_patience = 3 # changable + config.batch_size = 104 # changable + config.num_epochs = 18 # changable + config.loss = 'softmax' # changable + config.optimizer = 'adam' #changable config.mi_record = False - config.gpu_load = 0.85 + config.gpu_load = 0.7 config.num_classes = 10 config.class_labels = [str(i) for i in range(10)] config.upconv = 'upconv' @@ -58,17 +58,19 @@ def load_config(): # enter range with desired value e.g. [0, 0] for continuous # or fixed Value for discrete e.g. ['percent'] # Epochs are definded over max_amount_resources - config.hyperband = True - config.halving_proportion = 2 # min 2, default 3 - config.max_amount_resources = 2 # no of max epochs (min 2, max 1000) - config.lr_range = [0.00001, 0.1] # [0.1, 0.00001] - config.lr_decay_range = [0.00001, 0.1] # [0.1, 0.00001] - config.ref_steps_range = [0, 1] # 0, number of epochs /3 - config.ref_patience_range = [0, 1] # 0, number of epochs /3 + config.hyperband = False + config.halving_proportion = 3 # min 2, default 3 + config.max_amount_resources = 20 # no of max epochs (min 2, max 1000) + # lr and lr decay can be choices or continoous - if there is a change check and change + # _get_random_numbers(self) and _get_configspace(self) in Train Runner + config.lr_range = [0.1, 0.01, 0.001, 0.0001, 0.00001] # [0.1, 0.00001] + config.lr_decay_range = [0.1, 0.01, 0.001, 0.0001, 0.00001] # [0.1, 0.00001] + config.ref_steps_range = [0, 6] # 0, number of epochs /3 + config.ref_patience_range = [0, 6] # 0, number of epochs /3 config.batch_size_range = [config.num_classes, 512] # num_classes, max capacity of memory - config.loss_range = ['softmax', 'sigmoid', 'margin'] # ['softmax', 'sigmoid', 'margin', 'mse', 'mse_loss'] - config.accuracy_range = ['mse', 'percent'] # ['mse', 'percent'] - config.optimizer_range = ['adam', 'momentum', 'gradient'] # ['adam', 'momentum', 'gradient'] + config.loss_range = ['softmax'] # ['softmax', 'sigmoid', 'margin', 'mse', 'mse_loss'] + config.accuracy_range = ['percent'] # ['mse', 'percent'] + config.optimizer_range = ['adam'] # ['adam', 'momentum', 'gradient'] config.bohb = False config.bandwidth_factor = 3 diff --git a/configs/config_VGG16.py b/configs/config_VGG16.py index 5597ffe..725908b 100644 --- a/configs/config_VGG16.py +++ b/configs/config_VGG16.py @@ -24,16 +24,16 @@ def load_config(): config.training_mode = True config.data_set = 'CIFAR100' config.image_size = [32, 32, 3] - config.lr = 0.01 # changable - config.lr_decay = 0.2 # changable - config.ref_steps = 3 # changable - config.ref_patience = [60, 120, 160] # changable - config.batch_size = 120 # changable - config.num_epochs = 80 # changable - config.loss = 'cross-entropy' # changable - config.optimizer = 'adam' + config.lr = 0.019 # changable + config.lr_decay = 0.067 # changable + config.ref_steps = 10 # changable + config.ref_patience = 20 # changable + config.batch_size = 129 # changable + config.num_epochs = 15 # changable + config.loss = 'softmax' # changable + config.optimizer = 'momentum' config.mi_record = False - config.gpu_load = 0.8 + config.gpu_load = 0.7 config.num_classes = 100 config.class_labels = [str(i) for i in range(100)] config.upconv = 'upconv' @@ -60,7 +60,7 @@ def load_config(): # enter range with desired value e.g. [0, 0] for continuous # or fixed Value for discrete e.g. ['percent'] # Epochs are definded over max_amount_resources - config.hyperband = True + config.hyperband = False config.halving_proportion = 3 # min 2 config.max_amount_resources = 200 # no of max epochs (min 2, max 1000) config.lr_range = [0.00001, 0.1] # [0.1, 0.00001] diff --git a/network/TrainRunner.py b/network/TrainRunner.py index becae21..765039d 100644 --- a/network/TrainRunner.py +++ b/network/TrainRunner.py @@ -10,7 +10,8 @@ # Created on : 2/20/2019 4:01 PM $ # by : shepeleva $ # SVN $ -# +# Edited on : 9/26/2019 $ +# by : scchwagner $ # --- imports ----------------------------------------------------------------- @@ -625,18 +626,17 @@ def _run_hyperband(self): json_data.append(hb_prameters) json_runs = list() - set_of_configurations = list() - for s in range(smax, -1, -1): r = int(self.max_amount_resources * (self.halving_proportion ** -s)) n = int(np.floor((smax + 1) / (s + 1)) * self.halving_proportion ** s) + set_of_configurations = list() for i in range(0, s+1): results = list() ni = int(n * (self.halving_proportion ** -i)) ri = int(r*(self.halving_proportion**i)) - for x in range(0, n): + for x in range(0, ni): json_run = list() json_stats = dict() @@ -713,7 +713,7 @@ def _run_hyperband(self): json_best_result.append(json_best_result_config) json_data.append(json_best_result) - global_elapsed_time = time.time() - start_time + global_elapsed_time = time.time() - global_start_time json_global_time = dict() json_global_time['total_time'] = global_elapsed_time json_data.append(json_global_time) @@ -874,10 +874,12 @@ def _get_random_parameter_configurations(self): def _get_random_numbers(self): current_config = dict() - random_lr = np.random.uniform(self.lr_range[0], self.lr_range[1]) - current_config['lr'] = round(random_lr, len(str(self.lr_range[1]))) - random_lr_decay = np.random.uniform(self.lr_decay_range[0], self.lr_decay_range[1]) - current_config['lr_decay'] = round(random_lr_decay, len(str(self.lr_decay_range[1]))) + #random_lr = np.random.uniform(self.lr_range[0], self.lr_range[1]) + #current_config['lr'] = round(random_lr, len(str(self.lr_range[1]))) + #random_lr_decay = np.random.uniform(self.lr_decay_range[0], self.lr_decay_range[1]) + #current_config['lr_decay'] = round(random_lr_decay, len(str(self.lr_decay_range[1]))) + current_config['lr'] = random.choice(self.lr_range) + current_config['lr_decay'] = random.choice(self.lr_decay_range) current_config['ref_steps'] = random.randint(self.ref_steps_range[0], self.ref_steps_range[1]) current_config['ref_patience'] = random.randint(self.ref_patience_range[0], self.ref_patience_range[1]) current_config['batch_size'] = random.randint(self.batch_size_range[0], self.batch_size_range[1]) @@ -890,11 +892,13 @@ def _get_random_numbers(self): def _get_configspace(self): cs = CS.ConfigurationSpace() - lr = CSH.UniformFloatHyperparameter('lr', lower=self.lr_range[0], upper=self.lr_range[1], - default_value=self.lr_range[0], log=True) - lr_decay = CSH.UniformFloatHyperparameter('lr_decay', lower=self.lr_decay_range[0], - upper=self.lr_decay_range[1], default_value=self.lr_decay_range[0], - log=True) + lr = CSH.CategoricalHyperparameter('lr', self.lr_range) + lr_decay = CSH.CategoricalHyperparameter('lr_decay', self.lr_decay_range) + # lr = CSH.UniformFloatHyperparameter('lr', lower=self.lr_range[0], upper=self.lr_range[1], + # default_value=self.lr_range[0], log=True) + # lr_decay = CSH.UniformFloatHyperparameter('lr_decay', lower=self.lr_decay_range[0], + # upper=self.lr_decay_range[1], default_value=self.lr_decay_range[0], + # log=True) # IntegerHyperparameter cannot take 0 as lower bound, transform sequential integer to choices, ref_steps = CSH.CategoricalHyperparameter('ref_steps', self._transform_to_configspace_choices(self.ref_steps_range)) diff --git a/utils/DataParser.py b/utils/DataParser.py index 9ba7d7f..cf3e70d 100644 --- a/utils/DataParser.py +++ b/utils/DataParser.py @@ -10,6 +10,8 @@ # Created on : 2/20/2019 1:10 PM $ # by : shepeleva $ # SVN $ +# Edited on : 9/26/2019 $ +# by : scchwagner $ # # --- imports ----------------------------------------------------------------- @@ -444,46 +446,11 @@ def _load_cifar100(self, h5py_file_name, log_file_name): if self.is_training: cifar100 = tf.keras.datasets.cifar100 - (X_data_tmp, y_data), _ = cifar100.load_data('fine') - cur_mean = np.mean(X_data_tmp) - cur_std = np.std(X_data_tmp) - - for i in range(len(X_data_tmp)): - X_data_tmp[i] = transform_train(X_data_tmp[i]) - - X_data_tmp = np.reshape(X_data_tmp, (50000, 3, 32, 32)) - X_data = np.empty(X_data_tmp.shape) - - for i in range(len(X_data_tmp)): - axis1 = mean[0] + (X_data_tmp[i][0] - cur_mean) * (std[0]/cur_std) - axis2 = mean[1] + (X_data_tmp[i][1] - cur_mean) * (std[1] / cur_std) - axis3 = mean[2] + (X_data_tmp[i][2] - cur_mean) * (std[2] / cur_std) - X_data[i][0] = axis1 - X_data[i][1] = axis2 - X_data[i][2] = axis3 - - X_data = np.reshape(X_data, (50000, 32, 32, 3)) - + (X_data, y_data), _ = cifar100.load_data('fine') else: cifar100 = tf.keras.datasets.cifar100 - _, (X_data_tmp, self.y_data) = cifar100.load_data('fine') - cur_mean = np.mean(X_data_tmp) - cur_std = np.std(X_data_tmp) - - X_data_tmp = np.reshape(X_data_tmp, (10000, 3, 32, 32)) - X_data = np.empty(X_data_tmp.shape) - - for i in range(len(X_data_tmp)): - axis1 = mean[0] + (X_data_tmp[i][0] - cur_mean) * (std[0]/cur_std) - axis2 = mean[1] + (X_data_tmp[i][1] - cur_mean) * (std[1] / cur_std) - axis3 = mean[2] + (X_data_tmp[i][2] - cur_mean) * (std[2] / cur_std) - X_data[i][0] = axis1 - X_data[i][1] = axis2 - X_data[i][2] = axis3 - - self.X_data = np.reshape(X_data, (10000, 32, 32, 3)) - + _, (self.X_data, self.y_data) = cifar100.load_data('fine') #TODO check saving in pytorch elif self.framework is "pytorch": diff --git a/utils/metric.py b/utils/metric.py index 796a6df..2b1ecb7 100644 --- a/utils/metric.py +++ b/utils/metric.py @@ -10,7 +10,8 @@ # Created on : 11/07/2017 08:10 $ # by : shepeleva $ # SVN : $ -# +# Edited on : 9/26/2019 $ +# by : scchwagner $ # --- imports -----------------------------------------------------------------