Skip to content

Commit

Permalink
[UPDATED] Documentation and code refactoring.
Browse files Browse the repository at this point in the history
  • Loading branch information
scchwagner committed Sep 26, 2019
1 parent 5e73d69 commit 74faa69
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 81 deletions.
2 changes: 2 additions & 0 deletions configs/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
# Created on : 11/16/2017 11:07 $
# by : shepeleva $
# SVN : $
# Edited on : 9/26/2019 $
# by : scchwagner $
#

# --- imports -----------------------------------------------------------------
Expand Down
40 changes: 21 additions & 19 deletions configs/config_LeNet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ def load_config():
config.training_mode = True
config.data_set = 'MNIST'
config.image_size = [32, 32, 1]
config.lr = 0.02419 # changable
config.lr_decay = 0.05931 # changable
config.ref_steps = 2 # changable
config.ref_patience = 1 # changable
config.batch_size = 124 # changable
config.num_epochs = 20 # changable
config.loss = 'sigmoid' # changable
config.optimizer = 'gradient' #changable
config.lr = 0.001 # changable
config.lr_decay = 0.00001 # changable
config.ref_steps = 5 # changable
config.ref_patience = 3 # changable
config.batch_size = 104 # changable
config.num_epochs = 18 # changable
config.loss = 'softmax' # changable
config.optimizer = 'adam' #changable
config.mi_record = False
config.gpu_load = 0.85
config.gpu_load = 0.7
config.num_classes = 10
config.class_labels = [str(i) for i in range(10)]
config.upconv = 'upconv'
Expand All @@ -58,17 +58,19 @@ def load_config():
# enter range with desired value e.g. [0, 0] for continuous
# or fixed Value for discrete e.g. ['percent']
# Epochs are definded over max_amount_resources
config.hyperband = True
config.halving_proportion = 2 # min 2, default 3
config.max_amount_resources = 2 # no of max epochs (min 2, max 1000)
config.lr_range = [0.00001, 0.1] # [0.1, 0.00001]
config.lr_decay_range = [0.00001, 0.1] # [0.1, 0.00001]
config.ref_steps_range = [0, 1] # 0, number of epochs /3
config.ref_patience_range = [0, 1] # 0, number of epochs /3
config.hyperband = False
config.halving_proportion = 3 # min 2, default 3
config.max_amount_resources = 20 # no of max epochs (min 2, max 1000)
# lr and lr decay can be choices or continoous - if there is a change check and change
# _get_random_numbers(self) and _get_configspace(self) in Train Runner
config.lr_range = [0.1, 0.01, 0.001, 0.0001, 0.00001] # [0.1, 0.00001]
config.lr_decay_range = [0.1, 0.01, 0.001, 0.0001, 0.00001] # [0.1, 0.00001]
config.ref_steps_range = [0, 6] # 0, number of epochs /3
config.ref_patience_range = [0, 6] # 0, number of epochs /3
config.batch_size_range = [config.num_classes, 512] # num_classes, max capacity of memory
config.loss_range = ['softmax', 'sigmoid', 'margin'] # ['softmax', 'sigmoid', 'margin', 'mse', 'mse_loss']
config.accuracy_range = ['mse', 'percent'] # ['mse', 'percent']
config.optimizer_range = ['adam', 'momentum', 'gradient'] # ['adam', 'momentum', 'gradient']
config.loss_range = ['softmax'] # ['softmax', 'sigmoid', 'margin', 'mse', 'mse_loss']
config.accuracy_range = ['percent'] # ['mse', 'percent']
config.optimizer_range = ['adam'] # ['adam', 'momentum', 'gradient']

config.bohb = False
config.bandwidth_factor = 3
Expand Down
20 changes: 10 additions & 10 deletions configs/config_VGG16.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ def load_config():
config.training_mode = True
config.data_set = 'CIFAR100'
config.image_size = [32, 32, 3]
config.lr = 0.01 # changable
config.lr_decay = 0.2 # changable
config.ref_steps = 3 # changable
config.ref_patience = [60, 120, 160] # changable
config.batch_size = 120 # changable
config.num_epochs = 80 # changable
config.loss = 'cross-entropy' # changable
config.optimizer = 'adam'
config.lr = 0.019 # changable
config.lr_decay = 0.067 # changable
config.ref_steps = 10 # changable
config.ref_patience = 20 # changable
config.batch_size = 129 # changable
config.num_epochs = 15 # changable
config.loss = 'softmax' # changable
config.optimizer = 'momentum'
config.mi_record = False
config.gpu_load = 0.8
config.gpu_load = 0.7
config.num_classes = 100
config.class_labels = [str(i) for i in range(100)]
config.upconv = 'upconv'
Expand All @@ -60,7 +60,7 @@ def load_config():
# enter range with desired value e.g. [0, 0] for continuous
# or fixed Value for discrete e.g. ['percent']
# Epochs are definded over max_amount_resources
config.hyperband = True
config.hyperband = False
config.halving_proportion = 3 # min 2
config.max_amount_resources = 200 # no of max epochs (min 2, max 1000)
config.lr_range = [0.00001, 0.1] # [0.1, 0.00001]
Expand Down
32 changes: 18 additions & 14 deletions network/TrainRunner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
# Created on : 2/20/2019 4:01 PM $
# by : shepeleva $
# SVN $
#
# Edited on : 9/26/2019 $
# by : scchwagner $

# --- imports -----------------------------------------------------------------

Expand Down Expand Up @@ -625,18 +626,17 @@ def _run_hyperband(self):
json_data.append(hb_prameters)
json_runs = list()

set_of_configurations = list()

for s in range(smax, -1, -1):
r = int(self.max_amount_resources * (self.halving_proportion ** -s))
n = int(np.floor((smax + 1) / (s + 1)) * self.halving_proportion ** s)
set_of_configurations = list()

for i in range(0, s+1):
results = list()
ni = int(n * (self.halving_proportion ** -i))
ri = int(r*(self.halving_proportion**i))

for x in range(0, n):
for x in range(0, ni):

json_run = list()
json_stats = dict()
Expand Down Expand Up @@ -713,7 +713,7 @@ def _run_hyperband(self):
json_best_result.append(json_best_result_config)
json_data.append(json_best_result)

global_elapsed_time = time.time() - start_time
global_elapsed_time = time.time() - global_start_time
json_global_time = dict()
json_global_time['total_time'] = global_elapsed_time
json_data.append(json_global_time)
Expand Down Expand Up @@ -874,10 +874,12 @@ def _get_random_parameter_configurations(self):
def _get_random_numbers(self):
current_config = dict()

random_lr = np.random.uniform(self.lr_range[0], self.lr_range[1])
current_config['lr'] = round(random_lr, len(str(self.lr_range[1])))
random_lr_decay = np.random.uniform(self.lr_decay_range[0], self.lr_decay_range[1])
current_config['lr_decay'] = round(random_lr_decay, len(str(self.lr_decay_range[1])))
#random_lr = np.random.uniform(self.lr_range[0], self.lr_range[1])
#current_config['lr'] = round(random_lr, len(str(self.lr_range[1])))
#random_lr_decay = np.random.uniform(self.lr_decay_range[0], self.lr_decay_range[1])
#current_config['lr_decay'] = round(random_lr_decay, len(str(self.lr_decay_range[1])))
current_config['lr'] = random.choice(self.lr_range)
current_config['lr_decay'] = random.choice(self.lr_decay_range)
current_config['ref_steps'] = random.randint(self.ref_steps_range[0], self.ref_steps_range[1])
current_config['ref_patience'] = random.randint(self.ref_patience_range[0], self.ref_patience_range[1])
current_config['batch_size'] = random.randint(self.batch_size_range[0], self.batch_size_range[1])
Expand All @@ -890,11 +892,13 @@ def _get_random_numbers(self):
def _get_configspace(self):
cs = CS.ConfigurationSpace()

lr = CSH.UniformFloatHyperparameter('lr', lower=self.lr_range[0], upper=self.lr_range[1],
default_value=self.lr_range[0], log=True)
lr_decay = CSH.UniformFloatHyperparameter('lr_decay', lower=self.lr_decay_range[0],
upper=self.lr_decay_range[1], default_value=self.lr_decay_range[0],
log=True)
lr = CSH.CategoricalHyperparameter('lr', self.lr_range)
lr_decay = CSH.CategoricalHyperparameter('lr_decay', self.lr_decay_range)
# lr = CSH.UniformFloatHyperparameter('lr', lower=self.lr_range[0], upper=self.lr_range[1],
# default_value=self.lr_range[0], log=True)
# lr_decay = CSH.UniformFloatHyperparameter('lr_decay', lower=self.lr_decay_range[0],
# upper=self.lr_decay_range[1], default_value=self.lr_decay_range[0],
# log=True)
# IntegerHyperparameter cannot take 0 as lower bound, transform sequential integer to choices,
ref_steps = CSH.CategoricalHyperparameter('ref_steps',
self._transform_to_configspace_choices(self.ref_steps_range))
Expand Down
41 changes: 4 additions & 37 deletions utils/DataParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
# Created on : 2/20/2019 1:10 PM $
# by : shepeleva $
# SVN $
# Edited on : 9/26/2019 $
# by : scchwagner $
#

# --- imports -----------------------------------------------------------------
Expand Down Expand Up @@ -444,46 +446,11 @@ def _load_cifar100(self, h5py_file_name, log_file_name):

if self.is_training:
cifar100 = tf.keras.datasets.cifar100
(X_data_tmp, y_data), _ = cifar100.load_data('fine')
cur_mean = np.mean(X_data_tmp)
cur_std = np.std(X_data_tmp)

for i in range(len(X_data_tmp)):
X_data_tmp[i] = transform_train(X_data_tmp[i])

X_data_tmp = np.reshape(X_data_tmp, (50000, 3, 32, 32))
X_data = np.empty(X_data_tmp.shape)

for i in range(len(X_data_tmp)):
axis1 = mean[0] + (X_data_tmp[i][0] - cur_mean) * (std[0]/cur_std)
axis2 = mean[1] + (X_data_tmp[i][1] - cur_mean) * (std[1] / cur_std)
axis3 = mean[2] + (X_data_tmp[i][2] - cur_mean) * (std[2] / cur_std)
X_data[i][0] = axis1
X_data[i][1] = axis2
X_data[i][2] = axis3

X_data = np.reshape(X_data, (50000, 32, 32, 3))

(X_data, y_data), _ = cifar100.load_data('fine')

else:
cifar100 = tf.keras.datasets.cifar100
_, (X_data_tmp, self.y_data) = cifar100.load_data('fine')
cur_mean = np.mean(X_data_tmp)
cur_std = np.std(X_data_tmp)

X_data_tmp = np.reshape(X_data_tmp, (10000, 3, 32, 32))
X_data = np.empty(X_data_tmp.shape)

for i in range(len(X_data_tmp)):
axis1 = mean[0] + (X_data_tmp[i][0] - cur_mean) * (std[0]/cur_std)
axis2 = mean[1] + (X_data_tmp[i][1] - cur_mean) * (std[1] / cur_std)
axis3 = mean[2] + (X_data_tmp[i][2] - cur_mean) * (std[2] / cur_std)
X_data[i][0] = axis1
X_data[i][1] = axis2
X_data[i][2] = axis3

self.X_data = np.reshape(X_data, (10000, 32, 32, 3))

_, (self.X_data, self.y_data) = cifar100.load_data('fine')

#TODO check saving in pytorch
elif self.framework is "pytorch":
Expand Down
3 changes: 2 additions & 1 deletion utils/metric.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
# Created on : 11/07/2017 08:10 $
# by : shepeleva $
# SVN : $
#
# Edited on : 9/26/2019 $
# by : scchwagner $

# --- imports -----------------------------------------------------------------

Expand Down

0 comments on commit 74faa69

Please sign in to comment.