#12 - fix run on import of ml objective functions (non-optimal but be…

…tter)

src/objective_functions.py

@@ -13,7 +13,7 @@ def holder_table(x):
     inside_exp = np.abs(1-np.sqrt(x[0]*x[0]+x[1]*x[1])/np.pi)
     return np.abs(np.sin(x[0])*np.cos(x[1])*np.exp(inside_exp))
 
-holder_bounds = [(-10,10),(-10,10)]
+holder_bounds = [(-10,10)]*2
 
 def rosenbrock(x):
     if x.shape[0] != 3:
@@ -22,14 +22,14 @@ def rosenbrock(x):
     second = 100*(x[2] - x[1]**2)**2 + (x[1] - 1)**2
     return -(first + second)
 
-rosenbrock_bounds = [(-2.048,2.048), (-2.048,2.048), (-2.048,2.048)]
+rosenbrock_bounds = [(-2.048,2.048)]*3
 
 def sphere(x):
     if x.shape[0] != 4:
         raise ValueError('Input array first dimension should be size 4')
     return -np.sqrt(np.sum((x - np.pi/16)**2, axis=0))
 
-sphere_bounds = [(0,1), (0,1), (0,1), (0,1)]
+sphere_bounds = [(0,1)]*4
 
 def linear_slope(x):
     if x.shape[0] != 4:
@@ -38,14 +38,14 @@ def linear_slope(x):
     coef = 10.0**(np.arange(4)/4)
     return coef@(x-5)
 
-linear_slope_bounds = [(-5,5), (-5,5), (-5,5), (-5,5)]
+linear_slope_bounds = [(-5,5)]*4
 
 def deb_one(x):
     if x.shape[0] != 5:
         raise ValueError('Input array first dimension should be of size 5')
     return (1/5)*np.sum(np.sin(5*np.pi*x)**6, axis=0)
 
-deb_one_bounds = [(-5,5), (-5,5), (-5,5), (-5,5), (-5,5)]
+deb_one_bounds = [(-5,5)]*5
 
 synthetic_functions = {
     'Holder Table' : {'func': holder_table, 'bnds': holder_bounds},
@@ -72,24 +72,19 @@ def kernel_ridge_CV(X, y, cv, params):
     results = cross_validate(model, X, y, cv=cv)
     return np.mean(results['test_score'])
 
-X_housing, y_housing = get_data('./data/clean/housing.csv')
-
-def housing(x):
+def housing(x, path='./data/clean/housing.csv'):
+    X_housing, y_housing = get_data(path)
     return kernel_ridge_CV(X_housing, y_housing, 10, x)
 
-X_yacht, y_yacht = get_data('./data/clean/yacht_hydrodynamics.csv')
-
-def yacht(x):
+def yacht(x, path='./data/clean/yacht_hydrodynamics.csv'):
+    X_yacht, y_yacht = get_data(path)
     return kernel_ridge_CV(X_yacht, y_yacht, 10, x)
 
 real_bnds = [(-2, 4), (-5, 5)]
 
-objectives = {
-    'Holder Table': {'func': holder_table, 'bnds': holder_bounds},
-    'Rosenbrock': {'func': rosenbrock, 'bnds': rosenbrock_bounds},
-    'Linear Slope': {'func': linear_slope, 'bnds': linear_slope_bounds},
-    'Sphere': {'func': sphere, 'bnds': sphere_bounds},
-    'Deb N.1': {'func': deb_one, 'bnds': deb_one_bounds},
+ml_functions = {
     'Housing': {'func': housing, 'bnds': real_bnds},
-    'Yacht': {'func': housing, 'bnds': real_bnds}
+    'Yacht': {'func': yacht, 'bnds': real_bnds}
 }
+
+objectives = {**synthetic_functions, **ml_functions}

src/sequential.py

@@ -91,7 +91,6 @@ def pure_random_search(func, bounds, n, seed=None):
 def adaptive_lipo(func, 
                   bounds, 
                   n, 
-                  k_seq=np.array([(1 + (0.01/0.5))**i for i in range(-10000, 10000)]), 
                   p=0.1,
                   seed=None):
     """
@@ -111,6 +110,8 @@ def adaptive_lipo(func,
     # dimension of the domain
     d = len(bounds)
 
+    k_seq=(1+0.01/d)**np.arange(-10000,10000) # Page 16
+
     # preallocate the output arrays
     y = np.zeros(n) - np.Inf
     x = np.zeros((n, d))
@@ -126,24 +127,24 @@ def adaptive_lipo(func,
 
     # initialization with randomly drawn point in domain and k = 0
     k = 0
-    u = np.random.uniform(size=d)
+    u = np.random.rand(d)
     x_prop = u * (bound_maxs - bound_mins) + bound_mins
     x[0] = x_prop
     y[0] = func(x_prop)
 
-    upper_bound = lambda x_prop, y, x, k: np.min(y+k*np.linalg.norm(x_prop-x))
+    upper_bound = lambda x_prop, y, x, k: np.min(y+k*np.linalg.norm(x_prop-x,axis=1))
 
     for t in np.arange(1, n):
 
         # draw a uniformly distributed random variable
-        u = np.random.uniform(size=d)
+        u = np.random.rand(d)
         x_prop = u * (bound_maxs - bound_mins) + bound_mins
 
         # check if we are exploring or exploiting
-        if not np.random.binomial(n=1, p=p):
+        if np.random.rand() > p: # enter w/ prob (1-p)
             # exploiting - ensure we're drawing from potential maximizers
             while upper_bound(x_prop, y[:t], x[:t], k) < np.max(y):
-                u = np.random.uniform(size=d)
+                u = np.random.rand(d)
                 x_prop = u * (bound_maxs - bound_mins) + bound_mins 
 
         # add proposal to array of visited points
@@ -167,7 +168,12 @@ def adaptive_lipo(func,
         # and update estimate of lipschitz constant
         new_num_dist = old_num_dist + t
         k_est = np.max(y_dist[:new_num_dist] / x_dist[:new_num_dist]) 
-        k = k_seq[np.argmax(k_seq > k_est)]
+        # get the smallest element of k_seq that is bigger than k_est
+        # note: we're using argmax to get the first occurrence
+        # note: this relies on k_seq being sorted in nondecreasing order
+        k = k_seq[np.argmax(k_seq >= k_est)]
+        # note: in the paper, k_seq is called k_i 
+        #       and k is called \hat{k}_t
 
 
     output = {'loss': loss, 'x': x, 'y': y}