make some bounds more flexible than others

Readme.md

@@ -1,6 +1,18 @@
 LIPO is a package for derivative-free, global optimization. Is based on
 the `dlib` package and provides wrappers around its optimization routine.
 
+The algorithm outperforms random search - sometimes by margins as large as 10000x. It is often preferable to 
+Bayesian optimization which requires "tuning of the tuner". Performance is on par with moderately to well tuned Bayesian 
+optimization.
+
+The provided implementation has the option to automatically enlarge the search space if bounds are found to be 
+too restrictive (i.e. the optimum being to close to one of them).
+
+See the `LIPO algorithm implementation <http://dlib.net/python/index.html#dlib.find_max_global>`_ for details.
+
+A `great blog post <http://blog.dlib.net/2017/12/a-global-optimization-algorithm-worth.html>`_ by the author of 
+`dlib` exists, describing how it works.
+
 # Installation
 
 Execute

lipo/hyperparameter.py

@@ -30,8 +30,9 @@ def __init__(
         estimator,
         param_space,
         n_iter=10,
-        tolerance=0.0,
         flexible_bound_threshold=0.05,
+        flexible_bounds={},
+        tolerance=0.0,
         random_state=None,
         scoring=None,
         n_jobs=None,
@@ -62,6 +63,10 @@ def __init__(
                 - List of choices to test. List must either contain more than 2 elements or only strings.
 
             n_iter (int): number of iterations for fitting the estimator
+            flexible_bounds (Dict[str, List[bool]]): dictionary of parameters and list of booleans indicating
+                if parameters are deemed flexible or not. by default all parameters are deemed flexible
+            flexible_bound_threshold (float): if to enlarge bounds if optimum is top or bottom
+                ``flexible_bound_threshold`` quantile
             tolerance (float): Skip local search step if accuracy of found maximum is below tolerance.
                 Continue with global search.
             scoring (Union[str, callable, List, Tuple, Dict, None]: as in sklearn.model_selection.GridSearchCV
@@ -78,6 +83,7 @@ def __init__(
         self.param_space = param_space
         self.n_iter = n_iter
         self.tolerance = tolerance
+        self.flexible_bounds = flexible_bounds
         self.flexible_bound_threshold = flexible_bound_threshold
         self.random_state = random_state
 
@@ -111,6 +117,7 @@ def _run_search(self, evaluate_candidates):
             upper_bounds=upper_bounds,
             categories=categories,
             flexible_bound_threshold=self.flexible_bound_threshold,
+            flexible_bounds=self.flexible_bounds,
             epsilon=self.tolerance,
             maximize=True,
             random_state=self.random_state,

lipo/optimizer.py

@@ -55,6 +55,7 @@ def __init__(
         upper_bounds: Dict[str, Union[float, int]] = {},
         categories: Dict[str, List[str]] = {},
         log_args: Union[str, List[str]] = "auto",
+        flexible_bounds: Dict[str, List[bool]] = {},
         flexible_bound_threshold: float = 0.05,
         evaluations: List[Tuple[Dict[str, Union[float, int, str]], float]] = [],
         maximize: bool = True,
@@ -74,7 +75,8 @@ def __init__(
                 - The lower bound on the variable is > 0
                 - The ratio of the upper bound to lower bound is > 1000
                 - The variable is not an integer variable
-
+            flexible_bounds (Dict[str, List[bool]]): dictionary of parameters and list of booleans indicating
+                if parameters are deemed flexible or not. by default all parameters are deemed flexible
             flexible_bound_threshold (float): if to enlarge bounds if optimum is top or bottom
                 ``flexible_bound_threshold`` quantile
             evaluations List[Tuple[Dict[str], float]]: list of tuples of x and y values
@@ -154,6 +156,7 @@ def __init__(
         # check bound threshold
         assert flexible_bound_threshold < 0.5, "Quantile for bound flexibility has to be below 0.5"
         self.flexible_bound_threshold = flexible_bound_threshold
+        self.flexible_bounds = {name: flexible_bounds.get(name, [True, True]) for name in self.arg_names}
 
         # initialize search object
         self._init_search()
@@ -204,7 +207,8 @@ def get_candidate(self):
                     else:
                         val = optimum_args[name]
 
-                    if (val - lower) / span <= self.flexible_bound_threshold:
+                    # redefine lower bound
+                    if (val - lower) / span <= self.flexible_bound_threshold and self.flexible_bounds[name][0]:
                         # center value
                         proposed_val = val - (upper - val)
                         # limit change in log space
@@ -216,7 +220,8 @@ def get_candidate(self):
                             # restart search
                             reinit = True
 
-                    elif (upper - val) / span <= self.flexible_bound_threshold:
+                    # redefine upper bound
+                    elif (upper - val) / span <= self.flexible_bound_threshold and self.flexible_bounds[name][1]:
                         # center value
                         proposed_val = val + (val - lower)
                         # limit log space redefinition