diff --git a/Readme.md b/Readme.md index 095cb9a..fe5ef3d 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,18 @@ LIPO is a package for derivative-free, global optimization. Is based on the `dlib` package and provides wrappers around its optimization routine. +The algorithm outperforms random search - sometimes by margins as large as 10000x. It is often preferable to +Bayesian optimization which requires "tuning of the tuner". Performance is on par with moderately to well tuned Bayesian +optimization. + +The provided implementation has the option to automatically enlarge the search space if bounds are found to be +too restrictive (i.e. the optimum being to close to one of them). + +See the `LIPO algorithm implementation `_ for details. + +A `great blog post `_ by the author of +`dlib` exists, describing how it works. + # Installation Execute diff --git a/lipo/hyperparameter.py b/lipo/hyperparameter.py index 62c84e1..13ffcc8 100644 --- a/lipo/hyperparameter.py +++ b/lipo/hyperparameter.py @@ -30,8 +30,9 @@ def __init__( estimator, param_space, n_iter=10, - tolerance=0.0, flexible_bound_threshold=0.05, + flexible_bounds={}, + tolerance=0.0, random_state=None, scoring=None, n_jobs=None, @@ -62,6 +63,10 @@ def __init__( - List of choices to test. List must either contain more than 2 elements or only strings. n_iter (int): number of iterations for fitting the estimator + flexible_bounds (Dict[str, List[bool]]): dictionary of parameters and list of booleans indicating + if parameters are deemed flexible or not. by default all parameters are deemed flexible + flexible_bound_threshold (float): if to enlarge bounds if optimum is top or bottom + ``flexible_bound_threshold`` quantile tolerance (float): Skip local search step if accuracy of found maximum is below tolerance. Continue with global search. scoring (Union[str, callable, List, Tuple, Dict, None]: as in sklearn.model_selection.GridSearchCV @@ -78,6 +83,7 @@ def __init__( self.param_space = param_space self.n_iter = n_iter self.tolerance = tolerance + self.flexible_bounds = flexible_bounds self.flexible_bound_threshold = flexible_bound_threshold self.random_state = random_state @@ -111,6 +117,7 @@ def _run_search(self, evaluate_candidates): upper_bounds=upper_bounds, categories=categories, flexible_bound_threshold=self.flexible_bound_threshold, + flexible_bounds=self.flexible_bounds, epsilon=self.tolerance, maximize=True, random_state=self.random_state, diff --git a/lipo/optimizer.py b/lipo/optimizer.py index 1dba5ed..ca36699 100644 --- a/lipo/optimizer.py +++ b/lipo/optimizer.py @@ -55,6 +55,7 @@ def __init__( upper_bounds: Dict[str, Union[float, int]] = {}, categories: Dict[str, List[str]] = {}, log_args: Union[str, List[str]] = "auto", + flexible_bounds: Dict[str, List[bool]] = {}, flexible_bound_threshold: float = 0.05, evaluations: List[Tuple[Dict[str, Union[float, int, str]], float]] = [], maximize: bool = True, @@ -74,7 +75,8 @@ def __init__( - The lower bound on the variable is > 0 - The ratio of the upper bound to lower bound is > 1000 - The variable is not an integer variable - + flexible_bounds (Dict[str, List[bool]]): dictionary of parameters and list of booleans indicating + if parameters are deemed flexible or not. by default all parameters are deemed flexible flexible_bound_threshold (float): if to enlarge bounds if optimum is top or bottom ``flexible_bound_threshold`` quantile evaluations List[Tuple[Dict[str], float]]: list of tuples of x and y values @@ -154,6 +156,7 @@ def __init__( # check bound threshold assert flexible_bound_threshold < 0.5, "Quantile for bound flexibility has to be below 0.5" self.flexible_bound_threshold = flexible_bound_threshold + self.flexible_bounds = {name: flexible_bounds.get(name, [True, True]) for name in self.arg_names} # initialize search object self._init_search() @@ -204,7 +207,8 @@ def get_candidate(self): else: val = optimum_args[name] - if (val - lower) / span <= self.flexible_bound_threshold: + # redefine lower bound + if (val - lower) / span <= self.flexible_bound_threshold and self.flexible_bounds[name][0]: # center value proposed_val = val - (upper - val) # limit change in log space @@ -216,7 +220,8 @@ def get_candidate(self): # restart search reinit = True - elif (upper - val) / span <= self.flexible_bound_threshold: + # redefine upper bound + elif (upper - val) / span <= self.flexible_bound_threshold and self.flexible_bounds[name][1]: # center value proposed_val = val + (val - lower) # limit log space redefinition