Example with conditional model selection

[CreaperLost]

I would like to use this to optimize XGBoost, RF and other methods in CASH modelling. I can't see how I could configure the method to work with conditional hyper-parameters.

[ZebinYang]

Hi @CreaperLost

SeqUD simply ignores the dependency among hyper-parameters.

The example of tuning XGBoost can be found in cell No. 17 of https://github.com/ZebinYang/SeqUD/blob/master/examples/demo_sklearn_opt.ipynb.

[CreaperLost]

Thanks for the answer. I was looking more for the cash optimization. Could you please provide a simple example of optimizing over RF and SVMs? Because the nature of the algorithm is not categorical selection but actual conditional hyper-parameters. Is that gonna lead to a crash?

[ZebinYang]

For combined model selection and HPO, you can define a custom model pipeline to choose the model and configure the corresponding hyperparameters. See the example below.

import warnings

from sklearn.svm import SVC, SVR
from sklearn.pipeline import Pipeline
from xgboost import XGBClassifier, XGBRegressor
from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin


class RegPipe(BaseEstimator, RegressorMixin):

    def __init__(self, estimator_selector="xgb", 
                 xgb_params=None, svm_params=None, random_state=0):

        """
        A Custom BaseEstimator that can switch between regressors.
        :param estimator: sklearn object - The regressor
        """ 
        self.svm_params = svm_params if svm_params else {}
        self.xgb_params = xgb_params if xgb_params else {}
        self.random_state = random_state

        self.estimator_selector = estimator_selector

    def set_params(self, **params):

        for key, value in params.items():
            key, delim, sub_key = key.partition('__')
            if delim:
                if hasattr(self, key):
                    getattr(self, key).update({sub_key:value})
                else:
                    setattr(self, key, {})
            else:
                setattr(self, key, value)
        return self

    def get_params(self, deep=True):
        """
        Get parameters for this estimator.
        Parameters
        ----------
        deep : bool, default=True
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.
        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        out = dict()
        for key in self._get_param_names():
            try:
                value = getattr(self, key)
            except AttributeError:
                warnings.warn('From version 0.24, get_params will raise an '
                              'AttributeError if a parameter cannot be '
                              'retrieved as an instance attribute. Previously '
                              'it would return None.',
                              FutureWarning)
                value = None
            if deep and hasattr(value, 'get_params'):
                deep_items = value.get_params().items()
                out.update((key + '__' + k, val) for k, val in deep_items)
            out[key] = value
        return out

    def fit(self, X, y=None):
            
        # Model fitting
        if self.estimator_selector == "xgb":
            self.estimator = XGBRegressor(**self.xgb_params)
            self.estimator.set_params(**{"n_jobs":1})
            self.estimator.set_params(**{"random_state":self.random_state})
        elif self.estimator_selector == "svm":
            self.estimator = SVR()
            self.estimator.set_params(**self.svm_params)
        self.estimator.fit(X, y)
        return self

    def predict(self, X):
        return self.estimator.predict(X)


import numpy as np
from sequd import SeqUD
from sklearn.datasets import make_friedman1

ParaSpace = {'estimator_selector':        {'Type': 'categorical', 'Mapping': ['xgb', 'svm']},
        'svm_params__C':          {'Type': 'continuous', 'Range': [-6, 16], 'Wrapper': np.exp2}, 
        'svm_params__gamma':        {'Type': 'continuous', 'Range': [-16, 6], 'Wrapper': np.exp2},
        'xgb_params__booster':       {'Type': 'categorical', 'Mapping': ['gbtree', 'gblinear']},
        'xgb_params__n_estimators':    {'Type': 'integer',     'Mapping': np.linspace(100, 500, 401)},
        'xgb_params__learning_rate':    {'Type': 'continuous',  'Range': [-5, 0], 'Wrapper': lambda x: 10**x}}

estimator = RegPipe()
x, y = make_friedman1()
clf = SeqUD(ParaSpace, estimator=estimator, max_search_iter=10, random_state=0, verbose=True)
clf.fit(x, y)