""" ============== HPO NGBoost ============== """ import os import sys import site proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) print(f'project directory is: {proj_dir}') site.addsitedir(proj_dir) import warnings from typing import Union, List, Callable import os import math import json import numpy as np import pandas as pd from ngboost import NGBRegressor from ngboost.distns import Exponential, Normal, LogNormal from SeqMetrics import RegressionMetrics from ai4water import Model from ai4water._main import fill_val from ai4water.utils import TrainTestSplit from ai4water.utils.utils import get_version_info from ai4water.utils.utils import dateandtime_now, jsonize from ai4water.hyperopt import HyperOpt, Categorical, Real, Integer from utils import set_rcParams, \ version_info, make_data # %% for lib,ver in get_version_info().items(): print(lib, ver) # %% TRAIN_FRACTION = 0.7 def cross_val_score( rgr, x, y, scoring: Union [str, List[str], Callable], cross_validator, cross_validator_args = None, refit: bool = False, process_results:bool = False, **kwargs ) -> list: Metrics = RegressionMetrics scores = [] #cross_validator_args = self.config['cross_validator'][cross_validator] splitter = TrainTestSplit(test_fraction=1.0 - TRAIN_FRACTION) splits = getattr(splitter, cross_validator)(x, y, **cross_validator_args) for fold, ((train_x, train_y), (test_x, test_y)) in enumerate(splits): model = rgr(**kwargs) model.fit(train_x, train_y.reshape(-1, )) # since we have access to true y, it is better to provide it # it will be used for processing of results pred = model.predict(test_x,) if len(pred[pred<0.0]): warnings.warn(f'{len(pred[pred<0.0])} -ve values encountered!') metrics = Metrics(test_y.reshape(-1, 1), pred) val_scores = [] for score in scoring: if callable(score): val_scores.append(score(test_y.reshape(-1, 1), pred)) else: val_scores.append(getattr(metrics, score)()) scores.append(val_scores) scores = np.array(scores) cv_scores_ = np.nanmean(scores, axis=0) max_val = max(cv_scores_) avg_val = np.nanmean(cv_scores_).item() if np.isinf(cv_scores_).any(): # if there is inf, we should not fill it with very large value (999999999) # but it should be max(so far experienced) value cv_scores_no_inf = cv_scores_.copy() cv_scores_no_inf[np.isinf(cv_scores_no_inf)] = np.nan cv_scores_no_inf[np.isnan(cv_scores_no_inf)] = np.nanmax(cv_scores_no_inf) max_val = max(cv_scores_no_inf) # check for both infinity and nans separately # because they come due to different reasons cv_scores = [] for cv_score, metric_name in zip(cv_scores_, scoring): # math.isinf(np.nan) will be false therefore # first check if cv_score is nan or not, if true, fill it with avg_val if math.isnan(cv_score): cv_score = fill_val(metric_name, default_min=avg_val) # then check if cv_score is infinity because elif math.isinf(cv_score): cv_score = fill_val(metric_name, default_min=max_val) cv_scores.append(cv_score) return cv_scores # %% data, _, encoders= make_data(encoding='le') TrainX, TestX, TrainY, TestY = TrainTestSplit(seed=142).\ random_split_by_groups(x=data.iloc[:,0:-1], y=data.iloc[:, -1], groups=data['Adsorbent']) print(TrainX.shape, TestX.shape, TrainY.shape, TestY.shape) # %% input_features = data.columns.tolist()[0:-1] output_features = data.columns.tolist()[-1:] print(input_features) # %% print(output_features) # %% DISTS = { "Normal": Normal, "LogNormal": LogNormal, "Exponential": Exponential } ITER = 0 VAL_SCORES = [] SUGGESTIONS = [] num_iterations = 300 # number of hyperparameter iterations SEP = os.sep algorithm = "tpe" PREFIX = f"hpo_NGB_{algorithm}_{dateandtime_now()}" # folder name where to save the results PREFIX = f"/mnt/datawaha/hyex/atr/playground/results/abcabc/{SEP}{PREFIX}" # %% # define parameter space param_space = [ #Categorical(["Normal"], name="Dist"), Integer(100, 1000, name="n_estimators"), Real(0.001, 0.5, name="learning_rate"), Real(0.4, 1.0, name="minibatch_frac"), Real(0.4, 1.0, name="col_sample") ] # %% # initial values of hyperparameters x0 = [#"Normal", 100, 0.01, 1.0, 1.0 ] # %% # define objective function def objective_fn( prefix=None, return_model:bool = False, **suggestions )->Union[float, Model]: """ The output of this function will be minimized :param prefix : :param return_model: whether to return the trained model or the validation score. This will be set to True, after we have optimized the hyperparameters :param suggestions: contains values of hyperparameters at each iteration :return: the scalar value which we want to minimize. If return_model is True then it returns the trained model """ global ITER suggestions = jsonize(suggestions) SUGGESTIONS.append(suggestions) suggestions['Dist'] = Exponential suggestions['verbose'] = False suggestions['random_state'] = 313 if return_model: # build the model ngb = NGBRegressor(**suggestions) model = Model( model=ngb, mode="regression", category="ML", cross_validator={"GroupKFold": {"n_splits": 5}}, input_features=input_features, output_features=output_features, prefix=prefix or PREFIX, verbosity=2 ) model.fit(TrainX.values, TrainY.values, validation_data=(TestX, TestY.values)) model.evaluate(TestX, TestY, metrics=["r2", "r2_score"]) return model # get the cross validation score which we will minimize val_score_ = cross_val_score(NGBRegressor, x=TrainX.values, y=TrainY.values, cross_validator='GroupKFold', scoring=['r2_score'], cross_validator_args={'n_splits': 5, 'groups': TrainX.loc[:, 'Adsorbent']}, **suggestions ) assert len(val_score_) == 1 val_score_ = val_score_[0] # since cross val score is r2_score, we need to subtract it from 1. Because # we are interested in increasing r2_score, and HyperOpt algorithm always # minizes the objective function val_score = 1 - val_score_ VAL_SCORES.append(val_score) best_score = round(np.nanmin(VAL_SCORES).item(), 2) bst_iter = np.argmin(VAL_SCORES) ITER += 1 print(f"{ITER} {round(val_score, 2)} {round(val_score_, 2)}. Best was {best_score} at {bst_iter}") return val_score # %% # initialize the hpo # optimizer = HyperOpt( # algorithm=algorithm, # objective_fn=objective_fn, # param_space=param_space, # x0=x0, # num_iterations=num_iterations, # process_results=False, # we can turn it False if we want post-processing of results # opt_path=PREFIX # ) # %% # run the hpo # res = optimizer.fit() # %% # print optimized hyperparameters # print(optimizer.best_paras()) # %% # # plot convergence # # optimizer.plot_convergence(show=True) # %% # optimizer.plot_convergence(original=True, show=True) # # # %% # # plot explored hyperparameters as explored during hpo # # optimizer.plot_parallel_coords(show=True) # %% # # build and train the model with optimized hyperparameters # best_model = objective_fn(return_model=True, **optimizer.best_paras())