Hyperparameter tuning

import pandas as pd
import numpy as np
show(pd.DataFrame([
    {"": "Parameter (weight)", "Who finds it": "THE MODEL, during fit()",
     "Example": "A line's coefficient, a tree's threshold"},
    {"": "Hyperparameter", "Who finds it": "YOU, BEFORE fit()",
     "Example": "max_depth, C, learning_rate, n_neighbors"},
]))

from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, RandomizedSearchCV
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat = ["tuman"]
X, y = df[num + cat], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

def pipe(model):
    return Pipeline([
        ("t", ColumnTransformer([
            ("num", Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
        ])),
        ("m", model),
    ])

print(f"{'model (default settings)':<34} {'CV F1':>9} {'±':>8}")
print("-" * 54)
default = {}
for name, m in [("LogisticRegression", LogisticRegression(max_iter=3000)),
                ("RandomForest", RandomForestClassifier(random_state=42)),
                ("SVC", SVC(random_state=42))]:
    b = cross_val_score(pipe(m), X, y, cv=5, scoring="f1")
    default[name] = b
    print(f"{name:<34} {b.mean():>9.4f} {b.std():>8.4f}")

import time

grid = {
    "m__n_estimators": [100, 300],
    "m__max_depth": [None, 6, 12],
    "m__min_samples_leaf": [1, 5, 20],
    "m__max_features": ["sqrt", 0.5],
}
total = 1
for v in grid.values():
    total *= len(v)
print(f"Number of combinations: {total}")
print(f"CV folds              : 5")
print(f"Total trainings       : {total * 5}")

t0 = time.perf_counter()
gs = GridSearchCV(pipe(RandomForestClassifier(random_state=42)), grid,
                  cv=5, scoring="f1", n_jobs=-1)
gs.fit(X_tr, y_tr)
gs_time = time.perf_counter() - t0
print()
print(f"Time                  : {gs_time:.1f}s")
print(f"Best CV F1            : {gs.best_score_:.4f}")
print("Best settings:")
for k, v in gs.best_params_.items():
    print(f"  {k.replace('m__', ''):20s} = {v}")

print(f"Default settings CV F1: {default['RandomForest'].mean():.4f} ±{default['RandomForest'].std():.4f}")
print(f"After tuning          : {gs.best_score_:.4f}")
print(f"Gain from tuning      : {gs.best_score_ - default['RandomForest'].mean():+.4f}")
print(f"CV spread             : ±{default['RandomForest'].std():.4f}")
print()
gain = gs.best_score_ - default["RandomForest"].mean()
print("Verdict:", "the gain is LARGER than the spread: tuning was worth it"
      if gain > default["RandomForest"].std()
      else "the gain is WITHIN the spread: tuning gave almost nothing")

results = pd.DataFrame(gs.cv_results_)
cols = ["param_m__n_estimators", "param_m__max_depth",
        "param_m__min_samples_leaf", "param_m__max_features",
        "mean_test_score", "std_test_score"]
top = results[cols].sort_values("mean_test_score", ascending=False).head(8).copy()
top.columns = ["n_est", "depth", "min_leaf", "max_feat", "CV F1", "±"]
top["CV F1"] = top["CV F1"].round(4)
top["±"] = top["±"].round(4)
show(top)
print()
best = results.mean_test_score.max()
close = int((results.mean_test_score > best - results.std_test_score.max()).sum())
print(f"Combinations within one spread of the best score: {close} / {len(results)}")

from scipy.stats import randint, uniform

big_grid = {
    "m__n_estimators": randint(50, 500),
    "m__max_depth": [None, 4, 6, 8, 12, 20],
    "m__min_samples_leaf": randint(1, 40),
    "m__min_samples_split": randint(2, 40),
    "m__max_features": ["sqrt", "log2", 0.3, 0.5, 0.8],
}
print("If we did it with GridSearch (roughly):")
print(f"  450 x 6 x 39 x 38 x 5 = {450*6*39*38*5:,} combinations".replace(",", " "))
print("  -> impossible in practice")
print()
t0 = time.perf_counter()
rs = RandomizedSearchCV(pipe(RandomForestClassifier(random_state=42)), big_grid,
                        n_iter=40, cv=5, scoring="f1", random_state=42, n_jobs=-1)
rs.fit(X_tr, y_tr)
rs_time = time.perf_counter() - t0
print(f"RandomizedSearchCV (40 tries): {rs_time:.1f}s")
print(f"  best CV F1: {rs.best_score_:.4f}")
print(f"GridSearchCV ({total} combinations): {gs_time:.1f}s")
print(f"  best CV F1: {gs.best_score_:.4f}")

from sklearn.experimental import enable_halving_search_cv  # noqa: F401
from sklearn.model_selection import HalvingGridSearchCV

t0 = time.perf_counter()
hs = HalvingGridSearchCV(pipe(RandomForestClassifier(random_state=42)), grid,
                         cv=5, scoring="f1", random_state=42, n_jobs=-1, factor=3)
hs.fit(X_tr, y_tr)
hs_time = time.perf_counter() - t0
print(f"{'method':<24} {'time':>8} {'best CV F1':>18}")
print("-" * 52)
print(f"{'GridSearchCV':<24} {gs_time:>7.1f}s {gs.best_score_:>18.4f}")
print(f"{'RandomizedSearchCV':<24} {rs_time:>7.1f}s {rs.best_score_:>18.4f}")
print(f"{'HalvingGridSearchCV':<24} {hs_time:>7.1f}s {hs.best_score_:>18.4f}")

print("The WRONG way: choosing the best setting on TEST:")
from sklearn.metrics import f1_score
best_test, best_setting = -1, None
for d in [None, 4, 6, 8, 12, 20]:
    for msl in [1, 5, 10, 20, 40]:
        m = pipe(RandomForestClassifier(max_depth=d, min_samples_leaf=msl, random_state=42))
        m.fit(X_tr, y_tr)
        score = f1_score(y_te, m.predict(X_te))
        if score > best_test:
            best_test, best_setting = score, (d, msl)
print(f"  30 combinations tried on TEST")
print(f"  best test F1: {best_test:.4f}  (max_depth={best_setting[0]}, min_samples_leaf={best_setting[1]})")
print()
print("The RIGHT way: choose with CV, then check on test ONCE:")
print(f"  the best chosen by CV : {gs.best_score_:.4f}")
print(f"  that model on test    : {f1_score(y_te, gs.best_estimator_.predict(X_te)):.4f}")

from sklearn.model_selection import KFold, cross_val_score as cvs

small_grid = {"m__max_depth": [None, 6, 12], "m__min_samples_leaf": [1, 5, 20]}

# plain CV: the very CV used to choose the settings
plain = GridSearchCV(pipe(RandomForestClassifier(random_state=42)), small_grid,
                     cv=5, scoring="f1", n_jobs=-1).fit(X, y)
print(f"GridSearchCV best_score_ (optimistic): {plain.best_score_:.4f}")

# nested CV: the outer loop NEVER sees the tuning process
inner = GridSearchCV(pipe(RandomForestClassifier(random_state=42)), small_grid,
                     cv=3, scoring="f1", n_jobs=-1)
nested = cvs(inner, X, y, cv=KFold(5, shuffle=True, random_state=42), scoring="f1")
print(f"Nested CV (unbiased estimate)        : {nested.mean():.4f} ±{nested.std():.4f}")
print(f"Difference                           : {plain.best_score_ - nested.mean():+.4f}")

show(pd.DataFrame([
    {"Model": "LogisticRegression", "Important": "C, penalty", "Can leave alone": "solver, max_iter"},
    {"Model": "RandomForest", "Important": "max_depth, min_samples_leaf, max_features",
     "Can leave alone": "n_estimators (set it large, Chapter 19)"},
    {"Model": "XGBoost / LightGBM", "Important": "learning_rate, max_depth, subsample, colsample",
     "Can leave alone": "n_estimators (early_stopping, Chapter 20)"},
    {"Model": "SVC", "Important": "C, gamma (TOGETHER, Chapter 22)", "Can leave alone": "kernel (start with rbf)"},
    {"Model": "KNN", "Important": "n_neighbors, weights", "Can leave alone": "algorithm"},
]))

print("Tuning C for logistic regression:")
gs_lr = GridSearchCV(pipe(LogisticRegression(max_iter=3000)),
                     {"m__C": [0.001, 0.01, 0.1, 1, 10, 100]},
                     cv=5, scoring="f1", n_jobs=-1).fit(X, y)
print(f"  default (C=1)    : {default['LogisticRegression'].mean():.4f}")
print(f"  best C={gs_lr.best_params_['m__C']:<10}: {gs_lr.best_score_:.4f}")
print(f"  gain             : {gs_lr.best_score_ - default['LogisticRegression'].mean():+.4f}")

final = pd.DataFrame([
    {"Model": "LogisticRegression", "Default": round(default["LogisticRegression"].mean(), 4),
     "Tuned": round(gs_lr.best_score_, 4),
     "Gain": round(gs_lr.best_score_ - default["LogisticRegression"].mean(), 4)},
    {"Model": "RandomForest", "Default": round(default["RandomForest"].mean(), 4),
     "Tuned": round(gs.best_score_, 4),
     "Gain": round(gs.best_score_ - default["RandomForest"].mean(), 4)},
])
show(final)
print()
print(f"Largest CV spread: ±{max(b.std() for b in default.values()):.4f}")
print("Compare the gains with that spread (the Chapter 10 rule).")

show(pd.DataFrame([
    {"Concept": "Hyperparameter", "In one sentence": "A setting you choose BEFORE fit()"},
    {"Concept": "GridSearchCV", "In one sentence": "Every combination: for a small grid"},
    {"Concept": "RandomizedSearchCV", "In one sentence": "N random ones: for a big grid, the first choice in practice"},
    {"Concept": "HalvingGridSearchCV", "In one sentence": "The tournament method: drops the bad ones early"},
    {"Concept": "The gain from tuning", "In one sentence": "Usually 1-3%, sometimes zero: do it last"},
    {"Concept": "The biggest mistake", "In one sentence": "Choosing the setting on TEST: leakage"},
    {"Concept": "best_score_", "In one sentence": "A little optimistic: nested CV is less biased"},
    {"Concept": "How many parameters", "In one sentence": "Each model has 2-4 that matter; the rest waste time"},
]))