Boosting: XGBoost, LightGBM, CatBoost

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import root_mean_squared_error

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[num], df["narx_usd"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42)

# Step 1: a simple prediction, the mean for everyone
pred_tr = np.full(len(y_tr), y_tr.mean())
pred_te = np.full(len(y_te), y_tr.mean())
trees = []
lr = 0.1

print(f"{'step':>6} {'train RMSE':>12} {'test RMSE':>11}")
print("-" * 32)
print(f"{0:>6} {root_mean_squared_error(y_tr, pred_tr):>12.1f} "
      f"{root_mean_squared_error(y_te, pred_te):>11.1f}")

for step in range(1, 101):
    resid = y_tr - pred_tr                            # the CURRENT error
    d = DecisionTreeRegressor(max_depth=3, random_state=42).fit(X_tr, resid)
    trees.append(d)
    pred_tr = pred_tr + lr * d.predict(X_tr)          # fix the error a little
    pred_te = pred_te + lr * d.predict(X_te)
    if step in (1, 5, 20, 50, 100):
        print(f"{step:>6} {root_mean_squared_error(y_tr, pred_tr):>12.1f} "
              f"{root_mean_squared_error(y_te, pred_te):>11.1f}")

from sklearn.ensemble import GradientBoostingRegressor

sk = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1,
                               max_depth=3, random_state=42).fit(X_tr, y_tr)
print(f"Our hand-written version: test RMSE = {root_mean_squared_error(y_te, pred_te):.1f}")
print(f"sklearn GradientBoosting: test RMSE = {root_mean_squared_error(y_te, sk.predict(X_te)):.1f}")

show(pd.DataFrame([
    {"Library": "XGBoost", "Year": 2014, "Main strength": "The first popular one; regularisation built in",
     "Weakness": "Doesn't handle categories itself"},
    {"Library": "LightGBM", "Year": 2017, "Main strength": "THE FASTEST; wins on large data",
     "Weakness": "Prone to overfitting on small data"},
    {"Library": "CatBoost", "Year": 2017, "Main strength": "Handles categories ITSELF; good defaults",
     "Weakness": "Slower to train"},
]))

import time
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from sklearn.ensemble import RandomForestRegressor

print(f"{'model':<26} {'CV R2':>9} {'±':>8} {'time':>8}")
print("-" * 54)
for name, m in [
    ("RandomForest (Chapter 19)", RandomForestRegressor(n_estimators=200, random_state=42)),
    ("GradientBoosting", GradientBoostingRegressor(random_state=42)),
    ("XGBoost", XGBRegressor(n_estimators=200, verbosity=0, random_state=42)),
    ("LightGBM", LGBMRegressor(n_estimators=200, verbose=-1, random_state=42)),
    ("CatBoost", CatBoostRegressor(iterations=200, verbose=0, random_seed=42)),
]:
    t0 = time.perf_counter()
    b = cross_val_score(m, X, y, cv=5, scoring="r2")
    dt = time.perf_counter() - t0
    print(f"{name:<26} {b.mean():>9.4f} {b.std():>8.4f} {dt:>7.1f}s")

from xgboost import XGBRegressor
print(f"{'learning_rate':>14} {'n_estimators':>13} {'CV R2':>9}")
print("-" * 40)
for lr_ in [0.01, 0.05, 0.1, 0.3, 1.0]:
    b = cross_val_score(XGBRegressor(n_estimators=200, learning_rate=lr_, verbosity=0,
                                     random_state=42), X, y, cv=5, scoring="r2")
    print(f"{lr_:>14} {200:>13} {b.mean():>9.4f}")

print("A small learning_rate + many trees:")
print(f"{'lr':>6} {'trees':>11} {'CV R2':>9}")
print("-" * 28)
for lr_, n in [(0.3, 50), (0.1, 150), (0.05, 300), (0.01, 1500)]:
    b = cross_val_score(XGBRegressor(n_estimators=n, learning_rate=lr_, verbosity=0,
                                     random_state=42), X, y, cv=5, scoring="r2")
    print(f"{lr_:>6} {n:>11} {b.mean():>9.4f}")

X_tr2, X_val, y_tr2, y_val = train_test_split(X_tr, y_tr, test_size=0.25, random_state=42)
m_es = XGBRegressor(n_estimators=3000, learning_rate=0.05, early_stopping_rounds=50,
                    verbosity=0, random_state=42)
m_es.fit(X_tr2, y_tr2, eval_set=[(X_val, y_val)], verbose=False)
print(f"Trees allowed           : 3000")
print(f"Early stopping stopped at: {m_es.best_iteration} trees")
print(f"Test R2                 : {m_es.score(X_te, y_te):.4f}")

from catboost import CatBoostClassifier
yc = df["ijaraga_berildi_30k"]
Xk = df[num + ["tuman"]]

# CatBoost accepts a text column DIRECTLY
cb = CatBoostClassifier(iterations=200, verbose=0, random_seed=42,
                        cat_features=["tuman"]).fit(Xk, yc)
print("CatBoost accepted the text column WITHOUT encoding: no error")
print(f"  train accuracy: {cb.score(Xk, yc):.4f}")
print()
# the others need OneHot
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from xgboost import XGBClassifier
q = Pipeline([
    ("t", ColumnTransformer([("m", OneHotEncoder(handle_unknown="ignore"), ["tuman"])],
                            remainder="passthrough")),
    ("m", XGBClassifier(n_estimators=200, verbosity=0, random_state=42)),
])
from sklearn.metrics import f1_score
Xk_tr, Xk_te, yk_tr, yk_te = train_test_split(Xk, yc, test_size=0.3, random_state=42, stratify=yc)

q.fit(Xk_tr, yk_tr)
print(f"XGBoost + OneHot:  test F1 = {f1_score(yk_te, q.predict(Xk_te)):.4f}")

cb2 = CatBoostClassifier(iterations=200, verbose=0, random_seed=42,
                         cat_features=["tuman"]).fit(Xk_tr, yk_tr)
print(f"CatBoost (itself):  test F1 = {f1_score(yk_te, cb2.predict(Xk_te)):.4f}")

show(pd.DataFrame([
    {"Parameter": "learning_rate", "Meaning": "How much each tree corrects", "In practice": "0.01-0.1"},
    {"Parameter": "n_estimators", "Meaning": "The number of trees", "In practice": "found with early_stopping"},
    {"Parameter": "max_depth", "Meaning": "Tree depth", "In practice": "3-8 (SHALLOWER than RF)"},
    {"Parameter": "subsample", "Meaning": "Share of rows per tree", "In practice": "0.7-1.0"},
    {"Parameter": "colsample_bytree", "Meaning": "Share of columns per tree", "In practice": "0.7-1.0"},
    {"Parameter": "reg_lambda", "Meaning": "L2 penalty (Chapter 17)", "In practice": "1-10"},
]))

show(pd.DataFrame([
    {"Concept": "The boosting idea", "In one sentence": "Each tree learns the previous trees' MISTAKES"},
    {"Concept": "Three lines of code", "In one sentence": "resid -> fit(X, resid) -> pred += lr * new"},
    {"Concept": "Tree depth", "In one sentence": "Shallow (3-6), unlike in RF"},
    {"Concept": "learning_rate", "In one sentence": "The smaller it is, the more trees you need"},
    {"Concept": "n_estimators", "In one sentence": "Raising it DOES overfit, unlike RF"},
    {"Concept": "early_stopping", "In one sentence": "Ends training when validation stops improving"},
    {"Concept": "XGBoost 2.0", "In one sentence": "early_stopping_rounds moved to the constructor"},
    {"Concept": "CatBoost", "In one sentence": "Handles a text column itself with cat_features"},
]))