Bagging and Random Forest

import pandas as pd
import numpy as np

r = np.random.default_rng(42)
n_models, n_samples = 100, 2000
truth = r.choice([0, 1], n_samples)

# each "model" is 60% accurate, but its mistakes are INDEPENDENT
independent = np.array([np.where(r.random(n_samples) < 0.60, truth, 1 - truth)
                        for _ in range(n_models)])
vote = (independent.mean(axis=0) > 0.5).astype(int)

print(f"Accuracy of one model          : {(independent[0] == truth).mean():.4f}")
print(f"Average score of the 100 models: {(independent == truth).mean():.4f}")
print(f"The VOTE                       : {(vote == truth).mean():.4f}")

# every model makes the SAME mistakes (fully correlated)
one_pattern = np.where(r.random(n_samples) < 0.60, truth, 1 - truth)
correlated = np.array([one_pattern for _ in range(n_models)])
vote_c = (correlated.mean(axis=0) > 0.5).astype(int)
print(f"When the errors are CORRELATED, the vote: {(vote_c == truth).mean():.4f}")
print()
print("Conclusion: an ensemble only works if the models DIFFER from each other.")

sample = np.arange(10)
for i in range(3):
    bs = np.random.default_rng(i).choice(sample, size=10, replace=True)
    missing = sorted(set(sample) - set(bs))
    print(f"  bootstrap {i+1}: {bs}  |  left out: {missing}")
print()
# in every bootstrap about 37% of the rows are left out
left_out = [len(set(np.arange(1000)) - set(np.random.default_rng(s).choice(1000, 1000, replace=True)))/1000
            for s in range(50)]
print(f"Average share of rows left out: {np.mean(left_out):.1%}")
print("That's a mathematical fact: (1 - 1/n)^n -> 1/e = 0.368")

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[num], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

print(f"{'model':<40} {'CV F1':>9} {'±':>8}")
print("-" * 58)
for name, m in [
    ("One tree (unlimited)", DecisionTreeClassifier(random_state=42)),
    ("One tree (max_depth=6)", DecisionTreeClassifier(max_depth=6, random_state=42)),
    ("Bagging (100 unlimited trees)", BaggingClassifier(
        DecisionTreeClassifier(random_state=42), n_estimators=100, random_state=42)),
]:
    b = cross_val_score(m, X, y, cv=5, scoring="f1")
    print(f"{name:<40} {b.mean():>9.4f} {b.std():>8.4f}")

show(pd.DataFrame([
    {"Step": "1. Bootstrap sample", "Bagging": "YES", "Random Forest": "YES"},
    {"Step": "2. A SUBSET of the columns at each split", "Bagging": "no (all of them)",
     "Random Forest": "YES: max_features"},
]))
print()
print("That single addition makes the trees much more different.")

print(f"{'model':<40} {'CV F1':>9} {'±':>8}")
print("-" * 58)
for name, m in [
    ("Bagging (every column)", BaggingClassifier(
        DecisionTreeClassifier(random_state=42), n_estimators=100, random_state=42)),
    ("Random Forest (max_features='sqrt')", RandomForestClassifier(
        n_estimators=100, random_state=42)),
    ("Random Forest (max_features=None)", RandomForestClassifier(
        n_estimators=100, max_features=None, random_state=42)),
]:
    b = cross_val_score(m, X, y, cv=5, scoring="f1")
    print(f"{name:<40} {b.mean():>9.4f} {b.std():>8.4f}")
print()
print("The third row: with max_features=None, RF actually turns into Bagging.")

from sklearn.metrics import f1_score

print(f"{'random_state':>13} {'one tree':>14} {'Random Forest':>15}")
print("-" * 46)
tree_scores, rf_scores = [], []
for rs in [0, 1, 7, 13, 42, 99, 123, 2026]:
    Xa, Xb, ya, yb = train_test_split(X, y, test_size=0.3, random_state=rs, stratify=y)
    d = DecisionTreeClassifier(random_state=42).fit(Xa, ya)
    rf = RandomForestClassifier(n_estimators=100, random_state=42).fit(Xa, ya)
    df_ = f1_score(yb, d.predict(Xb)); rf_ = f1_score(yb, rf.predict(Xb))
    tree_scores.append(df_); rf_scores.append(rf_)
    print(f"{rs:>13} {df_:>14.4f} {rf_:>15.4f}")
print("-" * 46)
print(f"{'spread':>13} {np.std(tree_scores):>14.4f} {np.std(rf_scores):>15.4f}")

rf_oob = RandomForestClassifier(n_estimators=300, oob_score=True, random_state=42).fit(X_tr, y_tr)
print(f"OOB score (free, no CV needed): {rf_oob.oob_score_:.4f}")
print(f"Accuracy on the test set       : {rf_oob.score(X_te, y_te):.4f}")
cv_acc = cross_val_score(RandomForestClassifier(n_estimators=300, random_state=42), X, y, cv=5).mean()
print(f"5-fold CV accuracy (expensive) : {cv_acc:.4f}")

import time
print(f"{'n_estimators':>13} {'CV F1':>9} {'±':>8} {'time':>9}")
print("-" * 44)
for n in [1, 10, 50, 200, 500]:
    t0 = time.perf_counter()
    b = cross_val_score(RandomForestClassifier(n_estimators=n, random_state=42, n_jobs=1),
                        X, y, cv=5, scoring="f1")
    dt = time.perf_counter() - t0
    print(f"{n:>13} {b.mean():>9.4f} {b.std():>8.4f} {dt:>8.1f}s")

show(pd.DataFrame([
    {"": "Bagging / RF", "How the trees are grown": "IN PARALLEL, independently of each other",
     "What each tree does": "Solves the whole task on its own",
     "What it reduces": "VARIANCE (instability)",
     "Kind of tree": "Deep, unlimited"},
    {"": "Boosting", "How the trees are grown": "IN SEQUENCE, each looks at the previous one's mistakes",
     "What each tree does": "Fixes the previous mistakes",
     "What it reduces": "BIAS (lack of accuracy)",
     "Kind of tree": "Shallow (depth 3-6)"},
]))

from sklearn.inspection import permutation_importance

rf = RandomForestClassifier(n_estimators=300, random_state=42).fit(X_tr, y_tr)
d1 = DecisionTreeClassifier(max_depth=6, random_state=42).fit(X_tr, y_tr)
pi = permutation_importance(rf, X_te, y_te, n_repeats=20, random_state=42, scoring="f1")

show(pd.DataFrame({
    "feature": num,
    "one tree": d1.feature_importances_.round(4),
    "Random Forest": rf.feature_importances_.round(4),
    "RF permutation": pi.importances_mean.round(4),
}).sort_values("RF permutation", ascending=False))

from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

yn = df["narx_usd"]
print(f"{'model':<34} {'CV R2':>9} {'±':>8}")
print("-" * 52)
for name, m in [
    ("LinearRegression (Chapter 15)", Pipeline([("sc", StandardScaler()), ("m", LinearRegression())])),
    ("DecisionTreeRegressor", __import__("sklearn.tree", fromlist=["DecisionTreeRegressor"]).DecisionTreeRegressor(max_depth=8, random_state=42)),
    ("RandomForestRegressor", RandomForestRegressor(n_estimators=200, random_state=42)),
]:
    b = cross_val_score(m, X, yn, cv=5, scoring="r2")
    print(f"{name:<34} {b.mean():>9.4f} {b.std():>8.4f}")

big = pd.DataFrame([{"xonalar": 10, "maydon_m2": 500.0, "qavat": 3,
                     "qavatlar_soni": 5, "metro_km": 1.0, "mebel": 1}])
lin = Pipeline([("sc", StandardScaler()), ("m", LinearRegression())]).fit(X, yn)
rfr = RandomForestRegressor(n_estimators=200, random_state=42).fit(X, yn)
print(f"Highest price in training : {yn.max()}")
print(f"Prediction for a 500 m2 flat:")
print(f"  LinearRegression : {lin.predict(big)[0]:.0f}   <- went OUTSIDE the range")
print(f"  RandomForest     : {rfr.predict(big)[0]:.0f}   <- STAYED in the range")

show(pd.DataFrame([
    {"Concept": "The ensemble condition", "In one sentence": "The models must DIFFER from each other"},
    {"Concept": "Bootstrap", "In one sentence": "Sampling with replacement; ~37% of rows are left out"},
    {"Concept": "Bagging", "In one sentence": "A different sample for every tree"},
    {"Concept": "Random Forest", "In one sentence": "Bagging + a subset of the columns at each split"},
    {"Concept": "What it fixes", "In one sentence": "Variance: the instability of a single tree"},
    {"Concept": "OOB score", "In one sentence": "Free validation, 5 times cheaper than CV"},
    {"Concept": "n_estimators", "In one sentence": "Raising it does NOT overfit: it only costs time"},
    {"Concept": "Bagging vs Boosting", "In one sentence": "Parallel+independent vs sequential+dependent"},
    {"Concept": "The regression limitation", "In one sentence": "A tree doesn't extrapolate"},
]))