Regularisation: Ridge, Lasso, Elastic Net

import pandas as pd
import numpy as np

w = np.array([3.0, -4.0, 0.5])
print(f"weights: {w}")
print(f"  L2 penalty (sum of squares)    : {np.sum(w**2):.4f}")
print(f"  L1 penalty (absolute values)   : {np.sum(np.abs(w)):.4f}")

show(pd.DataFrame([
    {"Model": "Ridge", "Penalty": "L2: alpha * sum(w^2)",
     "Effect on the weights": "SHRINKS all of them, never to zero",
     "When": "When every feature is a little useful"},
    {"Model": "Lasso", "Penalty": "L1: alpha * sum(|w|)",
     "Effect on the weights": "Pushes some EXACTLY TO ZERO",
     "When": "When you need feature selection"},
    {"Model": "Elastic Net", "Penalty": "A mix of L1 and L2",
     "Effect on the weights": "In between the two",
     "When": "Many features that are related to each other"},
]))

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[num], df["narx_usd"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42)

print("Ridge, WITH and WITHOUT scaling, at different alphas:")
print(f"{'alpha':>8} {'no scaling':>14} {'scaling':>13} {'gap':>9}")
print("-" * 48)
for a in [1, 10, 100, 1000, 10000]:
    y1 = cross_val_score(Ridge(alpha=a), X, y, cv=5, scoring="r2").mean()
    y2 = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", Ridge(alpha=a))]),
                         X, y, cv=5, scoring="r2").mean()
    print(f"{a:>8} {y1:>14.4f} {y2:>13.4f} {y2-y1:>+9.4f}")

print(f"{'alpha':>10} {'CV R2':>9} {'±':>7}  {'coefficients (scaled)'}")
print("-" * 92)
for a in [0, 0.1, 1, 10, 100, 1000]:
    m = LinearRegression() if a == 0 else Ridge(alpha=a)
    pipe = Pipeline([("sc", StandardScaler()), ("m", m)])
    b = cross_val_score(pipe, X, y, cv=5, scoring="r2")
    pipe.fit(X_tr, y_tr)
    k = pipe.named_steps["m"].coef_
    name = "0 (plain)" if a == 0 else str(a)
    print(f"{name:>10} {b.mean():>9.4f} {b.std():>7.4f}  {np.round(k, 1)}")

result = {"Plain (LinearRegression)": [], "Ridge (alpha=10)": [], "Ridge (alpha=100)": []}
for rs in [0, 1, 7, 13, 42, 99, 123, 2026]:
    Xa, _, ya, _ = train_test_split(X, y, test_size=0.3, random_state=rs)
    for name, m in [("Plain (LinearRegression)", LinearRegression()),
                    ("Ridge (alpha=10)", Ridge(alpha=10)),
                    ("Ridge (alpha=100)", Ridge(alpha=100))]:
        q = Pipeline([("sc", StandardScaler()), ("m", m)]).fit(Xa, ya)
        result[name].append(q.named_steps["m"].coef_[:2])    # xonalar and maydon_m2

rows = []
for name, lst in result.items():
    arr = np.array(lst)
    rows.append({
        "Model": name,
        "xonalar: lowest..highest": f"{arr[:,0].min():.1f} .. {arr[:,0].max():.1f}",
        "xonalar spread": round(float(arr[:, 0].std()), 2),
        "maydon_m2 spread": round(float(arr[:, 1].std()), 2),
    })
show(pd.DataFrame(rows))

from sklearn.preprocessing import PolynomialFeatures

# back to the Chapter 15 problem: the polynomial created 83 features
pf = PolynomialFeatures(3, include_bias=False)
print(f"Features: {len(num)} -> {pf.fit_transform(X[:1]).shape[1]}")
print()
print(f"{'alpha':>8} {'CV R2':>9} {'coef equal to 0':>17} {'features kept':>18}")
print("-" * 58)
for a in [0.001, 0.01, 0.1, 1, 10]:
    pipe = Pipeline([("p", pf), ("sc", StandardScaler()),
                     ("m", Lasso(alpha=a, max_iter=50000))])
    b = cross_val_score(pipe, X, y, cv=5, scoring="r2")
    pipe.fit(X_tr, y_tr)
    k = pipe.named_steps["m"].coef_
    zero = int((k == 0).sum())
    print(f"{a:>8} {b.mean():>9.4f} {zero:>13} / {len(k)} {len(k)-zero:>18}")

pipe_l = Pipeline([("p", PolynomialFeatures(3, include_bias=False)),
                   ("sc", StandardScaler()), ("m", Lasso(alpha=1, max_iter=50000))]).fit(X_tr, y_tr)
k = pipe_l.named_steps["m"].coef_
names = pipe_l.named_steps["p"].get_feature_names_out(num)
alive = pd.DataFrame({"feature": names, "coefficient": k.round(3)})
alive = alive[alive.coefficient != 0].sort_values("coefficient", key=abs, ascending=False)
print(f"Features Lasso (alpha=1) kept: {len(alive)} of {len(k)}:")
show(alive.head(12))

print(f"{'model':<34} {'CV R2':>9} {'±':>7}")
print("-" * 52)
for name, m in [
    ("LinearRegression (no penalty)", LinearRegression()),
    ("Ridge (alpha=10)", Ridge(alpha=10)),
    ("Lasso (alpha=1)", Lasso(alpha=1, max_iter=50000)),
    ("ElasticNet (l1_ratio=0.5)", ElasticNet(alpha=1, l1_ratio=0.5, max_iter=50000)),
    ("ElasticNet (l1_ratio=0.9)", ElasticNet(alpha=1, l1_ratio=0.9, max_iter=50000)),
]:
    b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), X, y, cv=5, scoring="r2")
    print(f"{name:<34} {b.mean():>9.4f} {b.std():>7.4f}")

r = np.random.default_rng(42)
noise = pd.DataFrame(r.normal(size=(len(X), 60)),
                     columns=[f"noise_{i}" for i in range(60)], index=X.index)
X_noisy = pd.concat([X, noise], axis=1)
print(f"Features: {X.shape[1]} -> {X_noisy.shape[1]} (60 of them USELESS)")
print()
print(f"{'model':<34} {'CV R2':>9} {'±':>7}")
print("-" * 52)
for name, m in [
    ("LinearRegression (no penalty)", LinearRegression()),
    ("Ridge (alpha=10)", Ridge(alpha=10)),
    ("Lasso (alpha=1)", Lasso(alpha=1, max_iter=50000)),
]:
    b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), X_noisy, y, cv=5, scoring="r2")
    print(f"{name:<34} {b.mean():>9.4f} {b.std():>7.4f}")

print("How Lasso cleans out the noise, depending on alpha:")
print(f"{'alpha':>8} {'CV R2':>9} {'real kept':>18} {'noise kept':>18}")
print("-" * 58)
for a in [0.1, 1, 5, 20, 50]:
    q = Pipeline([("sc", StandardScaler()), ("m", Lasso(alpha=a, max_iter=50000))])
    b = cross_val_score(q, X_noisy, y, cv=5, scoring="r2").mean()
    q.fit(X_noisy, y)
    k_ = q.named_steps["m"].coef_
    real = sum(1 for n, v in zip(X_noisy.columns, k_) if v != 0 and not n.startswith("noise"))
    nz = sum(1 for n, v in zip(X_noisy.columns, k_) if v != 0 and n.startswith("noise"))
    print(f"{a:>8} {b:>9.4f} {real:>14} / 6 {nz:>14} / 60")

from sklearn.linear_model import RidgeCV, LassoCV

alphas = np.logspace(-3, 3, 40)
rcv = Pipeline([("sc", StandardScaler()), ("m", RidgeCV(alphas=alphas, cv=5))]).fit(X_tr, y_tr)
lcv = Pipeline([("sc", StandardScaler()), ("m", LassoCV(alphas=alphas, cv=5, max_iter=50000, random_state=42))]).fit(X_tr, y_tr)

print(f"alpha chosen by RidgeCV : {rcv.named_steps['m'].alpha_:.4f}")
print(f"alpha chosen by LassoCV : {lcv.named_steps['m'].alpha_:.4f}")
print()
from sklearn.metrics import r2_score
print(f"RidgeCV test R2: {r2_score(y_te, rcv.predict(X_te)):.4f}")
print(f"LassoCV test R2: {r2_score(y_te, lcv.predict(X_te)):.4f}")

from sklearn.linear_model import LogisticRegression

yc = df["ijaraga_berildi_30k"]
print("LogisticRegression's C parameter (C = 1/alpha):")
print(f"{'C':>8} {'penalty':>14} {'CV F1':>9} {'coef L2 norm':>17}")
print("-" * 52)
for C in [0.01, 0.1, 1.0, 100.0]:
    q = Pipeline([("sc", StandardScaler()), ("m", LogisticRegression(C=C, max_iter=3000))])
    b = cross_val_score(q, X, yc, cv=5, scoring="f1")
    q.fit(X, yc)
    norm = np.linalg.norm(q.named_steps["m"].coef_)
    strength = "very strong" if C <= 0.01 else ("strong" if C < 1 else ("default" if C == 1 else "almost none"))
    print(f"{C:>8} {strength:>14} {b.mean():>9.4f} {norm:>17.4f}")

show(pd.DataFrame([
    {"Situation": "Few features, all useful", "Choice": "LinearRegression or Ridge (small alpha)"},
    {"Situation": "Many features, some useless", "Choice": "Lasso: it selects"},
    {"Situation": "Many features AND related to each other", "Choice": "ElasticNet"},
    {"Situation": "The coefficients must be stable", "Choice": "Ridge"},
    {"Situation": "You need to know which features matter", "Choice": "Lasso (or SHAP, Chapter 26)"},
    {"Situation": "More features than rows (m > n)", "Choice": "Ridge or Lasso: REQUIRED"},
]))

show(pd.DataFrame([
    {"Concept": "Regularisation", "In one sentence": "Cost = MSE + alpha * a penalty on the weights"},
    {"Concept": "Ridge (L2)", "In one sentence": "Shrinks all of them, never to zero"},
    {"Concept": "Lasso (L1)", "In one sentence": "Pushes some EXACTLY to zero: selects features"},
    {"Concept": "Why L1 reaches zero", "In one sentence": "The derivative of |w| doesn't shrink near zero (diamond vs circle)"},
    {"Concept": "ElasticNet", "In one sentence": "A mix of L1 and L2, for related features"},
    {"Concept": "Scaling", "In one sentence": "MANDATORY in regularisation, so the penalty is fair"},
    {"Concept": "C and alpha", "In one sentence": "C = 1/alpha, inverted; large C = weak penalty"},
    {"Concept": "Fixing the instability", "In one sentence": "Ridge shrinks the spread of the coefficients"},
]))