Logistic regression

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[num], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

linear = LinearRegression().fit(X_tr, y_tr)
b = linear.predict(X_te)

print(f"The target is only 0 or 1: {sorted(y.unique().tolist())}")
print()
print(f"The linear model's predictions:")
print(f"  smallest : {b.min():.4f}")
print(f"  largest  : {b.max():.4f}")
print()
print(f"Predictions BELOW 0: {int((b < 0).sum())}")
print(f"Predictions ABOVE 1: {int((b > 1).sum())}")

import matplotlib.pyplot as plt

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

print(f"{'z':>8} {'sigmoid(z)':>12}   note")
print("-" * 46)
for z in [-6, -2, -1, 0, 1, 2, 6]:
    s = sigmoid(z)
    note = "sticks to 0" if s < 0.05 else ("sticks to 1" if s > 0.95 else
           ("EXACTLY in the middle" if z == 0 else "middle zone"))
    print(f"{z:>8} {s:>12.6f}   {note}")

z = np.linspace(-8, 8, 300)
figure, axes = plt.subplots(1, 2, figsize=(13.5, 4.6))

axes[0].plot(z, sigmoid(z), "b-", lw=2.5)
axes[0].axhline(0.5, color="gray", ls=":", lw=1)
axes[0].axvline(0, color="gray", ls=":", lw=1)
axes[0].axhline(0, color="red", ls="--", lw=1, alpha=.6)
axes[0].axhline(1, color="red", ls="--", lw=1, alpha=.6)
axes[0].set_xlabel("z = x @ w + b"); axes[0].set_ylabel("sigmoid(z) = probability")
axes[0].set_title("The sigmoid STAYS between 0 and 1", fontweight="bold")
axes[0].grid(alpha=.3)

axes[1].scatter(X_te.maydon_m2, y_te, alpha=.3, s=25, label="truth (0/1)", color="#5f6368")
order = np.argsort(X_te.maydon_m2.to_numpy())
axes[1].plot(X_te.maydon_m2.to_numpy()[order], b[order], "r-", lw=2, label="linear (leaves the bounds)")
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
log_one = Pipeline([("sc", StandardScaler()), ("m", LogisticRegression())]).fit(X_tr[["maydon_m2"]], y_tr)
p_one = log_one.predict_proba(X_te[["maydon_m2"]])[:, 1]
axes[1].plot(X_te.maydon_m2.to_numpy()[order], p_one[order], "g-", lw=2.5, label="logistic (inside 0-1)")
axes[1].axhline(0, color="k", ls=":", lw=.8); axes[1].axhline(1, color="k", ls=":", lw=.8)
axes[1].set_xlabel("maydon_m2"); axes[1].set_ylabel("probability")
axes[1].set_title("The two models side by side", fontweight="bold")
axes[1].legend(fontsize=8.5); axes[1].grid(alpha=.3)
fig("sigmoid.png", caption="Left: the sigmoid function. Right: the red line goes below 0 and above 1, while the green line stays within the bounds.")

def log_loss_one(p, y):
    p = np.clip(p, 1e-15, 1 - 1e-15)
    return -(y * np.log(p) + (1 - y) * np.log(1 - p))

print("When the truth is y=1 and the model gives various probabilities:")
print(f"{'predicted p':>12} {'MSE penalty':>13} {'log loss penalty':>18}")
print("-" * 46)
for p in [0.9, 0.7, 0.5, 0.3, 0.1, 0.01]:
    print(f"{p:>12} {(p - 1)**2:>13.4f} {log_loss_one(p, 1):>18.4f}")

pipe = Pipeline([("sc", StandardScaler()), ("m", LogisticRegression(max_iter=2000))]).fit(X_tr, y_tr)
k = pipe.named_steps["m"].coef_[0]

table = pd.DataFrame({
    "feature": num,
    "coefficient (log-odds)": k.round(4),
    "odds ratio (e^k)": np.exp(k).round(4),
}).assign(meaning=lambda d: [
    f"up 1 std -> odds {'up' if v > 1 else 'down'} {abs(v-1)*100:.1f}%"
    for v in d["odds ratio (e^k)"]
])
show(table.sort_values("coefficient (log-odds)", key=abs, ascending=False))

print("How odds and probability are related:")
print(f"{'probability':>12} {'odds':>10} {'log-odds':>11}")
print("-" * 36)
for p in [0.1, 0.25, 0.5, 0.75, 0.9]:
    odds = p / (1 - p)
    print(f"{p:>12} {odds:>10.3f} {np.log(odds):>11.3f}")
print()
print("At p = 0.5 the odds = 1 and the log-odds = 0 -> the centre of the sigmoid")

from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import cross_val_score

pg = pd.read_csv(DATA / "penguins.csv")
pnum = ["bill_length_mm", "bill_depth_mm", "flipper_length_mm", "body_mass_g"]
pcat = ["island", "sex"]
Xp, yp = pg[pnum + pcat], pg["species"]

def p_pipe(**kw):
    return Pipeline([
        ("t", ColumnTransformer([
            ("num", Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), pnum),
            ("cat", Pipeline([("i", SimpleImputer(strategy="most_frequent")),
                              ("o", OneHotEncoder(handle_unknown="ignore"))]), pcat),
        ])),
        ("m", LogisticRegression(max_iter=3000, **kw)),
    ])

for name, kw in [("multinomial (softmax)", {}), ("one-vs-rest", {"multi_class": "ovr"})]:
    try:
        bb = cross_val_score(p_pipe(**kw), Xp, yp, cv=5, scoring="f1_macro")
        print(f"  {name:24s} macro F1 = {bb.mean():.4f} ±{bb.std():.4f}")
    except Exception as e:
        print(f"  {name:24s} {type(e).__name__}")

mp = p_pipe().fit(Xp, yp)
print("Classes:", mp.named_steps["m"].classes_.tolist())
print("Coefficient shape:", mp.named_steps["m"].coef_.shape,
      "-> a separate set of coefficients for EACH CLASS")
print()
e = mp.predict_proba(Xp.head(4))
print("Probabilities for the first 4 penguins (each row sums to 1):")
print(pd.DataFrame(e.round(4), columns=mp.named_steps["m"].classes_).to_string(index=False))
print()
print("Row sums:", e.sum(axis=1).round(6).tolist())

from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score

proba = pipe.predict_proba(X_te)[:, 1]
print(f"{'threshold':>9} {'precision':>11} {'recall':>8} {'F1':>7}")
print("-" * 38)
for t in [0.3, 0.5, 0.7]:
    bb = (proba >= t).astype(int)
    print(f"{t:>9} {precision_score(y_te, bb, zero_division=0):>11.4f} "
          f"{recall_score(y_te, bb, zero_division=0):>8.4f} {f1_score(y_te, bb, zero_division=0):>7.4f}")
print()
print("That's why logistic regression is handy in practice: it gives a probability,")
print("and you choose the threshold to match the business need (Chapter 9).")

show(pd.DataFrame([
    {"Strength": "Gives a probability", "Why it matters": "You can move the threshold (Chapter 9)"},
    {"Strength": "Explainable", "Why it matters": "The odds ratio: banks and medicine demand it"},
    {"Strength": "Fast and cheap", "Why it matters": "Works even with millions of rows"},
    {"Strength": "Little overfitting", "Why it matters": "A simple model, few parameters"},
    {"Weakness": "ONLY a linear boundary", "Why it matters": "Can't see complex patterns -> a tree (Chapter 18)"},
    {"Weakness": "Needs scaling", "Why it matters": "For the gradient and regularisation"},
    {"Weakness": "Doesn't create features itself", "Why it matters": "Needs manual feature engineering (Chapter 23)"},
]))

from sklearn.tree import DecisionTreeClassifier

# an artificial "ring" pattern: no straight line can separate it
r = np.random.default_rng(42)
n = 600
angle = r.uniform(0, 2 * np.pi, n)
radius = np.where(r.random(n) < 0.5, r.normal(1, .25, n), r.normal(3, .3, n))
Xh = np.c_[radius * np.cos(angle), radius * np.sin(angle)]
yh = (radius > 2).astype(int)

for name, m in [("Logistic Regression", LogisticRegression()),
                ("Decision Tree", DecisionTreeClassifier(max_depth=6, random_state=42))]:
    bb = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), Xh, yh, cv=5)
    print(f"  {name:22s} accuracy = {bb.mean():.4f} ±{bb.std():.4f}")
print()
print("On ring-shaped data a linear boundary doesn't work.")
print("In Chapter 18 we see how a tree catches a pattern like this.")

show(pd.DataFrame([
    {"Concept": "The problem", "In one sentence": "A linear model goes outside 0-1"},
    {"Concept": "Sigmoid", "In one sentence": "Squeezes any number into [0,1]"},
    {"Concept": "The model", "In one sentence": "sigmoid(x @ w + b): the Chapter 12 operation + one function"},
    {"Concept": "Log loss", "In one sentence": "Punishes confident mistakes hard; makes the bowl convex"},
    {"Concept": "Coefficient", "In one sentence": "log-odds; exp(k) = the odds multiplier"},
    {"Concept": "Multiclass", "In one sentence": "Softmax: a probability per class, summing to 1"},
    {"Concept": "Its limitation", "In one sentence": "Only a linear boundary: it can't separate a ring"},
]))