ROC, PR, threshold and imbalanced data

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
X, y = df[num + ["tuman"]], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

pipe = Pipeline([
    ("prep", ColumnTransformer([
        ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                          ("scale", StandardScaler())]), num),
        ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
    ])),
    ("model", LogisticRegression(max_iter=2000)),
]).fit(X_tr, y_tr)

proba = pipe.predict_proba(X_te)[:, 1]
compare = pd.DataFrame({"probability": proba.round(3), "predict()": pipe.predict(X_te),
                        "truth": y_te.values}).head(8)
show(compare)

from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score

print(f"{'threshold':>9} {'precision':>10} {'recall':>8} {'F1':>7} {'accuracy':>9} {'yes count':>10}")
print("-" * 57)
for t in [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]:
    b = (proba >= t).astype(int)
    print(f"{t:>9} {precision_score(y_te, b, zero_division=0):>10.4f} "
          f"{recall_score(y_te, b, zero_division=0):>8.4f} "
          f"{f1_score(y_te, b, zero_division=0):>7.4f} "
          f"{accuracy_score(y_te, b):>9.4f} {int(b.sum()):>10}")

import numpy as np
needed = None
for t in np.arange(0.30, 0.96, 0.01):
    b = (proba >= t).astype(int)
    if b.sum() > 0 and precision_score(y_te, b, zero_division=0) >= 0.85:
        needed = t
        break
b = (proba >= needed).astype(int)
print(f"Lowest threshold with precision >= 0.85: {needed:.2f}")
print(f"  precision: {precision_score(y_te, b):.4f}")
print(f"  recall   : {recall_score(y_te, b):.4f}")
print(f"  times the model said yes: {int(b.sum())} (it was {int((proba >= 0.5).sum())} at 0.5)")

from sklearn.metrics import precision_recall_curve
prec, rec, thr = precision_recall_curve(y_te, proba)
f1_curve = 2 * prec * rec / np.maximum(prec + rec, 1e-12)
best = int(np.nanargmax(f1_curve[:-1]))
print(f"Threshold with the best F1: {thr[best]:.3f}  ->  F1 = {f1_curve[best]:.4f}")
print(f"Default 0.5               :          F1 = {f1_score(y_te, pipe.predict(X_te)):.4f}")

np.random.seed(42)
fast = y_te[y_te == 1].index
slow = y_te[y_te == 0].index
sample = list(slow) + list(np.random.choice(fast, size=max(1, int(len(slow) * 0.02)), replace=False))

y_imb = y_te.loc[sample]
p_imb = pd.Series(proba, index=y_te.index).loc[sample].values
print(f"New set: {len(y_imb)} rows, share of 'went fast' {y_imb.mean():.2%}")

from sklearn.metrics import roc_auc_score, average_precision_score

auc = roc_auc_score(y_imb, p_imb)
ap = average_precision_score(y_imb, p_imb)

print(f"ROC-AUC          : {auc:.4f}")
print(f"Average Precision: {ap:.4f}")
print()
print("Baseline for a random model:")
print(f"  ROC-AUC = 0.5000      (ALWAYS, whatever the imbalance)")
print(f"  AP      = {y_imb.mean():.4f}      (= share of the positive class)")

b_imb = (p_imb >= 0.5).astype(int)
from sklearn.metrics import confusion_matrix
tn, fp, fn, tp = confusion_matrix(y_imb, b_imb).ravel()
print(f"TN={tn}  FP={fp}  FN={fn}  TP={tp}")
print()
print("The second axis of ROC is FPR:")
print(f"  FPR = FP / (FP + TN) = {fp} / ({fp} + {tn}) = {fp/(fp+tn):.4f}")
print()
print("TN is huge (many negatives) -> huge denominator -> FPR is always small.")
print("So the ROC curve hugs the top-left corner and LOOKS BEAUTIFUL.")
print()
print("The PR curve does NOT use TN at all:")
print(f"  precision = TP / (TP + FP) = {tp} / ({tp} + {fp}) = {tp/(tp+fp) if tp+fp else 0:.4f}")
print(f"  recall    = TP / (TP + FN) = {tp} / ({tp} + {fn}) = {tp/(tp+fn) if tp+fn else 0:.4f}")

import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve

fpr, tpr, _ = roc_curve(y_imb, p_imb)
p2, r2, _ = precision_recall_curve(y_imb, p_imb)

figure, axes = plt.subplots(1, 2, figsize=(13, 5))
axes[0].plot(fpr, tpr, "b-", lw=2, label=f"AUC = {auc:.3f}")
axes[0].plot([0, 1], [0, 1], "k--", alpha=.5, label="Random (0.5)")
axes[0].set_xlabel("False Positive Rate"); axes[0].set_ylabel("True Positive Rate")
axes[0].set_title("ROC — LOOKS beautiful", color="darkorange", fontweight="bold")
axes[0].legend(); axes[0].grid(alpha=.3)

axes[1].plot(r2, p2, "r-", lw=2, label=f"AP = {ap:.3f}")
axes[1].axhline(y_imb.mean(), color="k", ls="--", alpha=.5, label=f"Random ({y_imb.mean():.3f})")
axes[1].set_xlabel("Recall"); axes[1].set_ylabel("Precision")
axes[1].set_title("PR — shows the TRUTH", color="green", fontweight="bold")
axes[1].legend(); axes[1].grid(alpha=.3)
fig("roc_pr.png", caption="Same model, same data. The chart on the left is lying to you.")

idx = list(set(sample))
X_imb = X_te.loc[idx]
y_imb2 = y_te.loc[idx]
X_itr, X_ite, y_itr, y_ite = train_test_split(X_imb, y_imb2, test_size=0.4,
                                              random_state=42, stratify=y_imb2)

def small_pipe(**kw):
    return Pipeline([
        ("prep", ColumnTransformer([
            ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                              ("scale", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", LogisticRegression(max_iter=3000, **kw)),
    ])

plain = small_pipe().fit(X_itr, y_itr)
balanced = small_pipe(class_weight="balanced").fit(X_itr, y_itr)

rows = []
for name, m in [("Plain", plain), ("class_weight=balanced", balanced)]:
    b = m.predict(X_ite)
    rows.append({"Model": name,
                 "Accuracy": round(accuracy_score(y_ite, b), 4),
                 "Precision": round(precision_score(y_ite, b, zero_division=0), 4),
                 "Recall": round(recall_score(y_ite, b, zero_division=0), 4),
                 "F1": round(f1_score(y_ite, b, zero_division=0), 4)})
show(pd.DataFrame(rows))

from sklearn.metrics import classification_report
print(classification_report(y_ite, plain.predict(X_ite),
                            target_names=["slow", "fast"], digits=4, zero_division=0))

show(pd.DataFrame([
    {"Concept": "predict() has a 0.5 inside", "In one sentence": "It is a default value, not a mathematical truth"},
    {"Concept": "Moving the threshold", "In one sentence": "Choose precision vs recall without retraining"},
    {"Concept": "Raise the threshold", "In one sentence": "Precision goes up, recall goes down, every time"},
    {"Concept": "ROC-AUC", "In one sentence": "Fine on balanced classes; LIES on imbalanced ones (because of TN)"},
    {"Concept": "PR-AUC (AP)", "In one sentence": "Does not use TN, so it tells the truth on imbalanced data"},
    {"Concept": "class_weight=balanced", "In one sentence": "Not a fix, a tilt of the scales"},
    {"Concept": "macro vs weighted", "In one sentence": "On imbalanced data, read macro"},
]))