Error analysis: what to do when the result is bad

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score, accuracy_score

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
X, y = df[num + ["tuman"]], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

pipe = Pipeline([
    ("prep", ColumnTransformer([
        ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                          ("scale", StandardScaler())]), num),
        ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
    ])),
    ("model", LogisticRegression(max_iter=2000)),
]).fit(X_tr, y_tr)

pred = pipe.predict(X_te)
proba = pipe.predict_proba(X_te)[:, 1]
print(f"Accuracy: {accuracy_score(y_te, pred):.4f}")
print(f"F1      : {f1_score(y_te, pred):.4f}")
print()
print("Now we look NOT AT THESE NUMBERS, BUT AT THE MISTAKES.")

analysis = X_te.copy()
analysis["truth"] = y_te
analysis["pred"] = pred
analysis["confidence"] = proba.round(3)
analysis["wrong"] = (analysis.truth != analysis.pred).astype(int)

print(f"Total mistakes: {analysis.wrong.sum()} / {len(analysis)} ({analysis.wrong.mean():.1%})")
print()
print("The 6 rows the model got wrong MOST CONFIDENTLY:")
worst = analysis[analysis.wrong == 1].assign(
    error=lambda d: (d.confidence - d.truth).abs()).nlargest(6, "error")
show(worst[["tuman", "xonalar", "maydon_m2", "metro_km", "mebel", "truth", "pred", "confidence"]])

def slice_by(col, label=None):
    g = analysis.groupby(col, observed=True).agg(total=("wrong", "size"), wrong=("wrong", "sum"))
    g["error_rate"] = (g.wrong / g.total).round(3)
    g = g[g.total >= 10].sort_values("error_rate", ascending=False)
    print(f"--- by {label or col} ---")
    print(g.to_string())
    print()

slice_by("tuman")
slice_by("xonalar")

analysis["area_bin"] = pd.cut(analysis.maydon_m2, bins=[0, 40, 60, 80, 300],
                              labels=["<40", "40-60", "60-80", "80+"])
analysis["metro_bin"] = pd.cut(analysis.metro_km, bins=[0, 1, 2.5, 5, 20],
                               labels=["<1km", "1-2.5km", "2.5-5km", "5km+"])
slice_by("area_bin", "area")
slice_by("metro_bin", "distance to metro")

fp = analysis[(analysis.truth == 0) & (analysis.pred == 1)]
fn = analysis[(analysis.truth == 1) & (analysis.pred == 0)]

print(f"FP (said yes, was no): {len(fp)}")
print(f"FN (said no, was yes): {len(fn)}")
print()
rows = []
for c in ["maydon_m2", "metro_km", "xonalar", "mebel", "qavat"]:
    rows.append({"Column": c,
                 "FP mean": round(fp[c].mean(), 2),
                 "FN mean": round(fn[c].mean(), 2),
                 "Correct mean": round(analysis[analysis.wrong == 0][c].mean(), 2)})
show(pd.DataFrame(rows))

import matplotlib.pyplot as plt

figure, axes = plt.subplots(1, 2, figsize=(13, 4.5))

axes[0].hist([analysis[analysis.truth == 0].confidence, analysis[analysis.truth == 1].confidence],
             bins=20, label=["actually slow", "actually fast"], color=["#c5221f", "#137333"],
             alpha=.75, stacked=False)
axes[0].axvline(0.5, color="k", ls="--", label="threshold 0.5")
axes[0].set_xlabel("Model confidence"); axes[0].set_ylabel("Number of rows")
axes[0].set_title("Confidence distribution"); axes[0].legend(); axes[0].grid(alpha=.3)

bins = pd.cut(analysis.confidence, bins=np.arange(0, 1.01, 0.1))
error_rate = analysis.groupby(bins, observed=True).wrong.mean()
axes[1].bar(range(len(error_rate)), error_rate.values, color="#1a73e8", alpha=.8)
axes[1].set_xticks(range(len(error_rate)))
axes[1].set_xticklabels([f"{i/10:.1f}" for i in range(len(error_rate))], rotation=45)
axes[1].set_xlabel("Confidence bin"); axes[1].set_ylabel("Error rate")
axes[1].set_title("At which confidence does it go wrong most"); axes[1].grid(alpha=.3, axis="y")

fig("xato_tahlili.png", caption="Left: how well the two classes separate by confidence. Right: in which confidence bin the mistakes gather.")

from sklearn.model_selection import learning_curve

sizes, train_scores, val_scores = learning_curve(
    pipe, X, y, train_sizes=np.linspace(0.15, 1.0, 8), cv=5,
    scoring="f1", shuffle=True, random_state=42)

table = pd.DataFrame({
    "train size": sizes,
    "train F1": train_scores.mean(axis=1).round(4),
    "val F1": val_scores.mean(axis=1).round(4),
    "GAP": (train_scores.mean(axis=1) - val_scores.mean(axis=1)).round(4),
})
show(table)

figure2, ax = plt.subplots(figsize=(9, 5))
ax.plot(sizes, train_scores.mean(axis=1), "b-o", lw=2, label="Train F1")
ax.plot(sizes, val_scores.mean(axis=1), "r-o", lw=2, label="Validation F1")
ax.fill_between(sizes, train_scores.mean(axis=1), val_scores.mean(axis=1), alpha=.15, color="red")
ax.set_xlabel("Number of training rows"); ax.set_ylabel("F1")
ax.set_title("Learning curve: would more data help?")
ax.legend(); ax.grid(alpha=.3)
fig("learning_curve.png", caption="When the two lines have come together and flattened, more data won't help.")

show(pd.DataFrame([
    {"What you see": "The two lines are CLOSE and both LOW", "Diagnosis": "Underfitting (high bias)",
     "Action": "A stronger model, more features. More DATA WON'T HELP"},
    {"What you see": "Train HIGH, validation LOW, a WIDE gap", "Diagnosis": "Overfitting (high variance)",
     "Action": "More data, regularisation, a simpler model"},
    {"What you see": "Validation is still RISING", "Diagnosis": "Too little data",
     "Action": "Collecting more data HELPS"},
    {"What you see": "Validation has FLATTENED", "Diagnosis": "Enough data",
     "Action": "You need a new column or another model"},
]))

last_gap = train_scores.mean(axis=1)[-1] - val_scores.mean(axis=1)[-1]
last_rise = val_scores.mean(axis=1)[-1] - val_scores.mean(axis=1)[-3]
print(f"GAP at the last point         : {last_gap:.4f}")
print(f"Latest rise in validation     : {last_rise:+.4f}")
print()
if last_gap > 0.08:
    print("DIAGNOSIS: signs of overfitting -> more data or regularisation")
elif last_rise > 0.01:
    print("DIAGNOSIS: validation is still rising -> more data HELPS")
else:
    print("DIAGNOSIS: enough data, the lines are close -> you need a NEW COLUMN or another model")

show(pd.DataFrame([
    {"Evidence": "The mistakes cluster in one group", "Action": "A new column for that group, or a separate model"},
    {"Evidence": "The FP and FN means differ", "Action": "Move the threshold (Chapter 9)"},
    {"Evidence": "The model is confidently wrong", "Action": "Check the labels, add the important column"},
    {"Evidence": "Learning curve: validation is rising", "Action": "Collect more data"},
    {"Evidence": "Learning curve: the GAP is wide", "Action": "Regularisation or a simpler model"},
    {"Evidence": "Learning curve: both low and flat", "Action": "A stronger model (tree, boosting)"},
    {"Evidence": "No evidence, everything is normal", "Action": "Stop. The model is good enough (Chapter 40)"},
]))

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

def with_pipe(model):
    return Pipeline([
        ("prep", ColumnTransformer([
            ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                              ("scale", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", model),
    ])

lr = cross_val_score(with_pipe(LogisticRegression(max_iter=2000)), X, y, cv=5, scoring="f1")
rf = cross_val_score(with_pipe(RandomForestClassifier(n_estimators=200, random_state=42)),
                     X, y, cv=5, scoring="f1")
print(f"Logistic Regression: {lr.mean():.4f} ±{lr.std():.4f}")
print(f"Random Forest      : {rf.mean():.4f} ±{rf.std():.4f}")
print(f"Gap                : {rf.mean() - lr.mean():+.4f}  (spread ±{max(lr.std(), rf.std()):.4f})")
print()
print("Verdict:", "Random Forest is reliably better" if rf.mean() - lr.mean() > max(lr.std(), rf.std())
      else "The gap is within the spread: keep the simpler model")

notebook = """
+----------------------------------------------------------------------+
|  ERROR-ANALYSIS NOTEBOOK                                             |
+----------------------------------------------------------------------+
|  Date        :                                                       |
|  Model       :                                                       |
|  Metric      : F1 = ______  (CV: ______ +- ______)                   |
|  Baseline    : ______                                                |
+----------------------------------------------------------------------+
|  1. Which group do the mistakes cluster in?                          |
|     ................................................................ |
|  2. How do FP and FN differ?                                         |
|     ................................................................ |
|  3. Is the model confidently wrong? (mistakes at 0.9+)               |
|     ................................................................ |
|  4. Learning curve diagnosis?                                        |
|     [ ] underfitting  [ ] overfitting  [ ] too little data  [ ] ok   |
+----------------------------------------------------------------------+
|  DECISION (ONLY ONE):                                                |
|     [ ] new feature: ..............................................  |
|     [ ] another model: ............................................  |
|     [ ] more data                                                    |
|     [ ] move the threshold: .......................................  |
|     [ ] stop                                                         |
+----------------------------------------------------------------------+
|  Result: ______  (previous: ______)   Gap: ______                    |
|  Was my hypothesis right?  [ ] yes   [ ] NO <- the most valuable     |
+----------------------------------------------------------------------+
"""
print(notebook)

show(pd.DataFrame([
    {"Step": "1. Read the mistakes", "Tool": "nlargest(confidence), with your own eyes"},
    {"Step": "2. Count by group", "Tool": "groupby(column).wrong.mean()"},
    {"Step": "3. Separate FP and FN", "Tool": "Two kinds of mistake, two different causes"},
    {"Step": "4. Confidence distribution", "Tool": "Where the model is unsure, where it is confidently wrong"},
    {"Step": "5. Learning curve", "Tool": "Do you need more data? A MEASUREMENT, not a guess"},
    {"Step": "6. ONE change", "Tool": "An evidence-based decision, then back to step 9"},
]))