Confusion matrix, precision, recall, F1

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression

df = pd.read_csv(DATA / "uy_ijara.csv")

num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat = ["tuman"]
X = df[num + cat]
y = df["ijaraga_berildi_30k"]

X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

pipe = Pipeline([
    ("prep", ColumnTransformer([
        ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                          ("scale", StandardScaler())]), num),
        ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
    ])),
    ("model", LogisticRegression(max_iter=2000)),
]).fit(X_tr, y_tr)

pred = pipe.predict(X_te)
print(f"test set : {len(y_te)} flats")
print(f"accuracy : {(pred == y_te).mean():.4f}")

from sklearn.metrics import confusion_matrix

cm = confusion_matrix(y_te, pred)
print("Confusion Matrix:")
print(cm)
print()
tn, fp, fn, tp = cm.ravel()
print(f"TN={tn}  FP={fp}  FN={fn}  TP={tp}")

show(pd.DataFrame([
    {"Cell": "TN", "Count": tn, "Model said": "goes slow", "Reality": "went slow",
     "For the business": "Correct. No ad spend, no loss"},
    {"Cell": "FP", "Count": fp, "Model said": "goes FAST", "Reality": "went slow",
     "For the business": "Ad money BURNED on an empty listing"},
    {"Cell": "FN", "Count": fn, "Model said": "goes slow", "Reality": "went FAST",
     "For the business": "Opportunity MISSED, we ignored it"},
    {"Cell": "TP", "Count": tp, "Model said": "goes FAST", "Reality": "went fast",
     "For the business": "Correct. The ad landed where it should"},
]))

show(pd.DataFrame([
    {"Metric": "Precision", "Formula": "TP / (TP + FP)",
     "Its question": "When the model says 'yes', how often is it right?",
     "Who asks": "The person spending the money"},
    {"Metric": "Recall", "Formula": "TP / (TP + FN)",
     "Its question": "Of all the real 'yes' cases, how many did it find?",
     "Who asks": "The person who cannot afford to miss an opportunity"},
]))

from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score

p = precision_score(y_te, pred)
r = recall_score(y_te, pred)
f1 = f1_score(y_te, pred)

print(f"Accuracy : {accuracy_score(y_te, pred):.4f}  = (TP+TN)/total = ({tp}+{tn})/{len(y_te)}")
print(f"Precision: {p:.4f}  = TP/(TP+FP) = {tp}/({tp}+{fp})")
print(f"Recall   : {r:.4f}  = TP/(TP+FN) = {tp}/({tp}+{fn})")
print(f"F1       : {f1:.4f}")

cases = [
    ("Balanced model", 0.75, 0.80),
    ("Lazy model (only 1 'yes')", 1.00, 0.01),
    ("Generous model ('yes' to all)", 0.51, 1.00),
]
rows = []
for name, pr, rc in cases:
    mean = (pr + rc) / 2
    harmonic = 2 * pr * rc / (pr + rc)
    rows.append({"Model": name, "Precision": pr, "Recall": rc,
                 "Plain mean": round(mean, 3), "F1 (harmonic)": round(harmonic, 3)})
show(pd.DataFrame(rows))

print(f"Times the model said 'goes fast'  : {pred.sum()}")
print(f"Flats in the test set that went fast: {y_te.sum()}")
print()
print("Is the model generous or strict:",
      "GENEROUS" if pred.sum() > y_te.sum() else "STRICT")

from sklearn.metrics import classification_report

print(classification_report(y_te, pred,
                            target_names=["went slow", "went fast"],
                            digits=4))

show(pd.DataFrame([
    {"Situation": "Ad budget is limited", "Prioritise": "Precision",
     "Why": "Don't spend on empty listings (FP is expensive)"},
    {"Situation": "Every opportunity is valuable", "Prioritise": "Recall",
     "Why": "Don't miss a flat that will go fast (FN is expensive)"},
    {"Situation": "Disease detection", "Prioritise": "Recall",
     "Why": "Sending a sick person home as healthy is the worst mistake"},
    {"Situation": "Spam filter", "Prioritise": "Precision",
     "Why": "Putting an important email in spam is the worst mistake"},
    {"Situation": "Both matter", "Prioritise": "F1",
     "Why": "Balance; if one is low, F1 is low"},
    {"Situation": "Classes very imbalanced", "Prioritise": "macro F1 / PR-AUC",
     "Why": "Both accuracy and weighted lie (Chapter 9)"},
]))

import numpy as np

# Build an imbalanced case: cut the share of "went fast" to ~5%
np.random.seed(42)
fast = y_te[y_te == 1].index
slow = y_te[y_te == 0].index
sample = list(slow) + list(np.random.choice(fast, size=int(len(slow) * 0.05), replace=False))

y_imb = y_te.loc[sample]
print(f"New set: {len(y_imb)} rows, share of 'went fast' {y_imb.mean():.1%}")
print()

lazy = np.zeros(len(y_imb), dtype=int)   # says "goes slow" to everyone
print(f"LAZY model ('slow' for everyone):")
print(f"  Accuracy : {accuracy_score(y_imb, lazy):.4f}   <-- beautiful!")
print(f"  Recall   : {recall_score(y_imb, lazy, zero_division=0):.4f}   <-- did not find A SINGLE one")
print(f"  F1       : {f1_score(y_imb, lazy, zero_division=0):.4f}")

show(pd.DataFrame([
    {"Concept": "Confusion matrix", "In one sentence": "Four cells: TN, FP, FN, TP. The whole story is there"},
    {"Concept": "FP and FN", "In one sentence": "Two kinds of mistake, two prices; accuracy doesn't separate them"},
    {"Concept": "Precision", "In one sentence": "When the model says 'yes', how often it is right: TP/(TP+FP)"},
    {"Concept": "Recall", "In one sentence": "Of all real 'yes' cases, how many it found: TP/(TP+FN)"},
    {"Concept": "F1", "In one sentence": "Harmonic mean; if either one is low, F1 is low"},
    {"Concept": "macro vs weighted", "In one sentence": "On imbalanced data read macro; weighted hides things"},
    {"Concept": "Choosing a metric", "In one sentence": "A business decision: is FP or FN more expensive?"},
]))

show(pd.DataFrame([
    {"Metric": "Accuracy", "Value": f"{accuracy_score(y_te, pred):.4f}", "How to read it": "Overall correctness"},
    {"Metric": "Precision", "Value": f"{p:.4f}", "How to read it": f"{p:.0%} of its 'yes' answers are right"},
    {"Metric": "Recall", "Value": f"{r:.4f}", "How to read it": f"Found {r:.0%} of the opportunities"},
    {"Metric": "F1", "Value": f"{f1:.4f}", "How to read it": "The balance score"},
    {"Metric": "FP (burned ads)", "Value": str(fp), "How to read it": "This many empty listings got ad money"},
    {"Metric": "FN (missed)", "Value": str(fn), "How to read it": "This many opportunities went unnoticed"},
]))