Cross-validation

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
X, y = df[num + ["tuman"]], df["ijaraga_berildi_30k"]

def build_pipe():
    return Pipeline([
        ("prep", ColumnTransformer([
            ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                              ("scale", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", LogisticRegression(max_iter=2000)),
    ])

results = []
for rs in [0, 1, 7, 13, 42, 99, 123, 2026]:
    X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=rs, stratify=y)
    a = accuracy_score(y_te, build_pipe().fit(X_tr, y_tr).predict(X_te))
    results.append(a)
    print(f"  random_state={rs:<5} accuracy = {a:.4f}")

print()
print(f"Lowest : {min(results):.4f}")
print(f"Highest: {max(results):.4f}")
print(f"Spread : {max(results) - min(results):.4f}")

diagramma(
    {
        "fold 1:  TEST | train | train | train | train": (0, 0, 7, 0.9, "xizmat"),
        "fold 2:  train | TEST | train | train | train": (0, 1.2, 7, 0.9, "qayta"),
        "fold 3:  train | train | TEST | train | train": (0, 2.4, 7, 0.9, "qayta"),
        "fold 4:  train | train | train | TEST | train": (0, 3.6, 7, 0.9, "qayta"),
        "fold 5:  train | train | train | train | TEST": (0, 4.8, 7, 0.9, "qayta"),
        "5 scores -> mean ± spread": (1.5, 6.2, 4, 0.9, "saqlash"),
    },
    [("fold 1:  TEST | train | train | train | train", "5 scores -> mean ± spread"),
     ("fold 5:  train | train | train | train | TEST", "5 scores -> mean ± spread")],
    nom="cv.png",
    izoh="5-fold cross-validation: the model is trained 5 times, and every row is tested once.",
)

from sklearn.model_selection import cross_val_score

scores = cross_val_score(build_pipe(), X, y, cv=5, scoring="accuracy")
print("Each fold:", scores.round(4).tolist())
print()
print(f"Mean   : {scores.mean():.4f}")
print(f"Spread : ±{scores.std():.4f}")
print()
print(f"In a report you write: {scores.mean():.3f} ± {scores.std():.3f}")

from sklearn.model_selection import KFold, StratifiedKFold

print("Share of 'yes' in each fold:")
print(f"{'fold':>5} {'KFold':>9} {'StratifiedKFold':>17}")
print("-" * 34)
kf = list(KFold(n_splits=5, shuffle=True, random_state=42).split(X, y))
sk = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=42).split(X, y))
for i in range(5):
    print(f"{i+1:>5} {y.iloc[kf[i][1]].mean():>8.1%} {y.iloc[sk[i][1]].mean():>16.1%}")
print()
print(f"True ratio: {y.mean():.1%}")

from sklearn.preprocessing import StandardScaler as SS

# WRONG: preprocess all the data first, then CV
imp = SimpleImputer(strategy="median").fit(X[num])
scaler = SS().fit(imp.transform(X[num]))
X_before = pd.DataFrame(scaler.transform(imp.transform(X[num])), columns=num, index=X.index)
bad = cross_val_score(LogisticRegression(max_iter=2000), X_before, y, cv=5)

# RIGHT: preprocessing inside the Pipeline, redone in every fold
good = cross_val_score(build_pipe(), X, y, cv=5)

print(f"WRONG (preprocess first, then CV): {bad.mean():.4f} ±{bad.std():.4f}")
print(f"RIGHT (Pipeline inside CV)       : {good.mean():.4f} ±{good.std():.4f}")
print(f"Difference                       : {bad.mean() - good.mean():+.4f}")

from sklearn.model_selection import cross_validate

result = cross_validate(build_pipe(), X, y, cv=5,
                        scoring=["accuracy", "precision", "recall", "f1", "roc_auc"],
                        return_train_score=True)
rows = []
for m in ["accuracy", "precision", "recall", "f1", "roc_auc"]:
    te = result[f"test_{m}"]
    tr = result[f"train_{m}"]
    rows.append({"Metric": m,
                 "Test mean": round(te.mean(), 4), "Test ±": round(te.std(), 4),
                 "Train mean": round(tr.mean(), 4),
                 "Train - Test": round(tr.mean() - te.mean(), 4)})
show(pd.DataFrame(rows))

from sklearn.ensemble import RandomForestClassifier

def with_pipe(model):
    return Pipeline([
        ("prep", ColumnTransformer([
            ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                              ("scale", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", model),
    ])

lr = cross_val_score(with_pipe(LogisticRegression(max_iter=2000)), X, y, cv=5)
rf = cross_val_score(with_pipe(RandomForestClassifier(n_estimators=200, random_state=42)), X, y, cv=5)

print("Logistic Regression:", lr.round(4).tolist())
print(f"  mean {lr.mean():.4f} ±{lr.std():.4f}")
print()
print("Random Forest      :", rf.round(4).tolist())
print(f"  mean {rf.mean():.4f} ±{rf.std():.4f}")
print()
print(f"Gap between means  : {abs(rf.mean() - lr.mean()):.4f}")
print(f"Largest spread     : ±{max(lr.std(), rf.std()):.4f}")
print()
leader = "Random Forest" if rf.mean() > lr.mean() else "Logistic Regression"
confidence = "gap is LARGER than the spread" if abs(rf.mean()-lr.mean()) > max(lr.std(), rf.std()) else "gap is WITHIN the spread"
print(f"Leader: {leader}  |  {confidence}")

show(pd.DataFrame([
    {"Case": "Gap > 2 x spread", "Conclusion": "The gap is reliable: take the better one"},
    {"Case": "Gap ~ spread", "Conclusion": "Doubtful: compare the fold ranges"},
    {"Case": "Gap < spread", "Conclusion": "There is NO difference: take the simpler/faster model"},
]))

print(f"{'cv':>4} {'mean':>10} {'spread':>11} {'time (rel.)':>14}")
print("-" * 42)
import time
for k in [3, 5, 10]:
    t0 = time.perf_counter()
    b = cross_val_score(build_pipe(), X, y, cv=k)
    dt = time.perf_counter() - t0
    print(f"{k:>4} {b.mean():>10.4f} {b.std():>10.4f} {dt:>13.2f}s")

from sklearn.model_selection import TimeSeriesSplit

tss = TimeSeriesSplit(n_splits=4)
fake = np.arange(20)
print("TimeSeriesSplit: train always comes BEFORE test:")
for i, (tr_i, te_i) in enumerate(tss.split(fake), 1):
    print(f"  fold {i}: train={tr_i.min()}..{tr_i.max()}   test={te_i.min()}..{te_i.max()}")

show(pd.DataFrame([
    {"Concept": "One split isn't enough", "In one sentence": "Change random_state and the number changes"},
    {"Concept": "cross_val_score", "In one sentence": "Trains k times; every row is tested once"},
    {"Concept": "Mean ± spread", "In one sentence": "Two numbers in a report, not one"},
    {"Concept": "StratifiedKFold", "In one sentence": "Automatic for classification; keeps the class ratio"},
    {"Concept": "Pipeline inside CV", "In one sentence": "CV gets RAW data, not cleaned data"},
    {"Concept": "cross_validate", "In one sentence": "Several metrics + the train/test gap (overfitting)"},
    {"Concept": "The comparison rule", "In one sentence": "If the gap is smaller than the spread, there is no gap"},
    {"Concept": "TimeSeriesSplit", "In one sentence": "Don't shuffle when there is time"},
]))