Cross-validation

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

df = pd.read_csv(DATA / "uy_ijara.csv")
son = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
X, y = df[son + ["tuman"]], df["ijaraga_berildi_30k"]

def quvur_qur():
    return Pipeline([
        ("tayyorla", ColumnTransformer([
            ("son", Pipeline([("toldir", SimpleImputer(strategy="median")),
                              ("shkala", StandardScaler())]), son),
            ("matn", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", LogisticRegression(max_iter=2000)),
    ])

natijalar = []
for rs in [0, 1, 7, 13, 42, 99, 123, 2026]:
    X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=rs, stratify=y)
    a = accuracy_score(y_te, quvur_qur().fit(X_tr, y_tr).predict(X_te))
    natijalar.append(a)
    print(f"  random_state={rs:<5} accuracy = {a:.4f}")

print()
print(f"Eng past : {min(natijalar):.4f}")
print(f"Eng baland: {max(natijalar):.4f}")
print(f"Tarqalish : {max(natijalar) - min(natijalar):.4f}")

diagramma(
    {
        "1-fold:  TEST | train | train | train | train": (0, 0, 7, 0.9, "xizmat"),
        "2-fold:  train | TEST | train | train | train": (0, 1.2, 7, 0.9, "qayta"),
        "3-fold:  train | train | TEST | train | train": (0, 2.4, 7, 0.9, "qayta"),
        "4-fold:  train | train | train | TEST | train": (0, 3.6, 7, 0.9, "qayta"),
        "5-fold:  train | train | train | train | TEST": (0, 4.8, 7, 0.9, "qayta"),
        "5 ta baho -> o'rtacha ± tarqalish": (1.5, 6.2, 4, 0.9, "saqlash"),
    },
    [("1-fold:  TEST | train | train | train | train", "5 ta baho -> o'rtacha ± tarqalish"),
     ("5-fold:  train | train | train | train | TEST", "5 ta baho -> o'rtacha ± tarqalish")],
    nom="cv.png",
    izoh="5-fold cross-validation: model 5 marta o'qitiladi, har qator bir marta test bo'ladi.",
)

from sklearn.model_selection import cross_val_score

ballar = cross_val_score(quvur_qur(), X, y, cv=5, scoring="accuracy")
print("Har bir fold:", ballar.round(4).tolist())
print()
print(f"O'rtacha  : {ballar.mean():.4f}")
print(f"Tarqalish : ±{ballar.std():.4f}")
print()
print(f"Hisobotga shunday yoziladi: {ballar.mean():.3f} ± {ballar.std():.3f}")

from sklearn.model_selection import KFold, StratifiedKFold

print("Har fold dagi 'ha' ulushi:")
print(f"{'fold':>5} {'KFold':>9} {'StratifiedKFold':>17}")
print("-" * 34)
kf = list(KFold(n_splits=5, shuffle=True, random_state=42).split(X, y))
sk = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=42).split(X, y))
for i in range(5):
    print(f"{i+1:>5} {y.iloc[kf[i][1]].mean():>8.1%} {y.iloc[sk[i][1]].mean():>16.1%}")
print()
print(f"Haqiqiy nisbat: {y.mean():.1%}")

from sklearn.preprocessing import StandardScaler as SS

# NOTO'G'RI: avval butun ma'lumotni tozalab, keyin CV
toldir = SimpleImputer(strategy="median").fit(X[son])
shkala = SS().fit(toldir.transform(X[son]))
X_oldin = pd.DataFrame(shkala.transform(toldir.transform(X[son])), columns=son, index=X.index)
yomon = cross_val_score(LogisticRegression(max_iter=2000), X_oldin, y, cv=5)

# TO'G'RI: tozalash Pipeline ichida, har fold da qaytadan
yaxshi = cross_val_score(quvur_qur(), X, y, cv=5)

print(f"NOTO'G'RI (oldin tozalab, keyin CV): {yomon.mean():.4f} ±{yomon.std():.4f}")
print(f"TO'G'RI   (Pipeline CV ichida)     : {yaxshi.mean():.4f} ±{yaxshi.std():.4f}")
print(f"Farq                               : {yomon.mean() - yaxshi.mean():+.4f}")

from sklearn.model_selection import cross_validate

natija = cross_validate(quvur_qur(), X, y, cv=5,
                        scoring=["accuracy", "precision", "recall", "f1", "roc_auc"],
                        return_train_score=True)
qatorlar = []
for m in ["accuracy", "precision", "recall", "f1", "roc_auc"]:
    te = natija[f"test_{m}"]
    tr = natija[f"train_{m}"]
    qatorlar.append({"Metrika": m,
                     "Test o'rtacha": round(te.mean(), 4), "Test ±": round(te.std(), 4),
                     "Train o'rtacha": round(tr.mean(), 4),
                     "Train - Test": round(tr.mean() - te.mean(), 4)})
show(pd.DataFrame(qatorlar))

from sklearn.ensemble import RandomForestClassifier

def quvur_bilan(model):
    return Pipeline([
        ("tayyorla", ColumnTransformer([
            ("son", Pipeline([("toldir", SimpleImputer(strategy="median")),
                              ("shkala", StandardScaler())]), son),
            ("matn", OneHotEncoder(handle_unknown="ignore"), ["tuman"]),
        ])),
        ("model", model),
    ])

lr = cross_val_score(quvur_bilan(LogisticRegression(max_iter=2000)), X, y, cv=5)
rf = cross_val_score(quvur_bilan(RandomForestClassifier(n_estimators=200, random_state=42)), X, y, cv=5)

print("Logistic Regression:", lr.round(4).tolist())
print(f"  o'rtacha {lr.mean():.4f} ±{lr.std():.4f}")
print()
print("Random Forest      :", rf.round(4).tolist())
print(f"  o'rtacha {rf.mean():.4f} ±{rf.std():.4f}")
print()
print(f"O'rtachalar farqi  : {abs(rf.mean() - lr.mean()):.4f}")
print(f"Eng katta tarqalish: ±{max(lr.std(), rf.std()):.4f}")
print()
yetakchi = "Random Forest" if rf.mean() > lr.mean() else "Logistic Regression"
ishonch = "farq tarqalishdan KATTA" if abs(rf.mean()-lr.mean()) > max(lr.std(), rf.std()) else "farq tarqalish ICHIDA"
print(f"Yetakchi: {yetakchi}  |  {ishonch}")

show(pd.DataFrame([
    {"Holat": "Farq > 2 x tarqalish", "Xulosa": "Farq ishonchli — kattasini tanla"},
    {"Holat": "Farq ~ tarqalish", "Xulosa": "Shubhali — fold diapazonlarini solishtir"},
    {"Holat": "Farq < tarqalish", "Xulosa": "Farq YO'Q — oddiy/tez modelni tanla"},
]))

print(f"{'cv':>4} {'o`rtacha':>10} {'tarqalish':>11} {'vaqt (nisbiy)':>14}")
print("-" * 42)
import time
for k in [3, 5, 10]:
    t0 = time.perf_counter()
    b = cross_val_score(quvur_qur(), X, y, cv=k)
    dt = time.perf_counter() - t0
    print(f"{k:>4} {b.mean():>10.4f} {b.std():>10.4f} {dt:>13.2f}s")

from sklearn.model_selection import TimeSeriesSplit

tss = TimeSeriesSplit(n_splits=4)
soxta = np.arange(20)
print("TimeSeriesSplit — train har doim test dan OLDIN:")
for i, (tr_i, te_i) in enumerate(tss.split(soxta), 1):
    print(f"  {i}-fold: train={tr_i.min()}..{tr_i.max()}   test={te_i.min()}..{te_i.max()}")

show(pd.DataFrame([
    {"Tushuncha": "Bitta split yetarli emas", "Bitta jumla": "random_state o'zgarsa raqam o'zgaradi"},
    {"Tushuncha": "cross_val_score", "Bitta jumla": "k marta o'qitadi, har qator bir marta test bo'ladi"},
    {"Tushuncha": "O'rtacha ± tarqalish", "Bitta jumla": "Hisobotda ikkita son, bittasi emas"},
    {"Tushuncha": "StratifiedKFold", "Bitta jumla": "Classification da avtomatik — sinf nisbati saqlanadi"},
    {"Tushuncha": "Pipeline CV ichida", "Bitta jumla": "CV ga XOM ma'lumot beriladi, tozalangan emas"},
    {"Tushuncha": "cross_validate", "Bitta jumla": "Bir nechta metrika + train/test farqi (overfitting)"},
    {"Tushuncha": "Taqqoslash qoidasi", "Bitta jumla": "Farq tarqalishdan kichik bo'lsa — farq yo'q"},
    {"Tushuncha": "TimeSeriesSplit", "Bitta jumla": "Vaqt bo'lsa aralashtirmaslik"},
]))