Tahrirsiz: titanic bilan birinchi uchrashuv

import pandas as pd

df = pd.read_csv(DATA / "titanic.csv")
print("shakli:", df.shape)
print()
print(df.dtypes.to_string())

show(df.head(5))

print(df.survived.value_counts(normalize=True).sort_index().to_string())
print()
print(f"Agar model HAMMAGA 'o'ldi' desa, accuracy = {1 - df.survived.mean():.4f}")

import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score

X, y = df.drop(columns=["survived"]), df["survived"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

def quvur_qur(son, matn):
    return Pipeline([
        ("tayyorla", ColumnTransformer([
            ("son", Pipeline([("toldir", SimpleImputer(strategy="median")),
                              ("shkala", StandardScaler())]), son),
            ("matn", Pipeline([("toldir", SimpleImputer(strategy="most_frequent")),
                               ("kod", OneHotEncoder(handle_unknown="ignore"))]), matn),
        ])),
        ("model", LogisticRegression(max_iter=3000)),
    ])

baza = accuracy_score(y_te, DummyClassifier(strategy="most_frequent")
                      .fit(X_tr[["pclass"]], y_tr).predict(X_te[["pclass"]]))

son = X.select_dtypes(include=[np.number, bool]).columns.tolist()
matn = [c for c in X.columns if c not in son]
print("son ustunlar :", son)
print("matn ustunlar:", matn)

q1 = quvur_qur(son, matn).fit(X_tr, y_tr)
a1 = accuracy_score(y_te, q1.predict(X_te))
print()
print(f"BAZA      : {baza:.4f}")
print(f"1-urinish : {a1:.4f}")

for ustun in matn:
    jadval = df.groupby(ustun, dropna=False).survived.agg(["count", "mean"])
    aniq = ((jadval["mean"] == 0) | (jadval["mean"] == 1)).all()
    belgi = "   <-- TARGET NI TO'LIQ AJRATADI" if aniq else ""
    print(f"{ustun}:{belgi}")
    print(jadval.round(3).to_string())
    print()

print("alive ustunining qiymatlari:", df.alive.unique().tolist())
print("'alive == yes'  'survived == 1' ga MOS keladimi:",
      bool(((df.alive == "yes") == (df.survived == 1)).all()))

X2_tr, X2_te = X_tr.drop(columns=["alive"]), X_te.drop(columns=["alive"])
son2 = [c for c in son if c != "alive"]
matn2 = [c for c in matn if c != "alive"]

q2 = quvur_qur(son2, matn2).fit(X2_tr, y_tr)
a2 = accuracy_score(y_te, q2.predict(X2_te))
print(f"BAZA      : {baza:.4f}")
print(f"1-urinish : {a1:.4f}   (leakage)")
print(f"2-urinish : {a2:.4f}   (alive tashlandi)")

yoq = df.isna().sum()
yoq = yoq[yoq > 0].sort_values(ascending=False)
for ustun, n in yoq.items():
    print(f"  {ustun:14s} {n:3d} ta yo'q  ({n/len(df):.1%})")

X3_tr, X3_te = X2_tr.drop(columns=["deck"]), X2_te.drop(columns=["deck"])
son3 = [c for c in son2 if c != "deck"]
matn3 = [c for c in matn2 if c != "deck"]

q3 = quvur_qur(son3, matn3).fit(X3_tr, y_tr)
a3 = accuracy_score(y_te, q3.predict(X3_te))
print(f"2-urinish : {a2:.4f}   (deck bor, 77% to'ldirilgan)")
print(f"3-urinish : {a3:.4f}   (deck tashlandi)")
print(f"O'zgarish : {a3 - a2:+.4f}")

print("class  ~  pclass:")
print(df.groupby("pclass")["class"].unique().to_string())
print()
print("adult_male  ==  (sex=='male' va age>=18):",
      bool((df.adult_male == ((df.sex == "male") & (df.age >= 18))).all()))
print("alone       ==  (sibsp + parch == 0)   :",
      bool((df.alone == ((df.sibsp + df.parch) == 0)).all()))
print("who         qiymatlari                 :", df.who.unique().tolist())
print("embark_town ~ embarked                 :",
      df.groupby("embarked")["embark_town"].unique().to_dict())

show(pd.DataFrame([
    {"Ustun": "class", "Nimadan hisoblangan": "pclass (1→First, 2→Second, 3→Third)"},
    {"Ustun": "adult_male", "Nimadan hisoblangan": "sex va age"},
    {"Ustun": "alone", "Nimadan hisoblangan": "sibsp + parch"},
    {"Ustun": "who", "Nimadan hisoblangan": "sex va age (man/woman/child)"},
    {"Ustun": "embark_town", "Nimadan hisoblangan": "embarked (S→Southampton, ...)"},
]))

halol = ["pclass", "sex", "age", "sibsp", "parch", "fare", "embarked"]
son4 = ["pclass", "age", "sibsp", "parch", "fare"]
matn4 = ["sex", "embarked"]

q4 = quvur_qur(son4, matn4).fit(X_tr[halol], y_tr)
a4 = accuracy_score(y_te, q4.predict(X_te[halol]))
print(f"3-urinish : {a3:.4f}   (13 ustun)")
print(f"4-urinish : {a4:.4f}   (7 'halol' ustun)")
print(f"O'zgarish : {a4 - a3:+.4f}")

from sklearn.model_selection import cross_val_score

cv3 = cross_val_score(quvur_qur(son3, matn3), X.drop(columns=["alive", "deck"]), y, cv=5)
cv4 = cross_val_score(quvur_qur(son4, matn4), X[halol], y, cv=5)

print("3-urinish (13 ustun):")
print("  fold lar:", cv3.round(4).tolist())
print(f"  o'rtacha: {cv3.mean():.4f}  ±{cv3.std():.4f}")
print()
print("4-urinish (7 ustun):")
print("  fold lar:", cv4.round(4).tolist())
print(f"  o'rtacha: {cv4.mean():.4f}  ±{cv4.std():.4f}")
print()
print(f"O'rtachalar farqi : {abs(cv3.mean() - cv4.mean()):.4f}")
print(f"Fold lar tarqalishi: ±{max(cv3.std(), cv4.std()):.4f}")

show(pd.DataFrame([
    {"Urinish": "0. Baza (hammaga 'o'ldi')", "Accuracy": f"{baza:.4f}", "Xulosa": "Chegara"},
    {"Urinish": "1. Hamma 14 ustun", "Accuracy": f"{a1:.4f}", "Xulosa": "LEAKAGE — alive = target"},
    {"Urinish": "2. alive tashlandi", "Accuracy": f"{a2:.4f}", "Xulosa": "Birinchi halol raqam"},
    {"Urinish": "3. deck ham tashlandi (77% bo'sh)", "Accuracy": f"{a3:.4f}", "Xulosa": "Ustun kamaydi, natija yaxshilandi"},
    {"Urinish": "4. Faqat 7 'halol' ustun", "Accuracy": f"{a4:.4f}", "Xulosa": "Gipotezam RAD ETILDI"},
    {"Urinish": "CV: 3-urinish", "Accuracy": f"{cv3.mean():.4f} ±{cv3.std():.4f}", "Xulosa": "Haqiqiy baho — 13 ustun yetakchi"},
    {"Urinish": "CV: 4-urinish", "Accuracy": f"{cv4.mean():.4f} ±{cv4.std():.4f}", "Xulosa": "Diapazonlar ustma-ust — ishonch cheklangan"},
]))

show(pd.DataFrame([
    {"Xato": "Hamma ustunni modelga berdim", "Oqibat": "1.0000 — yolg'on natija",
     "Dars": "Har ustunni target bilan groupby qilib ko'r"},
    {"Xato": "77% bo'sh ustunni to'ldirdim", "Oqibat": "Natija pastroq edi",
     "Dars": "Ko'p bo'sh ustun — to'ldirilmaydi, tashlanadi"},
    {"Xato": "Takrorlanuvchi ustunlarni tashladim", "Oqibat": "Natija yomonlashdi",
     "Dars": "'Ortiqcha' ni o'ylab emas, o'lchab tashla"},
    {"Xato": "Bitta split ga ishondim", "Oqibat": "Farq haqiqiy chiqdi, lekin diapazonlar ustma-ust",
     "Dars": "CV: o'rtachaga EMAS, fold diapazoniga qara"},
]))