Exercise 3: hidden leakage

import pandas as pd

td = pd.read_csv(DATA / "talaba_bitirdi.csv", keep_default_na=False, na_values=[""])
print("shape:", td.shape)
print()
print(td.dtypes.to_string())
print()
show(td.head(8))

print("Target:")
print(td.bitirdi.value_counts(normalize=True).sort_index().round(4).to_string())
print()
print(f"BASELINE (saying '1' to everyone): {td.bitirdi.mean():.4f}")
print()
missing = td.isna().sum()
print("Missing values:")
print(missing[missing > 0].to_string() if (missing > 0).any() else "  none")

import pandas as pd
show(pd.DataFrame([
    {"#": 1, "Answer to write down": "How many rows, how many columns"},
    {"#": 2, "Answer to write down": "\"I predict ___ from ___\": one sentence"},
    {"#": 3, "Answer to write down": "Task type: regression / binary / multiclass"},
    {"#": 4, "Answer to write down": "The target ratio and the BASELINE number"},
    {"#": 5, "Answer to write down": "Did you split? Do you need stratify?"},
    {"#": 6, "Answer to write down": "The dummy result"},
    {"#": 7, "Answer to write down": "Which columns you dropped, and WHY"},
    {"#": 8, "Answer to write down": "What you did about missing values and types"},
    {"#": 9, "Answer to write down": "Your first honest result"},
    {"#": 10, "Answer to write down": "Which metric you chose, and WHY"},
    {"#": 11, "Answer to write down": "Which group the mistakes cluster in"},
    {"#": 12, "Answer to write down": "Which ONE change you made, and the result"},
]))

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score

honest = ["kirish_bali", "davomat_foiz", "uy_vazifa_bali", "ishlaydi", "masofa_km"]
y = td["bitirdi"]
tr, te, ytr, yte = train_test_split(td, y, test_size=0.3, random_state=42, stratify=y)

baseline = accuracy_score(yte, DummyClassifier(strategy="most_frequent").fit(tr[honest], ytr).predict(te[honest]))
m1 = LogisticRegression(max_iter=2000).fit(tr[honest], ytr)
b1 = m1.predict(te[honest])

print(f"BASELINE                    : {baseline:.4f}")
print(f"5 honest columns: accuracy  : {accuracy_score(yte, b1):.4f}   (gap {accuracy_score(yte, b1)-baseline:+.4f})")
print(f"5 honest columns: F1        : {f1_score(yte, b1):.4f}")
print(f"5 honest columns: recall    : {recall_score(yte, b1):.4f}")

td2 = td.copy()
td2["has_diploma"] = td2.diplom_seriya.notna().astype(int)
tr2, te2 = td2.loc[tr.index], td2.loc[te.index]

m2 = LogisticRegression(max_iter=2000).fit(tr2[honest + ["has_diploma"]], ytr)
b2 = m2.predict(te2[honest + ["has_diploma"]])
print(f"+ diplom_seriya (present/absent): accuracy {accuracy_score(yte, b2):.4f}   <-- PERFECT")
print()
print("Why? Let's check:")
print(pd.crosstab(td2.has_diploma, td2.bitirdi).to_string())

m3 = LogisticRegression(max_iter=2000).fit(tr[honest + ["oxirgi_tolov_oyi"]], ytr)
b3 = m3.predict(te[honest + ["oxirgi_tolov_oyi"]])
print(f"+ oxirgi_tolov_oyi: accuracy {accuracy_score(yte, b3):.4f}  F1 {f1_score(yte, b3):.4f}")
print()
print("Means by group:")
print(td.groupby("bitirdi").oxirgi_tolov_oyi.agg(["mean", "min", "max"]).round(2).to_string())

show(pd.DataFrame([
    {"Column": "diplom_seriya",
     "Related to the target": "Completely (100%)",
     "Available at prediction time": "NO: issued after graduating",
     "Verdict": "LEAKAGE: drop it"},
    {"Column": "oxirgi_tolov_oyi",
     "Related to the target": "Partly",
     "Available at prediction time": "YES: known every month",
     "Verdict": "A real signal: use it (but carefully)"},
]))

show(pd.DataFrame([
    {"Exercise": "43. penguins", "What it tests": "A broken pattern: from binary to multiclass",
     "Main lesson": "average='macro' and a 3x3 matrix"},
    {"Exercise": "44. dirty sales", "What it tests": "Cleaning is 80% of the work",
     "Main lesson": "Cyrillic/Latin, text-as-number, duplicates; and the honest 'no model is possible'"},
    {"Exercise": "45. hidden leakage", "What it tests": "Finding the trap yourself",
     "Main lesson": "TIME separates leakage, not the strength of the relationship"},
]))