A new dataset: 12 steps

import pandas as pd
show(pd.DataFrame([
    {"Step": 1,  "What I do": "Open the file", "The decision it produces": "How many rows, how many columns; I write it down"},
    {"Step": 2,  "What I do": "Name the target", "The decision it produces": "One sentence: what I predict from what"},
    {"Step": 3,  "What I do": "Look at the target's type", "The decision it produces": "Regression / classification / clustering"},
    {"Step": 4,  "What I do": "COUNT the target", "The decision it produces": "Is it imbalanced? What is my baseline?"},
    {"Step": 5,  "What I do": "Split NOW", "The decision it produces": "The test set gets locked away"},
    {"Step": 6,  "What I do": "Dummy model", "The decision it produces": "The pass mark, as a single number"},
    {"Step": 7,  "What I do": "Find the poisonous columns", "The decision it produces": "A list of columns to drop"},
    {"Step": 8,  "What I do": "Missing values and types", "The decision it produces": "For each column: fill or drop"},
    {"Step": 9,  "What I do": "One simple model in a Pipeline", "The decision it produces": "Is it better than the dummy?"},
    {"Step": 10, "What I do": "Choose the metric", "The decision it produces": "Which number I judge by"},
    {"Step": 11, "What I do": "READ the rows it got wrong", "The decision it produces": "What the model doesn't understand"},
    {"Step": 12, "What I do": "Make a decision", "The decision it produces": "One change, then back to step 9"},
]))

import pandas as pd

df = pd.read_csv(DATA / "uy_ijara.csv")

print("shape:", df.shape)
print()
print(df.dtypes.to_string())

show(df.head(6))

print("ijaraga_berildi_30k:")
print("  dtype         :", df.ijaraga_berildi_30k.dtype)
print("  unique values :", sorted(df.ijaraga_berildi_30k.unique().tolist()))
print()
print("narx_usd:")
print("  dtype         :", df.narx_usd.dtype)
print("  unique values :", df.narx_usd.nunique())

show(pd.DataFrame([
    {"What the target looks like": "Two values (0/1, yes/no)", "Task": "Binary classification"},
    {"What the target looks like": "A few labels (3-10 kinds)", "Task": "Multiclass classification"},
    {"What the target looks like": "A continuous number (price, temperature, time)", "Task": "Regression"},
    {"What the target looks like": "NO target", "Task": "Clustering or PCA"},
]))

ratio = df.ijaraga_berildi_30k.value_counts(normalize=True).sort_index()
print("0 (not rented):", f"{ratio[0]:.1%}")
print("1 (rented)    :", f"{ratio[1]:.1%}")
print()
print("If the model says '1' to EVERYONE, accuracy =", f"{ratio[1]:.1%}")

from sklearn.model_selection import train_test_split

X = df.drop(columns=["ijaraga_berildi_30k", "narx_usd"])
y = df["ijaraga_berildi_30k"]

X_tr, X_te, y_tr, y_te = train_test_split(
    X, y, test_size=0.3, random_state=42, stratify=y
)

print(f"train: {len(X_tr)} rows, rented {y_tr.mean():.1%}")
print(f"test : {len(X_te)} rows, rented {y_te.mean():.1%}")

from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score

dummy = DummyClassifier(strategy="most_frequent").fit(X_tr[["xonalar"]], y_tr)
baseline = accuracy_score(y_te, dummy.predict(X_te[["xonalar"]]))

print(f"BASELINE (a model that learned nothing): {baseline:.4f}")
print()
print("Write this number down. Your model MUST beat it.")

print("Unique values per column (out of 1200 rows):")
for col in X_tr.columns:
    n = X_tr[col].nunique()
    flag = "   <-- SUSPICIOUS: different in almost every row" if n > len(X_tr) * 0.9 else ""
    print(f"  {col:22s} {n:5d}{flag}")

print("first 3 values of narx_matn:")
for v in X_tr.narx_matn.head(3):
    print("  ", repr(v))
print()
print("dtype:", X_tr.narx_matn.dtype, "<-- object = text")

check = X_tr.assign(rented=y_tr).groupby("rented").agent_komissiyasi.agg(["mean", "min", "max"])
show(check.reset_index())

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

honest = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]

m_honest = LogisticRegression(max_iter=2000).fit(X_tr[honest], y_tr)
m_leak = LogisticRegression(max_iter=2000).fit(X_tr[honest + ["agent_komissiyasi"]], y_tr)

acc_honest = accuracy_score(y_te, m_honest.predict(X_te[honest]))
acc_leak = accuracy_score(y_te, m_leak.predict(X_te[honest + ["agent_komissiyasi"]]))

print(f"Honest features         : {acc_honest:.4f}")
print(f"+ agent_komissiyasi     : {acc_leak:.4f}   <-- PERFECT")

dropped = {
    "elon_id": "identifier, different in every row",
    "narx_matn": "the text form of the second target",
    "agent_komissiyasi": "LEAKAGE, only known after renting",
}
show(pd.DataFrame([{"Column": k, "Why it is dropped": v} for k, v in dropped.items()]))

missing = X_tr.isna().sum()
missing = missing[missing > 0]
if len(missing):
    print("There are missing values:")
    for col, n in missing.items():
        print(f"  {col:16s} {n:3d} ({n/len(X_tr):.1%})")
else:
    print("No missing values")
print()
print("Text columns (need encoding):")
for c in X_tr.select_dtypes("object").columns:
    print(f"  {c:16s} {X_tr[c].nunique()} distinct values")

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer

num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat = ["tuman"]

prep = ColumnTransformer([
    ("num", Pipeline([
        ("impute", SimpleImputer(strategy="median")),
        ("scale", StandardScaler()),
    ]), num),
    ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
])

pipe = Pipeline([
    ("prep", prep),
    ("model", LogisticRegression(max_iter=2000)),
])

pipe.fit(X_tr, y_tr)
acc = accuracy_score(y_te, pipe.predict(X_te))

print(f"BASELINE   : {baseline:.4f}")
print(f"First model: {acc:.4f}")
print(f"Difference : {acc - baseline:+.4f}")

from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score

pred = pipe.predict(X_te)
tn, fp, fn, tp = confusion_matrix(y_te, pred).ravel()

print(f"TN={tn}  FP={fp}  FN={fn}  TP={tp}")
print()
print(f"Accuracy : {acc:.4f}")
print(f"Precision: {precision_score(y_te, pred):.4f}  = TP/(TP+FP) = {tp}/({tp}+{fp})")
print(f"Recall   : {recall_score(y_te, pred):.4f}  = TP/(TP+FN) = {tp}/({tp}+{fn})")
print(f"F1       : {f1_score(y_te, pred):.4f}")

proba = pipe.predict_proba(X_te)[:, 1]
analysis = X_te.assign(truth=y_te, pred=pred, confidence=proba.round(3))

wrong = analysis[analysis.truth != analysis.pred]
print(f"Rows it got wrong: {len(wrong)} ({len(wrong)/len(X_te):.1%})")
print()
print("The 5 rows the model got wrong MOST CONFIDENTLY:")
worst = wrong.assign(error=(wrong.confidence - wrong.truth).abs()).nlargest(5, "error")
show(worst[["tuman", "xonalar", "maydon_m2", "metro_km", "mebel", "truth", "pred", "confidence"]])

total = analysis.groupby("tuman").size()
errors = wrong.groupby("tuman").size()
result = pd.DataFrame({"total": total, "errors": errors}).fillna(0)
result["error_rate"] = (result.errors / result.total).round(3)
show(result.sort_values("error_rate", ascending=False).reset_index())

show(pd.DataFrame([
    {"Path": "Add a new feature", "When": "The mistakes cluster in one group",
     "Example": "price/area ratio, how close to the metro"},
    {"Path": "Another model", "When": "The relationship doesn't look linear",
     "Example": "Decision tree, Random Forest (Chapters 18-19)"},
    {"Path": "More data", "When": "Both train and test results are low",
     "Example": "Collect more listings (Chapter 37)"},
    {"Path": "Hyperparameter tuning", "When": "The model is good but not tuned",
     "Example": "GridSearchCV (Chapter 24)"},
    {"Path": "Stop", "When": "Well enough above the baseline, good enough for the business",
     "Example": "Save it and serve it (Chapters 40-41)"},
]))

def add_features(d):
    d = d.copy()
    d["area_per_room"] = d.maydon_m2 / d.xonalar
    d["high_floor"] = (d.qavat > d.qavatlar_soni * 0.7).astype(int)
    return d

num2 = num + ["area_per_room", "high_floor"]
prep2 = ColumnTransformer([
    ("num", Pipeline([("impute", SimpleImputer(strategy="median")),
                      ("scale", StandardScaler())]), num2),
    ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
])
pipe2 = Pipeline([("prep", prep2), ("model", LogisticRegression(max_iter=2000))])
pipe2.fit(add_features(X_tr), y_tr)

acc_new = accuracy_score(y_te, pipe2.predict(add_features(X_te)))
print(f"BASELINE              : {baseline:.4f}")
print(f"Attempt 1             : {acc:.4f}")
print(f"Attempt 2 (+2 features): {acc_new:.4f}")
print(f"Change                : {acc_new - acc:+.4f}")

diagramma(
    {
        "1. File\nshape, dtypes, head":            (0, 0, 3, 1.1, "manba"),
        "2. Name the target\none sentence":      (3.4, 0, 3, 1.1, "manba"),
        "3-4. Type and count\ntask + imbalance": (6.8, 0, 3.4, 1.1, "manba"),
        "5. SPLIT\ntest gets locked":            (0, 1.6, 3, 1.1, "saqlash"),
        "6. Dummy\nthe pass mark":               (3.4, 1.6, 3, 1.1, "saqlash"),
        "7-8. Poison and missing\ncolumn decisions": (6.8, 1.6, 3.4, 1.1, "saqlash"),
        "9. Pipeline + model":                   (0, 3.2, 3, 1.1, "qayta"),
        "10. Metric":                            (3.4, 3.2, 3, 1.1, "qayta"),
        "11. Read the mistakes":                 (6.8, 3.2, 3.4, 1.1, "qayta"),
        "12. ONE change":                        (3.4, 4.8, 3, 1.1, "xizmat"),
    },
    [
        ("1. File\nshape, dtypes, head", "2. Name the target\none sentence"),
        ("2. Name the target\none sentence", "3-4. Type and count\ntask + imbalance"),
        ("3-4. Type and count\ntask + imbalance", "5. SPLIT\ntest gets locked"),
        ("5. SPLIT\ntest gets locked", "6. Dummy\nthe pass mark"),
        ("6. Dummy\nthe pass mark", "7-8. Poison and missing\ncolumn decisions"),
        ("7-8. Poison and missing\ncolumn decisions", "9. Pipeline + model"),
        ("9. Pipeline + model", "10. Metric"),
        ("10. Metric", "11. Read the mistakes"),
        ("11. Read the mistakes", "12. ONE change"),
        ("12. ONE change", "9. Pipeline + model"),
    ],
    nom="protokol.png",
    izoh="12 steps. The last arrow goes back to step 9: it is a loop, not a straight line.",
)

show(pd.DataFrame([
    {"Step": "1-4", "In one sentence": "Understand the data without touching a model"},
    {"Step": "5", "In one sentence": "Lock the test set away (this is where leakage stops)"},
    {"Step": "6", "In one sentence": "Get the pass mark; without it you can't judge anything"},
    {"Step": "7-8", "In one sentence": "Drop the poisonous columns, handle the missing values"},
    {"Step": "9", "In one sentence": "ONE simple model in a Pipeline"},
    {"Step": "10", "In one sentence": "Choose the metric that fits the task"},
    {"Step": "11", "In one sentence": "Read the rows it got wrong; the next step comes from there"},
    {"Step": "12", "In one sentence": "ONE change and back to step 9"},
]))

show(pd.DataFrame([
    {"Model": "Dummy (baseline)", "Accuracy": f"{baseline:.4f}", "Note": "Learned nothing"},
    {"Model": "Logistic + Pipeline", "Accuracy": f"{acc:.4f}", "Note": "+0.27 over the baseline"},
    {"Model": "+ 2 new features", "Accuracy": f"{acc_new:.4f}", "Note": "Almost no change, an honest result"},
    {"Model": "+ agent_komissiyasi", "Accuracy": f"{acc_leak:.4f}", "Note": "LEAKAGE, a false result"},
]))

card = """
+--------------------------------------------------------------+
|  A NEW DATASET: 12 STEPS                                     |
+--------------------------------------------------------------+
|   1  shape, dtypes, head()                                   |
|   2  "I predict ___ from ___"                                |
|   3  target type -> regression / classification              |
|   4  value_counts(normalize=True)  <- YOUR BASELINE          |
|   5  train_test_split(..., stratify=y)   NOW!                |
|   6  DummyClassifier -> write the pass mark down             |
|   7  nunique() -> IDs;  "will I have it?" -> leakage         |
|   8  isna().sum();  object columns -> OneHot                 |
|   9  ColumnTransformer -> Pipeline -> fit                    |
|  10  confusion_matrix, precision, recall, F1                 |
|  11  READ the wrong rows (nlargest by confidence)            |
|  12  ONE change -> back to step 9                            |
+--------------------------------------------------------------+
|  A perfect result (>0.99) = a leakage signal, not a party    |
|  Without the baseline, accuracy means nothing                |
|  The test set is in the safe: it gets opened once            |
+--------------------------------------------------------------+
"""
print(card)