Preparing the data: encoding, missing values, scaling

model.fit(df[["tuman", "maydon_m2"]], y)

ValueError: could not convert string to float: 'Yunusobod'

import pandas as pd
import numpy as np

df = pd.read_csv(DATA / "uy_ijara.csv")
X = df.drop(columns=["ijaraga_berildi_30k", "narx_usd"])

print("Column types:")
print(X.dtypes.to_string())
print()
print("List of problems:")
text = X.select_dtypes("object").columns.tolist()
print(f"  1. Text columns (need encoding): {text}")
missing = X.isna().sum()
print(f"  2. Missing values: {missing[missing > 0].to_dict()}")
print(f"  3. Duplicate rows: {int(X.duplicated().sum())}")
scale = X.select_dtypes(include=[np.number]).agg(["min", "max"]).T
scale["range"] = scale["max"] - scale["min"]
print(f"  4. Different scales: smallest range {scale.range.min():.1f}, largest {scale.range.max():.1f}")

show(pd.DataFrame([
    {"Kind": "Ordinal (ordered)", "Example": "small < medium < large;  1st year < 2nd year",
     "Is there an order": "YES", "Method": "OrdinalEncoder or .map()"},
    {"Kind": "Nominal (unordered)", "Example": "district, colour, country",
     "Is there an order": "NO", "Method": "OneHotEncoder"},
]))

from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder

districts = X[["tuman"]].head(6)
print("Original values:")
print(districts.tuman.tolist())
print()

# WRONG: giving a nominal column numbers
ord_code = OrdinalEncoder().fit_transform(X[["tuman"]])
print("If we use OrdinalEncoder:")
for t, k in zip(districts.tuman, ord_code[:6].ravel()):
    print(f"  {t:16s} -> {k:.0f}")
print()
print("PROBLEM: the model now thinks 'Yunusobod (7) > Bektemir (0)'.")
print("         That is MEANINGLESS: districts cannot be ordered.")

# RIGHT: OneHot for a nominal column
oh = OneHotEncoder(sparse_output=False, handle_unknown="ignore")
encoded = oh.fit_transform(X[["tuman"]])
names = oh.get_feature_names_out(["tuman"])
print(f"OneHotEncoder: 1 column -> {encoded.shape[1]} columns")
print()
result = pd.DataFrame(encoded[:6].astype(int), columns=[n.replace("tuman_", "") for n in names])
result.insert(0, "original", districts.tuman.values)
show(result)

print("Proof: when a NEW district reaches the model:")
new = pd.DataFrame({"tuman": ["Uchtepa"]})
print("  with handle_unknown='ignore':", oh.transform(new).astype(int).tolist())
strict = OneHotEncoder(sparse_output=False).fit(X[["tuman"]])
try:
    strict.transform(new)
except Exception as e:
    print("  without handle_unknown     :", type(e).__name__, "->", str(e)[:58])

# A made-up example: put flats into condition categories
X_example = X.copy()
X_example["condition"] = pd.cut(X_example.qurilgan_yil.fillna(X_example.qurilgan_yil.median()),
                                bins=[0, 1990, 2010, 2100],
                                labels=["old", "mid", "new"])
order = {"old": 0, "mid": 1, "new": 2}
X_example["condition_code"] = X_example.condition.map(order)
show(X_example[["qurilgan_yil", "condition", "condition_code"]].head(6))
print()
print("Here the order MEANS something: new > mid > old. So .map() is correct.")

print(f"qurilgan_yil missing: {int(X.qurilgan_yil.isna().sum())} ({X.qurilgan_yil.isna().mean():.2%})")
print()
show(pd.DataFrame([
    {"Method": "Drop the row", "When": "< 1% missing and plenty of data",
     "Cost": "Loses data; if the gaps aren't random, it introduces bias"},
    {"Method": "Fill with the median", "When": "Numeric column, 1-30% missing",
     "Cost": "Reduces the spread; robust to outliers"},
    {"Method": "Fill with the mean", "When": "Numeric column, symmetric distribution",
     "Cost": "Outliers drag the mean"},
    {"Method": "The most frequent value", "When": "Text/category column",
     "Cost": "Makes the big category even bigger"},
    {"Method": "'MISSING' as its own category", "When": "The gap ITSELF means something",
     "Cost": "None; often the best choice"},
    {"Method": "Fill with a model (KNN)", "When": "Many gaps and related columns",
     "Cost": "Slow; high leakage risk"},
]))

from sklearn.impute import SimpleImputer

for strat in ["median", "mean", "most_frequent"]:
    t = SimpleImputer(strategy=strat).fit_transform(X[["qurilgan_yil"]])
    print(f"  {strat:14s} -> filled value: {t[X.qurilgan_yil.isna().values][0][0]:.1f}")
print()
print("An extra column: keeping a 'was missing' flag")
flagged = SimpleImputer(strategy="median", add_indicator=True).fit_transform(X[["qurilgan_yil"]])
print(f"  add_indicator=True -> {flagged.shape[1]} columns (value + missing flag)")

scale = X.select_dtypes(include=[np.number]).agg(["min", "max", "mean", "std"]).T.round(2)
show(scale.reset_index().rename(columns={"index": "column"}))

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

num = ["xonalar", "maydon_m2", "metro_km", "mebel"]
for name, sc in [("StandardScaler", StandardScaler()),
                 ("MinMaxScaler", MinMaxScaler()),
                 ("RobustScaler", RobustScaler())]:
    t = sc.fit_transform(X[num])
    print(f"{name:16s} mean={t.mean():+.3f}  std={t.std():.3f}  "
          f"min={t.min():+.2f}  max={t.max():+.2f}")

show(pd.DataFrame([
    {"Scaler": "StandardScaler", "What it does": "Mean 0, std 1",
     "When": "The default choice; when the distribution is close to normal"},
    {"Scaler": "MinMaxScaler", "What it does": "Squeezes into 0 to 1",
     "When": "Neural networks, when exact bounds are needed"},
    {"Scaler": "RobustScaler", "What it does": "Uses the median and IQR",
     "When": "When there are MANY outliers"},
]))

show(pd.DataFrame([
    {"Model": "KNN", "Scaling": "REQUIRED", "Why": "It computes distances"},
    {"Model": "SVM", "Scaling": "REQUIRED", "Why": "Distances and the margin"},
    {"Model": "Logistic / Linear Regression", "Scaling": "Recommended", "Why": "Gradients converge faster; coefficients become comparable"},
    {"Model": "Ridge / Lasso", "Scaling": "REQUIRED", "Why": "The penalty depends on coefficient size"},
    {"Model": "Neural network", "Scaling": "REQUIRED", "Why": "Exploding/vanishing gradients"},
    {"Model": "Decision Tree", "Scaling": "NOT NEEDED", "Why": "It only asks 'bigger or smaller?'"},
    {"Model": "Random Forest", "Scaling": "NOT NEEDED", "Why": "Built from trees"},
    {"Model": "XGBoost / LightGBM", "Scaling": "NOT NEEDED", "Why": "Built from trees"},
]))

from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

y = df["ijaraga_berildi_30k"]
Xs = X[num]

print(f"{'Model':<16} {'no scaling':>14} {'scaling':>13} {'gap':>8}")
print("-" * 54)
for name, m in [("KNN", KNeighborsClassifier()), ("DecisionTree", DecisionTreeClassifier(max_depth=6, random_state=42))]:
    without = cross_val_score(m, Xs, y, cv=5).mean()
    with_sc = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), Xs, y, cv=5).mean()
    print(f"{name:<16} {without:>14.4f} {with_sc:>13.4f} {with_sc-without:>+8.4f}")

print(f"Duplicates in our file: {int(X.duplicated().sum())}")
print()
print("But careful: ALMOST identical rows are a problem too:")
fake = pd.DataFrame({
    "tuman": ["Chilonzor", "chilonzor", "Chilonzor "],
    "maydon_m2": [55.0, 55.0, 55.0],
})
print(fake.to_string(index=False))
print()
print("Duplicates duplicated() finds:", int(fake.duplicated().sum()))
fake_n = fake.assign(tuman=fake.tuman.str.strip().str.lower())
print("After normalising          :", int(fake_n.duplicated().sum()))

from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LogisticRegression

num_cols = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat_cols = ["tuman"]

prep = ColumnTransformer([
    ("num", Pipeline([
        ("impute", SimpleImputer(strategy="median")),
        ("scale", StandardScaler()),
    ]), num_cols),
    ("cat", Pipeline([
        ("impute", SimpleImputer(strategy="most_frequent")),
        ("encode", OneHotEncoder(handle_unknown="ignore")),
    ]), cat_cols),
], remainder="drop")

pipe = Pipeline([("prep", prep), ("model", LogisticRegression(max_iter=2000))])
score = cross_val_score(pipe, X, y, cv=5, scoring="f1")
print(f"Full pipeline, 5-fold CV F1: {score.mean():.4f} ±{score.std():.4f}")
print()
prep.fit(X, y)
print(f"In : {X.shape[1]} columns")
print(f"Out: {prep.transform(X).shape[1]} columns  (OneHot expanded the district)")

import hashlib

def dataset_passport(frame, name, target):
    digest = hashlib.md5(pd.util.hash_pandas_object(frame, index=True).values).hexdigest()[:16]
    lines = [
        f"Dataset     : {name}",
        f"Rows        : {len(frame):,}".replace(",", " "),
        f"Columns     : {frame.shape[1]}",
        f"Target      : {target}",
        f"Hash (md5)  : {digest}",
        f"Missing     : {int(frame.isna().sum().sum())}",
        f"Duplicates  : {int(frame.duplicated().sum())}",
    ]
    if target in frame:
        n = frame[target].value_counts(normalize=True).sort_index()
        lines.append("Target ratio: " + ", ".join(f"{k}={v:.1%}" for k, v in n.items()))
    return "\n".join(lines)

print(dataset_passport(df, "uy_ijara.csv", "ijaraga_berildi_30k"))

show(pd.DataFrame([
    {"Problem": "A text column", "Fix": "Nominal -> OneHot, Ordinal -> .map()/OrdinalEncoder"},
    {"Problem": "Numbering a nominal column", "Fix": "It invents an order; DON'T"},
    {"Problem": "A new category shows up", "Fix": "handle_unknown='ignore'"},
    {"Problem": "Missing values", "Fix": "Median/mode/own category, depending on what the gap means"},
    {"Problem": "Different scales", "Fix": "StandardScaler; trees don't need it"},
    {"Problem": "Almost identical rows", "Fix": "Normalise first, then drop_duplicates"},
    {"Problem": "Tying it all together", "Fix": "ColumnTransformer + Pipeline, remainder='drop'"},
    {"Problem": "Remembering in three months", "Fix": "Dataset passport + md5 hash"},
]))