Feature engineering and selection

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat = ["tuman"]
X, y = df[num + cat], df["ijaraga_berildi_30k"]

def pipe(num_list, cat_list=cat, model=None):
    return Pipeline([
        ("t", ColumnTransformer([
            ("num", Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), num_list),
            ("cat", OneHotEncoder(handle_unknown="ignore"), cat_list),
        ])),
        ("m", model or LogisticRegression(max_iter=3000)),
    ])

base_score = cross_val_score(pipe(num), X, y, cv=5, scoring="f1")
print(f"STARTING POINT (7 features): F1 = {base_score.mean():.4f} ±{base_score.std():.4f}")
print()
show(pd.DataFrame([
    {"Method": "1. Ratios and differences", "Example": "price/area, area/rooms"},
    {"Method": "2. Splitting dates", "Example": "year, month, weekday, is it a holiday"},
    {"Method": "3. Grouping (binning)", "Example": "age -> 'young/middle/old'"},
    {"Method": "4. Interactions", "Example": "furnished AND close to the metro"},
    {"Method": "5. Group statistics", "Example": "the district's average price"},
]))

def ratios(d):
    d = d.copy()
    d["area_per_room"] = d.maydon_m2 / d.xonalar
    d["floor_share"] = d.qavat / d.qavatlar_soni
    d["age"] = 2026 - d.qurilgan_yil
    return d

new1 = num + ["area_per_room", "floor_share", "age"]
b1 = cross_val_score(pipe(new1), ratios(X), y, cv=5, scoring="f1")
print(f"Starting point: {base_score.mean():.4f} ±{base_score.std():.4f}")
print(f"+ 3 ratios    : {b1.mean():.4f} ±{b1.std():.4f}")
print(f"Change        : {b1.mean()-base_score.mean():+.4f}  (spread ±{max(b1.std(), base_score.std()):.4f})")

sv = pd.read_csv(DATA / "iflos_savdo.csv")
date = pd.to_datetime(sv.order_date, format="mixed", dayfirst=True, errors="coerce")
tmp = pd.DataFrame({"date": date}).dropna()
tmp["year"] = tmp.date.dt.year
tmp["month"] = tmp.date.dt.month
tmp["weekday"] = tmp.date.dt.dayofweek
tmp["quarter"] = tmp.date.dt.quarter
tmp["day_of_month"] = tmp.date.dt.day
tmp["weekend"] = (tmp.date.dt.dayofweek >= 5).astype(int)
tmp["month_start"] = (tmp.date.dt.day <= 5).astype(int)
show(tmp.head(6))
print()
print("One `date` column produced 7 features.")

print("Cancellation rate by weekday: is there a periodic pattern:")
sv2 = sv.assign(date=date, cancelled=(sv.status == "cancelled").astype(int)).dropna(subset=["date"])
sv2["weekday"] = sv2.date.dt.dayofweek
days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
g = sv2.groupby("weekday").cancelled.agg(["size", "mean"])
g.index = [days[i] for i in g.index]
g["mean"] = g["mean"].round(4)
show(g.reset_index().rename(columns={"index": "day", "size": "orders", "mean": "cancel_rate"}))

def bins(d):
    d = d.copy()
    d["area_bin"] = pd.cut(d.maydon_m2, bins=[0, 40, 60, 90, 1000],
                           labels=[0, 1, 2, 3]).astype(float)
    d["metro_bin"] = pd.cut(d.metro_km, bins=[0, 1, 2.5, 5, 100],
                            labels=[0, 1, 2, 3]).astype(float)
    return d

new3 = num + ["area_bin", "metro_bin"]
b3 = cross_val_score(pipe(new3), bins(X), y, cv=5, scoring="f1")
print(f"Starting point: {base_score.mean():.4f}")
print(f"+ 2 bins      : {b3.mean():.4f}  ({b3.mean()-base_score.mean():+.4f})")

def interact(d):
    d = d.copy()
    d["furnished_x_metro"] = d.mebel * (d.metro_km < 2).astype(int)
    d["big_and_central"] = ((d.maydon_m2 > 70) & (d.metro_km < 2)).astype(int)
    return d

new4 = num + ["furnished_x_metro", "big_and_central"]
b4 = cross_val_score(pipe(new4), interact(X), y, cv=5, scoring="f1")
print(f"Starting point   : {base_score.mean():.4f}")
print(f"+ 2 interactions : {b4.mean():.4f}  ({b4.mean()-base_score.mean():+.4f})")
print()
print("A tree finds combinations like this ITSELF (Chapter 18); a linear model does NOT.")

# Target encoding: CARDINALITY decides everything
def te_score(X_extra):
    q = Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler()),
                  ("m", LogisticRegression(max_iter=3000))])
    return cross_val_score(q, X_extra, y, cv=5, scoring="f1").mean()

r2 = np.random.default_rng(0)
num_only = df[num]

# 1) low cardinality: tuman, 8 groups
te_8 = num_only.assign(te=df.tuman.map(df.groupby("tuman").ijaraga_berildi_30k.mean()))
# 2) high cardinality: 400 artificial groups
group_400 = r2.integers(0, 400, len(df))
te_400 = num_only.assign(te=pd.Series(y.values).groupby(group_400).transform("mean").values)
# 3) the limit case: every row in its own group
te_1200 = num_only.assign(te=y.values.astype(float))

print(f"{'case':<34} {'rows per group':>14} {'CV F1':>9}")
print("-" * 60)
print(f"{'baseline (NO target encoding)':<34} {'-':>14} {te_score(num_only):>9.4f}")
print(f"{'8 groups (tuman)':<34} {len(df)//8:>14} {te_score(te_8):>9.4f}")
print(f"{'400 groups':<34} {len(df)//400:>14} {te_score(te_400):>9.4f}")
print(f"{'1200 groups (each row itself)':<34} {1:>14} {te_score(te_1200):>9.4f}")

from sklearn.base import BaseEstimator, TransformerMixin

class GroupStats(BaseEstimator, TransformerMixin):
    """Computes the group mean ONLY from the data fit() has seen."""
    def __init__(self, group_col="tuman", target_col="narx_usd"):
        self.group_col = group_col
        self.target_col = target_col

    def fit(self, X, y=None):
        self.map_ = X.groupby(self.group_col)[self.target_col].mean()
        self.overall_ = X[self.target_col].mean()
        return self

    def transform(self, X):
        d = X.copy()
        d["group_mean"] = d[self.group_col].map(self.map_).fillna(self.overall_)
        d["diff_from_group"] = d[self.target_col] - d["group_mean"]
        return d

q_clean = Pipeline([
    ("stat", GroupStats()),
    ("t", ColumnTransformer([
        ("num", Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]),
         num + ["group_mean", "diff_from_group"]),
        ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
    ])),
    ("m", LogisticRegression(max_iter=3000)),
])
X_price = df[num + cat + ["narx_usd"]]
b5_clean = cross_val_score(q_clean, X_price, y, cv=5, scoring="f1")
print(f"Starting point                     : {base_score.mean():.4f}")
print(f"+ group stats (inside the Pipeline): {b5_clean.mean():.4f} ±{b5_clean.std():.4f}")
print()
print("You CAN trust this number: the statistics are recomputed in every fold.")

results = [
    ("Starting point (7 features)", base_score),
    ("+ ratios", b1),
    ("+ binning", b3),
    ("+ interactions", b4),
    ("+ group stats (in the Pipeline)", b5_clean),
]
rows = []
for name, b in results:
    rows.append({"Variant": name, "CV F1": round(b.mean(), 4), "±": round(b.std(), 4),
                 "vs start": round(b.mean() - base_score.mean(), 4)})
show(pd.DataFrame(rows))
print()
spread = max(b.std() for _, b in results)
best = max(results, key=lambda x: x[1].mean())
print(f"Highest: {best[0]} ({best[1].mean():.4f})")
print(f"Largest spread: ±{spread:.4f}")
print(f"Verdict: {'the gap is LARGER than the spread' if best[1].mean()-base_score.mean() > spread else 'the gap is WITHIN the spread: the features gave no noticeable gain'}")

r = np.random.default_rng(42)
noise = pd.DataFrame(r.normal(size=(len(X), 40)),
                     columns=[f"noise_{i}" for i in range(40)], index=X.index)
X_many = pd.concat([X[num], noise], axis=1)
print(f"Features: {len(num)} real + 40 noise = {X_many.shape[1]}")
print()
show(pd.DataFrame([
    {"Family": "Filter", "How it works": "Scores with statistics, no model",
     "sklearn": "SelectKBest, VarianceThreshold", "Speed": "Very fast"},
    {"Family": "Wrapper", "How it works": "Tries by training the model again and again",
     "sklearn": "RFE, RFECV", "Speed": "Slow"},
    {"Family": "Embedded", "How it works": "The model selects while it trains",
     "sklearn": "Lasso, tree importance", "Speed": "Fast (free)"},
]))

from sklearn.feature_selection import SelectKBest, f_classif, RFE
from sklearn.ensemble import RandomForestClassifier
import time

def simple_pipe(sel=None, model=None):
    # NOTE: the imputer has to come FIRST, because SelectKBest doesn't accept NaN
    steps = [("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]
    if sel is not None:
        steps.append(("sel", sel))
    steps.append(("m", model or LogisticRegression(max_iter=3000)))
    return Pipeline(steps)

print(f"{'method':<34} {'CV F1':>9} {'±':>8} {'time':>8}")
print("-" * 62)
for name, sel in [
    ("No selection (47 features)", None),
    ("Filter: SelectKBest(k=7)", SelectKBest(f_classif, k=7)),
    ("Wrapper: RFE(n=7)", RFE(LogisticRegression(max_iter=2000), n_features_to_select=7)),
]:
    t0 = time.perf_counter()
    b = cross_val_score(simple_pipe(sel), X_many, y, cv=5, scoring="f1")
    dt = time.perf_counter() - t0
    print(f"{name:<34} {b.mean():>9.4f} {b.std():>8.4f} {dt:>7.1f}s")

# did the selection find the REAL features?
Xk_clean = SimpleImputer(strategy="median").fit_transform(X_many)
sel = SelectKBest(f_classif, k=7).fit(StandardScaler().fit_transform(Xk_clean), y)
chosen = X_many.columns[sel.get_support()].tolist()
real = [t for t in chosen if not t.startswith("noise")]
print(f"SelectKBest chose 7: {chosen}")
print(f"  real ones among them: {len(real)} / 7")
print()
rfe = RFE(LogisticRegression(max_iter=2000), n_features_to_select=7).fit(
    StandardScaler().fit_transform(Xk_clean), y)
t_rfe = X_many.columns[rfe.get_support()].tolist()
print(f"RFE chose 7: {t_rfe}")
print(f"  real ones among them: {len([t for t in t_rfe if not t.startswith('noise')])} / 7")

show(pd.DataFrame([
    {"Method": "Ratios and differences", "Who it helps": "Linear models", "Watch out": "A tree finds them itself"},
    {"Method": "Splitting dates", "Who it helps": "Everyone", "Watch out": "Don't pass a date as a number"},
    {"Method": "Binning", "Who it helps": "Linear models", "Watch out": "Information is lost"},
    {"Method": "Interactions", "Who it helps": "Linear models", "Watch out": "A tree finds them itself"},
    {"Method": "Group statistics", "Who it helps": "Everyone: the strongest", "Watch out": "The highest LEAKAGE risk"},
    {"Method": "Filter selection", "Who it helps": "Fast screening", "Watch out": "Ignores the model"},
    {"Method": "Wrapper (RFE)", "Who it helps": "Precise selection", "Watch out": "Slow"},
    {"Method": "Embedded (Lasso)", "Who it helps": "Free (Chapter 17)", "Watch out": "You need to find alpha"},
]))