The confusion-solver

import warnings
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

rent = pd.read_csv(DATA / "uy_ijara.csv")
NUM = ["maydon_m2", "xonalar", "qavat", "metro_km"]
X = rent[NUM].fillna(rent[NUM].median())
y = rent["ijaraga_berildi_30k"]
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
print(f"For the examples: train {len(Xtr)}, test {len(Xte)}")

from sklearn.preprocessing import StandardScaler

right = StandardScaler().fit(Xtr)
wrong = StandardScaler().fit(pd.concat([Xtr, Xte]))        # it saw the test set too

print("RIGHT: the scaler only sees train:")
print(f"  train mean: {right.mean_[0]:.4f}")
print(f"  test mean (after transform): {right.transform(Xte)[:, 0].mean():.4f}")
print("  the test mean is NOT 0, and that's correct: test is a different distribution")
print()
print("WRONG: the scaler saw the test set too:")
print(f"  mean: {wrong.mean_[0]:.4f}  <- test data mixed in")
print()
print("Rule: fit ONLY on train. On test, only transform.")
print("If you use a Pipeline, this mistake becomes IMPOSSIBLE.")

from sklearn.linear_model import LogisticRegression

model = LogisticRegression(max_iter=2000).fit(Xtr, ytr)
proba = model.predict_proba(Xte)[:, 1]
pred = model.predict(Xte)

print(f"predict_proba shape : {model.predict_proba(Xte).shape}  (a column per class)")
print(f"predict shape       : {pred.shape}")
print()
compare = pd.DataFrame({"proba_1": proba[:6].round(4), "predict": pred[:6]})
show(compare)
print(f"Checking that predict = (proba >= 0.5): {bool(((proba >= 0.5).astype(int) == pred).all())}")
print()
print("In other words, predict() is predict_proba() with a 0.5 threshold on top.")
print("We saw in Chapter 9: 0.5 isn't a magic number; you can and should change it.")

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# build an imbalanced case: 10% positive
r = np.random.default_rng(0)
imbalanced_y = np.zeros(1000, dtype=int)
imbalanced_y[r.choice(1000, 100, replace=False)] = 1
no_to_all = np.zeros(1000, dtype=int)

show(pd.DataFrame([
    {"metric": "accuracy", "value": round(accuracy_score(imbalanced_y, no_to_all), 4),
     "meaning": "the share of correct answers"},
    {"metric": "precision", "value": round(precision_score(imbalanced_y, no_to_all, zero_division=0), 4),
     "meaning": "of my 'yes' answers, how many were really yes"},
    {"metric": "recall", "value": round(recall_score(imbalanced_y, no_to_all, zero_division=0), 4),
     "meaning": "of the real 'yes' cases, how many I found"},
    {"metric": "f1", "value": round(f1_score(imbalanced_y, no_to_all, zero_division=0), 4),
     "meaning": "the balance of precision and recall"},
]))
print("The model said 'no' to EVERYONE: it learned nothing.")
print("Accuracy is high and everything else is zero. That's why looking at accuracy")
print("on imbalanced data is fooling yourself (Chapters 8 and 9).")

m = LogisticRegression(C=0.5, max_iter=2000).fit(Xtr, ytr)
print("HYPERPARAMETERS: you choose them, the model doesn't learn them:")
print(f"  C         = {m.C}")
print(f"  max_iter  = {m.max_iter}")
print()
print("PARAMETERS: the model learns them from data, you don't choose them:")
print(f"  coef_     = {m.coef_[0].round(4)}")
print(f"  intercept_= {m.intercept_.round(4)}")
print()
print("An easy sign: if the name ends in _ (coef_, mean_), it's a LEARNED parameter.")
print("If you write it in the constructor (C=0.5), it's a hyperparameter. Chapter 24 tunes them.")

from sklearn.preprocessing import MinMaxScaler

std = StandardScaler().fit_transform(Xtr)
mnm = MinMaxScaler().fit_transform(Xtr)
show(pd.DataFrame([
    {"method": "StandardScaler", "mean": round(float(std.mean()), 4),
     "std": round(float(std.std()), 4), "min": round(float(std.min()), 2),
     "max": round(float(std.max()), 2), "when": "when the distribution is close to normal"},
    {"method": "MinMaxScaler", "mean": round(float(mnm.mean()), 4),
     "std": round(float(mnm.std()), 4), "min": round(float(mnm.min()), 2),
     "max": round(float(mnm.max()), 2), "when": "when you need exact bounds (images: 0..1)"},
]))
print("StandardScaler: mean 0, std 1, but NO bounds.")
print("MinMaxScaler: within 0..1, but one extreme value squeezes everything else.")

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

show(pd.DataFrame([
    {"aspect": "How the trees are built", "Bagging (RandomForest)": "IN PARALLEL, independently",
     "Boosting": "IN SEQUENCE, each fixes the previous one's mistakes"},
    {"aspect": "What it reduces", "Bagging (RandomForest)": "variance (spread)",
     "Boosting": "bias (systematic error)"},
    {"aspect": "Overfitting risk", "Bagging (RandomForest)": "low",
     "Boosting": "high: the number of steps is controlled"},
    {"aspect": "Parallelising", "Bagging (RandomForest)": "easy (n_jobs=-1)",
     "Boosting": "hard: the sequence is required"},
]))
rf = RandomForestClassifier(n_estimators=100, random_state=0).fit(Xtr, ytr)
gb = GradientBoostingClassifier(random_state=0).fit(Xtr, ytr)
print(f"RandomForest accuracy : {rf.score(Xte, yte):.4f}")
print(f"GradientBoosting      : {gb.score(Xte, yte):.4f}")
print("(Chapters 19 and 20: which one wins depends on the data; there's no rule)")

from sklearn.metrics import log_loss

a = LogisticRegression(C=100, max_iter=2000).fit(Xtr, ytr)
b = LogisticRegression(C=0.01, max_iter=2000).fit(Xtr, ytr)
show(pd.DataFrame([
    {"model": "C=100", "accuracy": round(a.score(Xte, yte), 4),
     "log_loss": round(log_loss(yte, a.predict_proba(Xte)), 4)},
    {"model": "C=0.01", "accuracy": round(b.score(Xte, yte), 4),
     "log_loss": round(log_loss(yte, b.predict_proba(Xte)), 4)},
]))
print("LOSS: the model minimises it during training; it's continuous and has a derivative.")
print("METRIC: you evaluate with it; it has a business meaning and may have no derivative.")
print("The two may not rank models the same way, so watch both.")

show(pd.DataFrame([
    {"set": "train", "how many times it's used": "many", "what for": "learning the model's weights"},
    {"set": "validation", "how many times it's used": "many", "what for": "choosing hyperparameters, choosing models"},
    {"set": "test", "how many times it's used": "ONCE", "what for": "the final honest estimate"},
]))
print("If you make any decision by looking at the test set, it is now validation.")
print("And you no longer have an honest estimate. Chapter 6 is about this.")

from sklearn.linear_model import Lasso, Ridge
from sklearn.preprocessing import StandardScaler as SS

Xs = SS().fit_transform(Xtr)
yr = rent.loc[Xtr.index, "narx_usd"]

rows = []
for alpha in [1, 10, 50, 100, 200, 400]:
    l1 = Lasso(alpha=alpha, max_iter=10000).fit(Xs, yr)
    l2 = Ridge(alpha=alpha).fit(Xs, yr)
    rows.append({"alpha": alpha,
                 "L1 zero coefficients": int((l1.coef_ == 0).sum()),
                 "L2 zero coefficients": int((l2.coef_ == 0).sum()),
                 "L1 coefficients": str(l1.coef_.round(1))})
show(pd.DataFrame(rows))
print(f"Number of columns: {len(NUM)}")

rf2 = RandomForestClassifier(n_estimators=100, random_state=0).fit(Xtr, ytr)
show(pd.DataFrame({"column": NUM,
                   "correlation (with the target)": [round(float(np.corrcoef(Xtr[c], ytr)[0, 1]), 4) for c in NUM],
                   "feature importance": rf2.feature_importances_.round(4)}))
print("Three DIFFERENT things:")
print("  correlation  : the strength of a linear relationship")
print("  importance   : how much the model used this column")
print("  causality    : whether the column CHANGES the target")
print()
print("If a model reports importance, that does NOT mean cause. To establish a cause")
print("you need an experiment (an A/B test); observational data isn't enough.")

print(f"pandas version: {pd.__version__}")
orig = rent[["maydon_m2", "narx_usd"]].head(5).copy()
print("\nOriginal prices:", list(orig.narx_usd))

piece = orig[orig.maydon_m2 > 0]      # this is a NEW object
with warnings.catch_warnings(record=True) as caught:
    warnings.simplefilter("always")
    piece["narx_usd"] = 0
    kinds = [w.category.__name__ for w in caught]

print(f"Warning          : {kinds or 'NONE'}")
print(f"piece changed    : {bool((piece.narx_usd == 0).all())}")
print(f"original changed : {bool((orig.narx_usd == 0).any())}")
print("\nThe original prices are still:", list(orig.narx_usd))

orig2 = rent[["maydon_m2", "narx_usd"]].head(5).copy()
orig2.loc[orig2.maydon_m2 > 0, "narx_usd"] = 0
print(f"With .loc, the original table changed: {bool((orig2.narx_usd == 0).all())}")
print("Rule: NOT filter, then assign. Do it in one .loc operation.")

show(pd.DataFrame([
    {"kind of data": "Independent rows (flats, customers)", "shuffle": "ALLOWED",
     "split": "train_test_split(shuffle=True)"},
    {"kind of data": "A time series", "shuffle": "NOT ALLOWED",
     "split": "chronological, or TimeSeriesSplit (Chapter 35)"},
    {"kind of data": "Grouped (one flat, many listings)", "shuffle": "CAREFUL",
     "split": "GroupShuffleSplit (Chapter 38)"},
    {"kind of data": "Imbalanced classes", "shuffle": "ALLOWED",
     "split": "with stratify=y"},
]))

show(pd.DataFrame([
    {"Error": "could not convert string to float", "Cause": "a text column reached the model",
     "Fix": "OneHotEncoder or ColumnTransformer (Chapter 5)"},
    {"Error": "Input contains NaN", "Cause": "a missing value is left",
     "Fix": "SimpleImputer, inside the Pipeline (Chapter 5)"},
    {"Error": "X has N features, but expects M", "Cause": "train and test have different columns",
     "Fix": "save the Pipeline, check the columns (Chapter 40)"},
    {"Error": "ConvergenceWarning", "Cause": "max_iter ran out or there's no scaling",
     "Fix": "add scaling, then raise max_iter (Chapter 16)"},
    {"Error": "columns are missing", "Cause": "a column is missing from the incoming frame",
     "Fix": "check it against a contract (Chapters 38, 41)"},
    {"Error": "Found input variables with inconsistent numbers", "Cause": "X and y have different lengths",
     "Fix": "when you filter, filter both together"},
]))

show(pd.DataFrame([
    {"Question": "Can I use fit_transform on test?", "Answer": "No: only transform"},
    {"Question": "Is the 0.5 threshold mandatory?", "Answer": "No, it's just the default value"},
    {"Question": "Is accuracy a good metric?", "Answer": "If the classes are balanced, yes"},
    {"Question": "How much data do I need?", "Answer": "The learning curve tells you (Chapter 11)"},
    {"Question": "Is deep learning always better?", "Answer": "No: this book measured that several times"},
    {"Question": "Does feature importance show the cause?", "Answer": "No"},
    {"Question": "How many times do I use the test set?", "Answer": "Once"},
    {"Question": "My model got 0.95, is that good?", "Answer": "Tell me what the baseline got (Chapter 7)"},
]))