Cheat sheet and protocol card

import pandas as pd

show(pd.DataFrame([
    {"#": 1, "Step": "Open the file, look at its shape", "Code": "df.shape, df.head(), df.info()"},
    {"#": 2, "Step": "Identify the target", "Code": "df[target].value_counts()"},
    {"#": 3, "Step": "Decide the task type", "Code": "number of classes -> classification / regression"},
    {"#": 4, "Step": "Count the missing values", "Code": "df.isna().sum()"},
    {"#": 5, "Step": "Drop the trap columns", "Code": "IDs, leakage, numbers stored as text"},
    {"#": 6, "Step": "SPLIT (before everything)", "Code": "train_test_split(stratify=y)"},
    {"#": 7, "Step": "Build a baseline", "Code": "DummyClassifier / DummyRegressor"},
    {"#": 8, "Step": "The first model, with a Pipeline", "Code": "Pipeline([preprocessing, model])"},
    {"#": 9, "Step": "Choose the right metric", "Code": "F1 / PR-AUC / MAE"},
    {"#": 10, "Step": "Cross-validation", "Code": "cross_val_score(cv=5)"},
    {"#": 11, "Step": "Error analysis", "Code": "which rows does it get wrong?"},
    {"#": 12, "Step": "Save and document", "Code": "joblib + manifest + DATASHEET"},
]))

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.dummy import DummyClassifier

df = pd.read_csv("new.csv")
print(df.shape)
print(df.dtypes.to_string())
print(df.isna().sum().to_string())
print(df[TARGET].value_counts(normalize=True))

# how many unique values per column: exposes the ID and number-as-text traps
for c in df.columns:
    print(f"{c:<24} unique={df[c].nunique():<6} dtype={df[c].dtype}")

y = df[TARGET]
X = df.drop(columns=[TARGET])
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2,
                                      random_state=42, stratify=y)
baseline = DummyClassifier(strategy="most_frequent").fit(Xtr, ytr)
print("BASELINE:", baseline.score(Xte, yte))

from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder

NUM = X.select_dtypes("number").columns.tolist()
CAT = X.select_dtypes("object").columns.tolist()

prep = ColumnTransformer([
    ("num", Pipeline([("imp", SimpleImputer(strategy="median")),
                      ("sc", StandardScaler())]), NUM),
    ("cat", Pipeline([("imp", SimpleImputer(strategy="most_frequent")),
                      ("oh", OneHotEncoder(handle_unknown="ignore"))]), CAT),
])

pipe = Pipeline([("prep", prep), ("model", MODEL)])
pipe.fit(Xtr, ytr)

show(pd.DataFrame([
    {"Situation": "Tabular, little data (<1000)", "First choice": "LogisticRegression / Ridge",
     "Then": "RandomForest"},
    {"Situation": "Tabular, lots of data", "First choice": "LightGBM / XGBoost",
     "Then": "stacking"},
    {"Situation": "Many categorical columns", "First choice": "CatBoost",
     "Then": "target encoding + boosting"},
    {"Situation": "Text", "First choice": "TF-IDF + LogisticRegression",
     "Then": "a fine-tuned transformer"},
    {"Situation": "Images", "First choice": "a pretrained CNN (transfer learning)",
     "Then": "fine-tuning"},
    {"Situation": "A time series", "First choice": "naive baseline + lags + Ridge",
     "Then": "boosting (on differenced data)"},
    {"Situation": "An explanation is required", "First choice": "a linear model / a single tree",
     "Then": "boosting with SHAP"},
]))

show(pd.DataFrame([
    {"Case": "Classification, balanced", "Metric": "accuracy, ROC-AUC"},
    {"Case": "Classification, imbalanced", "Metric": "F1, PR-AUC, recall@precision"},
    {"Case": "Asymmetric cost of mistakes", "Metric": "custom: weights on FN and FP"},
    {"Case": "Regression with outliers", "Metric": "MAE (not RMSE)"},
    {"Case": "Regression where big errors are very bad", "Metric": "RMSE"},
    {"Case": "Relative error matters", "Metric": "MAPE (careful near zero)"},
    {"Case": "Ranking / ordering", "Metric": "NDCG, MAP"},
]))

show(pd.DataFrame([
    {"Sign": "Train low, test low", "Diagnosis": "underfitting",
     "Action": "a more complex model, more features, less regularisation"},
    {"Sign": "Train high, test low", "Diagnosis": "overfitting",
     "Action": "more data, regularisation, a simpler model"},
    {"Sign": "Both very high", "Diagnosis": "LIKELY LEAKAGE",
     "Action": "look at feature importance, remove the column and measure"},
    {"Sign": "CV is scattered (large std)", "Diagnosis": "too little or uneven data",
     "Action": "more folds, stratification, more data"},
    {"Sign": "One class is never found", "Diagnosis": "imbalance",
     "Action": "class_weight, the threshold, change the metric"},
    {"Sign": "Good offline, bad in production", "Diagnosis": "drift or a pipeline difference",
     "Action": "compare the input distribution, check the Pipeline is identical"},
]))

show(pd.DataFrame([
    {"Topic": "Split", "Reminder": "BEFORE everything. stratify=y for classification"},
    {"Topic": "Scaler", "Reminder": "fit on train only. Inside a Pipeline it's not a problem"},
    {"Topic": "Time series", "Reminder": "A chronological split, TimeSeriesSplit, differencing"},
    {"Topic": "Grouped data", "Reminder": "GroupShuffleSplit: one object stays in one part"},
    {"Topic": "Trees", "Reminder": "No scaling needed"},
    {"Topic": "KNN, SVM, NN", "Reminder": "Scaling is REQUIRED"},
    {"Topic": "random_state", "Reminder": "Set it everywhere, for reproducibility"},
    {"Topic": "Neural networks", "Reminder": "At least 3 seeds, and record the worst one too"},
    {"Topic": "The test set", "Reminder": "Once. Look a second time and it's validation"},
    {"Topic": "Saving", "Reminder": "Pipeline + manifest + a round-trip test"},
]))

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import (train_test_split, cross_val_score,
                                     GridSearchCV, StratifiedKFold,
                                     GroupShuffleSplit, TimeSeriesSplit)
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.dummy import DummyClassifier, DummyRegressor
from sklearn.linear_model import LogisticRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import (accuracy_score, precision_score, recall_score,
                             f1_score, roc_auc_score, average_precision_score,
                             confusion_matrix, classification_report,
                             mean_absolute_error, r2_score)
import joblib

show(pd.DataFrame([
    {"Rule": "1. The baseline first", "Why": "Without a baseline, a result means nothing"},
    {"Rule": "2. Split before everything", "Why": "Every step done after it risks leakage"},
    {"Rule": "3. A Pipeline is mandatory", "Why": "It stops leakage at the level of structure"},
    {"Rule": "4. Every number must be measured", "Why": "A guessed number is a lie"},
    {"Rule": "5. If your guess is rejected, write it down", "Why": "That's where the most valuable knowledge is"},
]))
print()
print("In this book I guessed wrong several times, and measurement rejected me:")
print("the CNN didn't win on centred images, the LSTM came out unstable, TF-IDF")
print("beat the transformer, and the column order turned out not to matter.")
print()
print("Each time I wrote down the MEASUREMENT, not the guess. Do the same.")