KNN and Naive Bayes

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[num], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

pipe = Pipeline([("sc", StandardScaler()), ("m", KNeighborsClassifier(n_neighbors=5))]).fit(X_tr, y_tr)

# show the neighbours of one flat
sc = pipe.named_steps["sc"]
knn = pipe.named_steps["m"]
dist, index = knn.kneighbors(sc.transform(X_te.iloc[[0]]), n_neighbors=5)

print("The flat being checked:")
print(X_te.iloc[0].to_string())
print(f"\nThe true answer: {y_te.iloc[0]}")
print("\nThe 5 nearest neighbours:")
neighbours = X_tr.iloc[index[0]].copy()
neighbours["answer"] = y_tr.iloc[index[0]].values
neighbours["distance"] = dist[0].round(3)
show(neighbours)
print(f"The neighbours' vote: {y_tr.iloc[index[0]].mean():.1%} 'yes' -> prediction {pipe.predict(X_te.iloc[[0]])[0]}")

import time
from sklearn.linear_model import LogisticRegression

print(f"{'model':<24} {'fit time':>12} {'predict time':>15}")
print("-" * 54)
for name, m in [("KNN", Pipeline([("sc", StandardScaler()), ("m", KNeighborsClassifier())])),
                ("LogisticRegression", Pipeline([("sc", StandardScaler()), ("m", LogisticRegression(max_iter=2000))]))]:
    t0 = time.perf_counter(); m.fit(X_tr, y_tr); ft = (time.perf_counter()-t0)*1000
    t0 = time.perf_counter(); m.predict(X_te); pt = (time.perf_counter()-t0)*1000
    print(f"{name:<24} {ft:>10.2f} ms {pt:>13.2f} ms")

print(f"{'case':<22} {'CV F1':>9} {'±':>8}")
print("-" * 40)
b1 = cross_val_score(KNeighborsClassifier(), X, y, cv=5, scoring="f1")
b2 = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", KNeighborsClassifier())]),
                     X, y, cv=5, scoring="f1")
print(f"{'no scaling':<22} {b1.mean():>9.4f} {b1.std():>8.4f}")
print(f"{'scaling':<22} {b2.mean():>9.4f} {b2.std():>8.4f}")
print(f"{'gap':<22} {b2.mean()-b1.mean():>+9.4f}")

from sklearn.metrics import f1_score
print(f"{'K':>5} {'train F1':>10} {'test F1':>9} {'CV F1':>9} {'note'}")
print("-" * 52)
for k in [1, 3, 5, 15, 50, 200]:
    q = Pipeline([("sc", StandardScaler()), ("m", KNeighborsClassifier(n_neighbors=k))]).fit(X_tr, y_tr)
    tr_f = f1_score(y_tr, q.predict(X_tr))
    te_f = f1_score(y_te, q.predict(X_te))
    cv = cross_val_score(Pipeline([("sc", StandardScaler()),
                                   ("m", KNeighborsClassifier(n_neighbors=k))]),
                         X, y, cv=5, scoring="f1").mean()
    note = "K=1: every point finds itself" if k == 1 else ("very smooth" if k >= 200 else "")
    print(f"{k:>5} {tr_f:>10.4f} {te_f:>9.4f} {cv:>9.4f} {note}")

show(pd.DataFrame([
    {"K": "small (1-3)", "Model": "Very flexible", "Risk": "Overfitting, sensitive to noise"},
    {"K": "medium (5-20)", "Model": "Balanced", "Risk": "Usually the best"},
    {"K": "large (100+)", "Model": "Very smooth", "Risk": "Underfitting: the same answer everywhere"},
]))
print()
print("Rule: make K an ODD number (so there are no tied votes in binary classification).")

print(f"{'metric':<16} {'CV F1':>9}  note")
print("-" * 58)
for name, p in [("euclidean", 2), ("manhattan", 1)]:
    b = cross_val_score(Pipeline([("sc", StandardScaler()),
                                  ("m", KNeighborsClassifier(n_neighbors=15, p=p))]),
                        X, y, cv=5, scoring="f1")
    note = "sqrt(sum(d^2)): the default" if p == 2 else "sum(|d|): robust to outliers"
    print(f"{name:<16} {b.mean():>9.4f}  {note}")
print()
b_w = cross_val_score(Pipeline([("sc", StandardScaler()),
                                ("m", KNeighborsClassifier(n_neighbors=15, weights="distance"))]),
                      X, y, cv=5, scoring="f1")
print(f"{'weights=distance':<16} {b_w.mean():>9.4f}  nearer neighbours get more of a vote")

r = np.random.default_rng(42)
print("Distances between random points, as the dimension grows:")
print(f"{'dimension':>9} {'nearest':>11} {'farthest':>10} {'ratio':>8}")
print("-" * 42)
for m in [2, 10, 50, 200, 1000]:
    P = r.normal(size=(500, m))
    d = np.linalg.norm(P[1:] - P[0], axis=1)
    print(f"{m:>9} {d.min():>11.3f} {d.max():>10.3f} {d.max()/d.min():>8.2f}")

print("If we add noise columns to KNN:")
print(f"{'noise columns added':>20} {'CV F1':>9}")
print("-" * 32)
for n_noise in [0, 10, 50, 200]:
    if n_noise == 0:
        Xn = X
    else:
        noise = pd.DataFrame(r.normal(size=(len(X), n_noise)),
                             columns=[f"n_{i}" for i in range(n_noise)], index=X.index)
        Xn = pd.concat([X, noise], axis=1)
    b = cross_val_score(Pipeline([("sc", StandardScaler()),
                                  ("m", KNeighborsClassifier(n_neighbors=15))]),
                        Xn, y, cv=5, scoring="f1")
    print(f"{n_noise:>20} {b.mean():>9.4f}")

from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB

print(f"{'model':<24} {'CV F1':>9} {'fit time':>12}")
print("-" * 48)
for name, m in [("GaussianNB", GaussianNB()),
                ("KNN (K=15)", Pipeline([("sc", StandardScaler()),
                                         ("m", KNeighborsClassifier(n_neighbors=15))])),
                ("LogisticRegression", Pipeline([("sc", StandardScaler()),
                                                 ("m", LogisticRegression(max_iter=2000))]))]:
    t0 = time.perf_counter()
    b = cross_val_score(m, X, y, cv=5, scoring="f1")
    dt = (time.perf_counter()-t0)*1000
    print(f"{name:<24} {b.mean():>9.4f} {dt:>10.1f} ms")

show(pd.DataFrame([
    {"Variant": "GaussianNB", "Kind of data": "Continuous numbers",
     "Assumption": "Each feature is normally distributed", "Example": "Height, weight, price"},
    {"Variant": "MultinomialNB", "Kind of data": "Counts",
     "Assumption": "Word frequencies", "Example": "SPAM FILTER: text classification"},
    {"Variant": "BernoulliNB", "Kind of data": "Binary (0/1)",
     "Assumption": "A word is present/absent", "Example": "Short texts"},
]))

from sklearn.feature_extraction.text import CountVectorizer

messages = [
    "Congratulations you won claim your cash now",
    "Free loan fast cash claim apply",
    "The meeting is tomorrow at 10",
    "The report is ready please review it",
    "Prize cash reward free claim",
    "Project documents were sent please take a look",
    "Free bonus claim cash now",
    "The class timetable changed I will let you know",
]
label = [1, 1, 0, 0, 1, 0, 1, 0]        # 1 = spam

vec = CountVectorizer()
Xt = vec.fit_transform(messages)
nb = MultinomialNB().fit(Xt, label)

new = ["free cash claim now", "the report will be ready tomorrow"]
e = nb.predict_proba(vec.transform(new))
for text, p in zip(new, e):
    print(f'  "{text}"')
    print(f"     spam probability: {p[1]:.4f}  ->  {'SPAM' if p[1] > 0.5 else 'normal message'}")

print("The effect of smoothing:")
unknown = ["a message written in completely unfamiliar words"]
for a in [1.0, 1e-10]:
    nb_a = MultinomialNB(alpha=a).fit(Xt, label)
    p = nb_a.predict_proba(vec.transform(unknown))[0]
    print(f"  alpha={a:<8} spam probability = {p[1]:.6f}")

show(pd.DataFrame([
    {"Model": "KNN", "Good when": "Small data, few columns, local patterns",
     "Bad when": "Many columns, big data, real time",
     "Scaling": "REQUIRED"},
    {"Model": "GaussianNB", "Good when": "You need a very fast baseline",
     "Bad when": "The features are strongly related",
     "Scaling": "Not needed"},
    {"Model": "MultinomialNB", "Good when": "TEXT classification, spam, little data",
     "Bad when": "Continuous numbers",
     "Scaling": "Not needed"},
]))

show(pd.DataFrame([
    {"Concept": "The KNN idea", "In one sentence": "Learns nothing: it looks at the neighbours"},
    {"Concept": "The cost of KNN", "In one sentence": "Fast fit, slow predict; it keeps the whole training set"},
    {"Concept": "KNN and scaling", "In one sentence": "Without it, the large-scale column drowns out the rest"},
    {"Concept": "The role of K", "In one sentence": "Small K overfits, large K underfits"},
    {"Concept": "The curse of dimensionality", "In one sentence": "With many columns, every point is equally far away"},
    {"Concept": "Naive Bayes", "In one sentence": "Bayes' theorem + the false assumption that features are independent"},
    {"Concept": "NB's home", "In one sentence": "Text classification: MultinomialNB"},
    {"Concept": "Smoothing", "In one sentence": "alpha=1 solves the zero-frequency problem"},
]))