Support Vector Machine (SVM)

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.svm import SVC, LinearSVC
from sklearn.datasets import make_blobs

X_d, y_d = make_blobs(n_samples=60, centers=2, cluster_std=1.1, random_state=7)

svm = SVC(kernel="linear", C=1000).fit(X_d, y_d)   # C katta = qattiq margin
w, b0 = svm.coef_[0], svm.intercept_[0]

rasm, ax = plt.subplots(figsize=(9, 5.5))
ax.scatter(X_d[:, 0], X_d[:, 1], c=y_d, cmap="coolwarm", s=45, edgecolors="k", linewidths=.5)
xx = np.linspace(X_d[:, 0].min() - 1, X_d[:, 0].max() + 1, 100)
yy = -(w[0] * xx + b0) / w[1]
chet = 1 / np.linalg.norm(w)
ax.plot(xx, yy, "k-", lw=2, label="qaror chizig'i")
ax.plot(xx, yy + chet * np.sqrt(1 + (w[0]/w[1])**2), "k--", lw=1, label="margin cheti")
ax.plot(xx, yy - chet * np.sqrt(1 + (w[0]/w[1])**2), "k--", lw=1)
ax.scatter(svm.support_vectors_[:, 0], svm.support_vectors_[:, 1],
           s=260, facecolors="none", edgecolors="green", linewidths=2.2,
           label=f"support vektorlar ({len(svm.support_vectors_)} ta)")
ax.legend(fontsize=9); ax.set_title("SVM: eng keng ko'cha", fontweight="bold")
ax.grid(alpha=.3)
fig("svm_margin.png", caption="Qora chiziq — qaror chegarasi. Shtrix chiziqlar — margin chetlari. Yashil doiralar — support vektorlar.")

print(f"Jami nuqta        : {len(X_d)}")
print(f"Support vektorlar : {len(svm.support_vectors_)}")
print(f"Margin kengligi   : {2/np.linalg.norm(w):.4f}")

# isbot: support vektorlardan boshqasini olib tashlaymiz
sv_idx = svm.support_
svm2 = SVC(kernel="linear", C=1000).fit(X_d[sv_idx], y_d[sv_idx])
print(f"Butun ma'lumotda  : w = {svm.coef_[0].round(4)}, b = {svm.intercept_[0]:.4f}")
print(f"Faqat SV larda    : w = {svm2.coef_[0].round(4)}, b = {svm2.intercept_[0]:.4f}")
print(f"Bir xilmi         : {bool(np.allclose(svm.coef_, svm2.coef_, atol=1e-3))}")

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

df = pd.read_csv(DATA / "uy_ijara.csv")
son = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel"]
X, y = df[son], df["ijaraga_berildi_30k"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

print(f"{'C':>8} {'margin':>10} {'SV soni':>9} {'CV F1':>9} {'izoh'}")
print("-" * 54)
for C in [0.01, 0.1, 1, 10, 100]:
    q = Pipeline([("sc", StandardScaler()), ("m", SVC(kernel="linear", C=C))]).fit(X_tr, y_tr)
    b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", SVC(kernel="linear", C=C))]),
                        X, y, cv=5, scoring="f1")
    m = q.named_steps["m"]
    izoh = "yumshoq margin" if C <= 0.1 else ("qattiq margin" if C >= 100 else "")
    print(f"{C:>8} {'keng' if C <= 0.1 else 'tor':>10} {m.n_support_.sum():>9} {b.mean():>9.4f} {izoh}")

show(pd.DataFrame([
    {"C": "kichik (0.01-0.1)", "Margin": "KENG", "Xatoga munosabat": "Ruxsat beradi",
     "Natija": "Soddaroq model, ko'p support vektor", "Xavf": "Underfitting"},
    {"C": "katta (100+)", "Margin": "TOR", "Xatoga munosabat": "Qattiq jazolaydi",
     "Natija": "Murakkab chegara, kam support vektor", "Xavf": "Overfitting"},
]))

r = np.random.default_rng(42)
n = 600
burchak = r.uniform(0, 2 * np.pi, n)
radius = np.where(r.random(n) < 0.5, r.normal(1, .25, n), r.normal(3, .3, n))
Xh = np.c_[radius * np.cos(burchak), radius * np.sin(burchak)]
yh = (radius > 2).astype(int)

from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier

print(f"{'model':<34} {'CV accuracy':>13} {'±':>8}")
print("-" * 58)
for nom, m in [
    ("LogisticRegression (15-bob)", LogisticRegression()),
    ("DecisionTree (17-bob)", DecisionTreeClassifier(max_depth=6, random_state=42)),
    ("SVM kernel='linear'", SVC(kernel="linear")),
    ("SVM kernel='rbf'", SVC(kernel="rbf")),
]:
    b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", m)]), Xh, yh, cv=5)
    print(f"{nom:<34} {b.mean():>13.4f} {b.std():>8.4f}")

show(pd.DataFrame([
    {"Kernel": "linear", "Qachon": "Ustunlar ko'p, ma'lumot matnli (TF-IDF)",
     "Parametrlar": "C", "Tezlik": "Tez"},
    {"Kernel": "rbf", "Qachon": "SUKUT TANLOV — murakkab chegara kerak bo'lsa",
     "Parametrlar": "C, gamma", "Tezlik": "Sekinroq"},
    {"Kernel": "poly", "Qachon": "Polinom bog'lanish ma'lum bo'lsa",
     "Parametrlar": "C, degree, gamma", "Tezlik": "Sekin"},
    {"Kernel": "sigmoid", "Qachon": "Kamdan-kam ishlatiladi",
     "Parametrlar": "C, gamma", "Tezlik": "Sekin"},
]))

print(f"{'gamma':>10} {'CV accuracy':>13} {'izoh'}")
print("-" * 48)
for g in [0.001, 0.01, 0.1, 1, 10, 100]:
    b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", SVC(kernel="rbf", gamma=g))]),
                        Xh, yh, cv=5)
    izoh = "juda silliq (underfit)" if g <= 0.001 else ("har nuqta o'z oroli (overfit)" if g >= 100 else "")
    print(f"{g:>10} {b.mean():>13.4f} {izoh}")
print()
print("'scale' (sukut) — ma'lumot dispersiyasiga qarab avtomatik tanlanadi:")
b = cross_val_score(Pipeline([("sc", StandardScaler()), ("m", SVC(kernel='rbf', gamma='scale'))]), Xh, yh, cv=5)
print(f"  gamma='scale'  CV accuracy = {b.mean():.4f}")

print("SVM — scaling bilan va busiz:")
for nom, m in [("scaling YO'Q", SVC()),
               ("scaling BOR", Pipeline([("sc", StandardScaler()), ("m", SVC())]))]:
    b = cross_val_score(m, X, y, cv=5, scoring="f1")
    print(f"  {nom:14s} CV F1 = {b.mean():.4f} ±{b.std():.4f}")

import time
from sklearn.datasets import make_classification

print(f"{'qatorlar':>10} {'SVC (rbf)':>12} {'LinearSVC':>12} {'LogisticReg':>13}")
print("-" * 52)
for n_q in [500, 2000, 8000]:
    Xb, yb = make_classification(n_samples=n_q, n_features=20, n_informative=10, random_state=42)
    satr = [f"{n_q:>10}"]
    for m in [SVC(kernel="rbf"), LinearSVC(dual="auto", max_iter=5000), LogisticRegression(max_iter=2000)]:
        t0 = time.perf_counter()
        Pipeline([("sc", StandardScaler()), ("m", m)]).fit(Xb, yb)
        satr.append(f"{(time.perf_counter()-t0)*1000:>11.0f}ms")
    print(" ".join(satr))

show(pd.DataFrame([
    {"Qatorlar": "< 10 000", "Tanlov": "SVC(kernel='rbf')", "Izoh": "Kernel ning kuchidan foydalanish mumkin"},
    {"Qatorlar": "10 000 - 100 000", "Tanlov": "LinearSVC yoki SGDClassifier", "Izoh": "Kernel juda qimmat"},
    {"Qatorlar": "> 100 000", "Tanlov": "LogisticRegression / LightGBM", "Izoh": "SVM umuman mos emas"},
]))

svc = Pipeline([("sc", StandardScaler()), ("m", SVC())]).fit(X_tr, y_tr)
try:
    svc.predict_proba(X_te[:3])
except AttributeError as e:
    print("SVC().predict_proba() ->", type(e).__name__)
    print(f"  {str(e)[:70]}")
print()
svc_p = Pipeline([("sc", StandardScaler()), ("m", SVC(probability=True, random_state=42))]).fit(X_tr, y_tr)
print("probability=True bilan:")
print(f"  {svc_p.predict_proba(X_te[:3])[:, 1].round(4)}")
print()
print("decision_function — SVM ning TABIIY chiqishi (chegaradan masofa):")
print(f"  {svc.decision_function(X_te[:3]).round(4)}")

show(pd.DataFrame([
    {"Klass": "SVC", "Vazifa": "Classification", "Kernel": "Bor", "Qachon": "n < 10 000"},
    {"Klass": "LinearSVC", "Vazifa": "Classification", "Kernel": "Yo'q (faqat chiziqli)", "Qachon": "Katta n, matn"},
    {"Klass": "SVR", "Vazifa": "Regression", "Kernel": "Bor", "Qachon": "Kichik regression"},
    {"Klass": "LinearSVR", "Vazifa": "Regression", "Kernel": "Yo'q", "Qachon": "Katta regression"},
    {"Klass": "OneClassSVM", "Vazifa": "Anomaliya aniqlash", "Kernel": "Bor", "Qachon": "Faqat 'normal' ma'lumot bo'lsa"},
]))

from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from xgboost import XGBClassifier

print(f"{'model':<28} {'CV F1':>9} {'±':>8} {'scaling':>9}")
print("-" * 60)
natijalar = []
for nom, m, sc_kerak in [
    ("LogisticRegression (15)", LogisticRegression(max_iter=2000), True),
    ("DecisionTree (17)", DecisionTreeClassifier(max_depth=6, min_samples_leaf=20, random_state=42), False),
    ("RandomForest (18)", RandomForestClassifier(n_estimators=200, random_state=42), False),
    ("XGBoost (19)", XGBClassifier(n_estimators=200, verbosity=0, random_state=42), False),
    ("KNN (20)", KNeighborsClassifier(n_neighbors=15), True),
    ("GaussianNB (20)", GaussianNB(), False),
    ("SVM rbf (21)", SVC(), True),
]:
    quvur = Pipeline([("sc", StandardScaler()), ("m", m)]) if sc_kerak else m
    b = cross_val_score(quvur, X, y, cv=5, scoring="f1")
    natijalar.append((nom, b.mean(), b.std()))
    print(f"{nom:<28} {b.mean():>9.4f} {b.std():>8.4f} {'ha' if sc_kerak else 'yo`q':>9}")

print()
eng = max(natijalar, key=lambda x: x[1])
tarqalish = max(n[2] for n in natijalar)
yaqin = [n[0] for n in natijalar if eng[1] - n[1] < tarqalish]
print(f"Eng baland     : {eng[0]} ({eng[1]:.4f})")
print(f"Tarqalish ichida: {len(yaqin)} ta model -> {', '.join(yaqin)}")

show(pd.DataFrame([
    {"Tushuncha": "Margin", "Bitta jumla": "Ikki sinf orasidagi eng keng ko'cha"},
    {"Tushuncha": "Support vektorlar", "Bitta jumla": "Faqat chegaradagi nuqtalar modelni aniqlaydi"},
    {"Tushuncha": "C", "Bitta jumla": "Katta C = tor margin = kuchsiz regularizatsiya"},
    {"Tushuncha": "Kernel trick", "Bitta jumla": "Yuqori o'lchovga ko'chirmasdan chiziqsiz chegara"},
    {"Tushuncha": "gamma", "Bitta jumla": "Support vektorning ta'sir radiusi; C bilan birga sozlanadi"},
    {"Tushuncha": "Scaling", "Bitta jumla": "SHART — masofaga tayanadi"},
    {"Tushuncha": "Tezlik", "Bitta jumla": "O(n^2)-O(n^3) — 10 000 qatordan keyin mos emas"},
    {"Tushuncha": "Ehtimollik", "Bitta jumla": "Tabiiy ravishda yo'q; probability=True 5x qimmat"},
]))