Chalkashlik-yechgich

import warnings
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

uy = pd.read_csv(DATA / "uy_ijara.csv")
SON = ["maydon_m2", "xonalar", "qavat", "metro_km"]
X = uy[SON].fillna(uy[SON].median())
y = uy["ijaraga_berildi_30k"]
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
print(f"Misollar uchun: train {len(Xtr)}, test {len(Xte)}")

from sklearn.preprocessing import StandardScaler

togri = StandardScaler().fit(Xtr)
xato = StandardScaler().fit(pd.concat([Xtr, Xte]))        # test ni ham ko'rdi

print("TO'G'RI — scaler faqat train ni ko'radi:")
print(f"  train o'rtachasi: {togri.mean_[0]:.4f}")
print(f"  test o'rtachasi (transform dan keyin): {togri.transform(Xte)[:, 0].mean():.4f}")
print("  test o'rtachasi 0 EMAS — va bu to'g'ri, test boshqa taqsimot")
print()
print("XATO — scaler test ni ham ko'rdi:")
print(f"  o'rtacha: {xato.mean_[0]:.4f}  <- test ma'lumoti aralashgan")
print()
print("Qoida: fit FAQAT train da. test da faqat transform.")
print("Pipeline ishlatsangiz, bu xatoni qilish IMKONSIZ bo'ladi.")

from sklearn.linear_model import LogisticRegression

model = LogisticRegression(max_iter=2000).fit(Xtr, ytr)
ehtimol = model.predict_proba(Xte)[:, 1]
bashorat = model.predict(Xte)

print(f"predict_proba shakli : {model.predict_proba(Xte).shape}  (har sinf uchun ustun)")
print(f"predict shakli       : {bashorat.shape}")
print()
qiyos = pd.DataFrame({"ehtimol_1": ehtimol[:6].round(4), "predict": bashorat[:6]})
show(qiyos)
print(f"predict = (ehtimol >= 0.5) ekanini tekshiramiz: {bool(((ehtimol >= 0.5).astype(int) == bashorat).all())}")
print()
print("Ya'ni predict() — bu predict_proba() ustiga qo'yilgan 0.5 chegarasi.")
print("8-bobda ko'rganmiz: 0.5 — sehrli raqam emas, uni o'zgartirish mumkin va kerak.")

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# nomutanosib holat yasaymiz: 10% musbat
r = np.random.default_rng(0)
nomutanosib_y = np.zeros(1000, dtype=int)
nomutanosib_y[r.choice(1000, 100, replace=False)] = 1
hammaga_yoq = np.zeros(1000, dtype=int)

show(pd.DataFrame([
    {"metrika": "accuracy", "qiymat": round(accuracy_score(nomutanosib_y, hammaga_yoq), 4),
     "ma'nosi": "to'g'ri javoblar ulushi"},
    {"metrika": "precision", "qiymat": round(precision_score(nomutanosib_y, hammaga_yoq, zero_division=0), 4),
     "ma'nosi": "'ha' deganlarimdan nechtasi haqiqatan ha"},
    {"metrika": "recall", "qiymat": round(recall_score(nomutanosib_y, hammaga_yoq, zero_division=0), 4),
     "ma'nosi": "haqiqiy 'ha' lardan nechtasini topdim"},
    {"metrika": "f1", "qiymat": round(f1_score(nomutanosib_y, hammaga_yoq, zero_division=0), 4),
     "ma'nosi": "precision va recall muvozanati"},
]))
print("Model HAMMAGA 'yo'q' dedi — hech narsa o'rganmadi.")
print("Accuracy yuqori, qolgan hammasi nol. Shuning uchun nomutanosib")
print("ma'lumotda accuracy ga qarash — o'zini aldash (7 va 8-bob).")

m = LogisticRegression(C=0.5, max_iter=2000).fit(Xtr, ytr)
print("GIPERPARAMETR — siz tanlaysiz, model o'rganmaydi:")
print(f"  C         = {m.C}")
print(f"  max_iter  = {m.max_iter}")
print()
print("PARAMETR — model ma'lumotdan o'rganadi, siz tanlamaysiz:")
print(f"  coef_     = {m.coef_[0].round(4)}")
print(f"  intercept_= {m.intercept_.round(4)}")
print()
print("Oson belgi: ism oxiri _ bilan tugasa (coef_, mean_) — bu O'RGANILGAN parametr.")
print("Konstruktorga yozsangiz (C=0.5) — bu giperparametr. 23-bob ularni sozlaydi.")

from sklearn.preprocessing import MinMaxScaler

std = StandardScaler().fit_transform(Xtr)
mnm = MinMaxScaler().fit_transform(Xtr)
show(pd.DataFrame([
    {"usul": "StandardScaler", "o'rtacha": round(float(std.mean()), 4),
     "std": round(float(std.std()), 4), "min": round(float(std.min()), 2),
     "maks": round(float(std.max()), 2), "qachon": "taqsimot normalga yaqin bo'lsa"},
    {"usul": "MinMaxScaler", "o'rtacha": round(float(mnm.mean()), 4),
     "std": round(float(mnm.std()), 4), "min": round(float(mnm.min()), 2),
     "maks": round(float(mnm.max()), 2), "qachon": "chegara aniq kerak bo'lsa (rasm: 0..1)"},
]))
print("StandardScaler: o'rtacha 0, std 1 — lekin chegara YO'Q.")
print("MinMaxScaler: 0..1 oralig'ida — lekin chetdagi qiymat hammani siqib qo'yadi.")

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

show(pd.DataFrame([
    {"jihat": "Daraxtlar qanday quriladi", "Bagging (RandomForest)": "PARALLEL, mustaqil",
     "Boosting": "KETMA-KET, har biri oldingining xatosini tuzatadi"},
    {"jihat": "Nimani kamaytiradi", "Bagging (RandomForest)": "variance (tarqoqlik)",
     "Boosting": "bias (siljish)"},
    {"jihat": "Overfitting xavfi", "Bagging (RandomForest)": "past",
     "Boosting": "yuqori — qadam soni nazorat qilinadi"},
    {"jihat": "Parallellashtirish", "Bagging (RandomForest)": "oson (n_jobs=-1)",
     "Boosting": "qiyin — ketma-ketlik majburiy"},
]))
rf = RandomForestClassifier(n_estimators=100, random_state=0).fit(Xtr, ytr)
gb = GradientBoostingClassifier(random_state=0).fit(Xtr, ytr)
print(f"RandomForest aniqlik : {rf.score(Xte, yte):.4f}")
print(f"GradientBoosting     : {gb.score(Xte, yte):.4f}")
print("(18 va 19-bob: qaysi biri yutishi ma'lumotga bog'liq, qoida yo'q)")

from sklearn.metrics import log_loss

a = LogisticRegression(C=100, max_iter=2000).fit(Xtr, ytr)
b = LogisticRegression(C=0.01, max_iter=2000).fit(Xtr, ytr)
show(pd.DataFrame([
    {"model": "C=100", "accuracy": round(a.score(Xte, yte), 4),
     "log_loss": round(log_loss(yte, a.predict_proba(Xte)), 4)},
    {"model": "C=0.01", "accuracy": round(b.score(Xte, yte), 4),
     "log_loss": round(log_loss(yte, b.predict_proba(Xte)), 4)},
]))
print("LOSS — model o'qitish paytida minimallashtiradi, u uzluksiz va hosilaga ega.")
print("METRIKA — siz baholaysiz, u biznes ma'nosiga ega, hosilasi bo'lmasligi mumkin.")
print("Ikkisi bir xil tartibda joylashmasligi mumkin — shuning uchun ikkisini ham kuzating.")

show(pd.DataFrame([
    {"to'plam": "train", "nechta marta ishlatiladi": "ko'p", "nima uchun": "model vaznlarini o'rganish"},
    {"to'plam": "validation", "nechta marta ishlatiladi": "ko'p", "nima uchun": "giperparametr tanlash, model tanlash"},
    {"to'plam": "test", "nechta marta ishlatiladi": "BIR MARTA", "nima uchun": "yakuniy halol baho"},
]))
print("Test to'plamiga qarab biror qaror qabul qilsangiz, u endi validation.")
print("Va sizda halol baho qolmaydi. 5-bobda shu haqda.")

from sklearn.linear_model import Lasso, Ridge
from sklearn.preprocessing import StandardScaler as SS

Xs = SS().fit_transform(Xtr)
yr = uy.loc[Xtr.index, "narx_usd"]

qatorlar = []
for alpha in [1, 10, 50, 100, 200, 400]:
    l1 = Lasso(alpha=alpha, max_iter=10000).fit(Xs, yr)
    l2 = Ridge(alpha=alpha).fit(Xs, yr)
    qatorlar.append({"alpha": alpha,
                     "L1 nol koeffitsiyent": int((l1.coef_ == 0).sum()),
                     "L2 nol koeffitsiyent": int((l2.coef_ == 0).sum()),
                     "L1 koeffitsiyentlari": str(l1.coef_.round(1))})
show(pd.DataFrame(qatorlar))
print(f"Ustunlar soni: {len(SON)}")

rf2 = RandomForestClassifier(n_estimators=100, random_state=0).fit(Xtr, ytr)
show(pd.DataFrame({"ustun": SON,
                   "korrelyatsiya (target bilan)": [round(float(np.corrcoef(Xtr[c], ytr)[0, 1]), 4) for c in SON],
                   "feature importance": rf2.feature_importances_.round(4)}))
print("Uchta BOSHQA narsa:")
print("  korrelyatsiya  — chiziqli bog'liqlik kuchi")
print("  muhimlik       — model shu ustundan qancha foydalangani")
print("  sababiyat      — ustun target ni O'ZGARTIRADIMI")
print()
print("Model muhimlik bersa, bu sabab degani EMAS. Sababni aniqlash uchun")
print("tajriba (A/B test) kerak, kuzatuv ma'lumoti yetarli emas.")

print(f"pandas versiyasi: {pd.__version__}")
asl = uy[["maydon_m2", "narx_usd"]].head(5).copy()
print("\nAsl narxlar:", list(asl.narx_usd))

bolak = asl[asl.maydon_m2 > 0]        # bu YANGI obyekt
with warnings.catch_warnings(record=True) as ogoh:
    warnings.simplefilter("always")
    bolak["narx_usd"] = 0
    turlari = [w.category.__name__ for w in ogoh]

print(f"Ogohlantirish  : {turlari or 'YO`Q'}")
print(f"bolak o'zgardi : {bool((bolak.narx_usd == 0).all())}")
print(f"asl o'zgardi   : {bool((asl.narx_usd == 0).any())}")
print("\nAsl narxlar hali ham:", list(asl.narx_usd))

asl2 = uy[["maydon_m2", "narx_usd"]].head(5).copy()
asl2.loc[asl2.maydon_m2 > 0, "narx_usd"] = 0
print(f".loc bilan — asl jadval o'zgardi: {bool((asl2.narx_usd == 0).all())}")
print("Qoida: filtrlab, keyin o'zlashtirish EMAS. Bitta .loc amalida qiling.")

show(pd.DataFrame([
    {"ma'lumot turi": "Mustaqil qatorlar (uy, mijoz)", "shuffle": "MUMKIN",
     "split": "train_test_split(shuffle=True)"},
    {"ma'lumot turi": "Vaqt qatori", "shuffle": "MUMKIN EMAS",
     "split": "xronologik yoki TimeSeriesSplit (34-bob)"},
    {"ma'lumot turi": "Guruhlangan (bitta uy, ko'p e'lon)", "shuffle": "EHTIYOT",
     "split": "GroupShuffleSplit (37-bob)"},
    {"ma'lumot turi": "Nomutanosib sinflar", "shuffle": "MUMKIN",
     "split": "stratify=y bilan"},
]))

show(pd.DataFrame([
    {"Xato": "could not convert string to float", "Sababi": "matn ustun modelga tushdi",
     "Yechim": "OneHotEncoder yoki ColumnTransformer (4-bob)"},
    {"Xato": "Input contains NaN", "Sababi": "bo'sh qiymat qolgan",
     "Yechim": "SimpleImputer, Pipeline ichida (4-bob)"},
    {"Xato": "X has N features, but expects M", "Sababi": "train va test ustunlari boshqa",
     "Yechim": "Pipeline saqlang, ustunlarni tekshiring (39-bob)"},
    {"Xato": "ConvergenceWarning", "Sababi": "max_iter yetmadi yoki scaling yo'q",
     "Yechim": "scaling qo'shing, keyin max_iter oshiring (15-bob)"},
    {"Xato": "columns are missing", "Sababi": "kiruvchi kadrda ustun yo'q",
     "Yechim": "shartnoma bilan tekshiring (37, 40-bob)"},
    {"Xato": "Found input variables with inconsistent numbers", "Sababi": "X va y uzunligi boshqa",
     "Yechim": "filtrlashda ikkisini birga filtrlang"},
]))

show(pd.DataFrame([
    {"Savol": "fit_transform ni test da ishlatsam bo'ladimi?", "Javob": "Yo'q — faqat transform"},
    {"Savol": "0.5 chegara majburiymi?", "Javob": "Yo'q, u shunchaki standart qiymat"},
    {"Savol": "Accuracy yaxshi metrikami?", "Javob": "Sinflar muvozanatli bo'lsa — ha"},
    {"Savol": "Qancha ma'lumot kerak?", "Javob": "O'quv egri chizig'i aytadi (10-bob)"},
    {"Savol": "Deep learning har doim yaxshiroqmi?", "Javob": "Yo'q — kitobda bir necha marta o'lchandi"},
    {"Savol": "Feature importance sababni ko'rsatadimi?", "Javob": "Yo'q"},
    {"Savol": "Test ni necha marta ishlataman?", "Javob": "Bir marta"},
    {"Savol": "Modelim 0.95 berdi, yaxshimi?", "Javob": "Baza nima berganini ayting (6-bob)"},
]))