Modelni saqlash va yuklash

import time
import joblib
import numpy as np
import pandas as pd
import sklearn
from pathlib import Path
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler

uy = pd.read_csv(DATA / "uy_ijara.csv")
SON = ["maydon_m2", "xonalar", "qavat", "metro_km", "qurilgan_yil"]
MATN = ["tuman"]
X = uy[SON + MATN]
y = uy["ijaraga_berildi_30k"]
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)

tayyorlash = ColumnTransformer([
    ("son", Pipeline([("imp", SimpleImputer(strategy="median")),
                      ("sc", StandardScaler())]), SON),
    ("matn", OneHotEncoder(handle_unknown="ignore"), MATN),
])
quvur = Pipeline([("tayyorlash", tayyorlash),
                  ("model", LogisticRegression(max_iter=2000))]).fit(Xtr, ytr)
asl_bashorat = quvur.predict(Xte)

print(f"sklearn versiyasi : {sklearn.__version__}")
print(f"Quvur aniqligi    : {accuracy_score(yte, asl_bashorat):.4f}")
print(f"Quvur qadamlari   : {[nom for nom, _ in quvur.steps]}")

SAQLASH = ISH / "modellar"
SAQLASH.mkdir(parents=True, exist_ok=True)

joblib.dump(quvur, SAQLASH / "quvur.joblib")
joblib.dump(quvur.named_steps["model"], SAQLASH / "faqat_model.joblib")

for nom in ["quvur.joblib", "faqat_model.joblib"]:
    print(f"  {nom:<22} {(SAQLASH / nom).stat().st_size:>7} bayt")

faqat_model = joblib.load(SAQLASH / "faqat_model.joblib")
print("Faqat modelga XOM ma'lumot beramiz:")
try:
    faqat_model.predict(Xte)
    print("  xato chiqmadi")
except Exception as x:
    print(f"  {type(x).__name__}: {str(x)[:95]}")
print()
print("Yaxshi xabar: bu xato BALAND OVOZDA chiqdi — 'tuman' matn ustuni sonli emas.")

xom_son = Xte[SON].fillna(Xte[SON].median())          # scaling YO'Q
kerakli = faqat_model.n_features_in_
qoshimcha = np.zeros((len(xom_son), kerakli - len(SON)))   # OneHot o'rniga nollar
soxta_kirish = np.hstack([xom_son.values, qoshimcha])

soxta_bashorat = faqat_model.predict(soxta_kirish)
mos = float((soxta_bashorat == asl_bashorat).mean())
print(f"Model kutgan ustunlar soni : {kerakli}")
print(f"Xato chiqdimi              : yo'q")
print(f"To'g'ri javob bilan mos    : {mos:.4f}")
print(f"Haqiqiy aniqlik            : {accuracy_score(yte, soxta_bashorat):.4f}")
print(f"Asl quvur aniqligi         : {accuracy_score(yte, asl_bashorat):.4f}")
print()
print("Model ishladi, javob berdi, hech kim xato ko'rmadi — va javoblar buzuq.")

yuklangan = joblib.load(SAQLASH / "quvur.joblib")
sinovlar = []

def sina(nom, kadr):
    try:
        b = yuklangan.predict(kadr)
        return {"holat": nom, "natija": "ishladi",
                "asl bilan mos": f"{float((b == asl_bashorat).mean()):.4f}"}
    except Exception as x:
        return {"holat": nom, "natija": type(x).__name__,
                "asl bilan mos": "-"}

sinovlar.append(sina("asl holat", Xte))
sinovlar.append(sina("ustunlar aralashtirildi", Xte[MATN + SON]))
sinovlar.append(sina("sonlar teskari tartibda", Xte[SON[::-1] + MATN]))
sinovlar.append(sina("bitta ustun YO'Q", Xte.drop(columns=["metro_km"])))
qoshimchali = Xte.copy()
qoshimchali["yangi_ustun"] = 1.0
sinovlar.append(sina("qo'shimcha ustun qo'shildi", qoshimchali))
nomi_almashgan = Xte.rename(columns={"maydon_m2": "xonalar", "xonalar": "maydon_m2"})
sinovlar.append(sina("ikki ustun NOMI almashtirildi", nomi_almashgan))
show(pd.DataFrame(sinovlar))

"""Model bilan birga saqlanadigan ma'lumot."""
import json
import platform
from datetime import date
from pathlib import Path


def manifest_yoz(yol, model, ustunlar, metrikalar, izoh=""):
    """Model yonidagi `manifest.json` ni yozadi.

    Nega kerak: `.joblib` fayl o'zi haqida hech narsa aytmaydi. Versiya mos
    kelmasa, yuklash jim buzilishi mumkin — manifest buni ko'rsatadi.
    """
    import sklearn
    import numpy
    yozuv = {
        "yaratilgan": str(date.today()),
        "python": platform.python_version(),
        "sklearn": sklearn.__version__,
        "numpy": numpy.__version__,
        "model_turi": type(model).__name__,
        "kutilgan_ustunlar": list(ustunlar),
        "metrikalar": metrikalar,
        "izoh": izoh,
    }
    Path(yol).write_text(json.dumps(yozuv, ensure_ascii=False, indent=2), encoding="utf-8")
    return yozuv


def versiyani_tekshir(manifest_yoli):
    """Hozirgi muhit manifestga mos keladimi. Mos kelmasa — ogohlantirish ro'yxati."""
    import sklearn
    import numpy
    m = json.loads(Path(manifest_yoli).read_text(encoding="utf-8"))
    ogoh = []
    for nom, hozir in [("sklearn", sklearn.__version__), ("numpy", numpy.__version__)]:
        if m.get(nom) != hozir:
            ogoh.append(f"{nom}: model {m.get(nom)} bilan saqlangan, hozir {hozir}")
    return ogoh

import sys
sys.path.insert(0, str(ISH))
from modellar.manifest import manifest_yoz, versiyani_tekshir

yozuv = manifest_yoz(
    SAQLASH / "manifest.json", quvur, SON + MATN,
    {"aniqlik": round(accuracy_score(yte, asl_bashorat), 4),
     "test_hajmi": int(len(yte))},
    izoh="uy_ijara.csv, ijaraga_berildi_30k target",
)
for k, v in yozuv.items():
    print(f"  {k:<18} {v}")
print()
print(f"Versiya tekshiruvi: {versiyani_tekshir(SAQLASH / 'manifest.json') or 'mos keladi'}")

"""Saqlash -> yuklash -> bashorat zanjiri buzilmaganini tekshiradi."""
import joblib
import numpy as np


def aylanma_sinov(model, X_namuna, yol):
    """Modelni saqlaydi, qayta yuklaydi va bashoratlar AYNAN mos kelishini tekshiradi.

    Bu sinov har deploy dan oldin ishga tushiriladi. U arzon, lekin
    "saqladim deb o'yladim" turidagi xatolarning hammasini ushlaydi.
    """
    oldin = model.predict(X_namuna)
    joblib.dump(model, yol)
    keyin = joblib.load(yol).predict(X_namuna)
    if not np.array_equal(oldin, keyin):
        farq = int((oldin != keyin).sum())
        raise AssertionError(f"saqlash/yuklash bashoratni o'zgartirdi: {farq} qatorda farq")
    return len(X_namuna)

from modellar.aylanma_sinov import aylanma_sinov

n = aylanma_sinov(quvur, Xte, SAQLASH / "quvur.joblib")
print(f"Aylanma sinov: {n} qatorda bashorat aynan bir xil — OK")

eski_model = Pipeline([
    ("tayyorlash", tayyorlash),
    ("model", LogisticRegression(max_iter=2000, C=0.0005)),   # boshqa sozlama
]).fit(Xtr, ytr)
joblib.dump(eski_model, SAQLASH / "eski_artefakt.joblib")

oldin = quvur.predict(Xte)
keyin = joblib.load(SAQLASH / "eski_artefakt.joblib").predict(Xte)
farq = int((oldin != keyin).sum())
print(f"Kutilgan model aniqligi : {accuracy_score(yte, oldin):.4f}")
print(f"Papkadagi eski model    : {accuracy_score(yte, keyin):.4f}")
print(f"Nechta qatorda farq     : {farq} / {len(Xte)}")
print()
if farq:
    print("Aylanma sinov shu holatda AssertionError beradi va deploy to'xtaydi.")
else:
    print("Farq yo'q — bu sozlamada ikki model bir xil javob berdi.")

import hashlib

def fayl_xeshi(yol):
    return hashlib.sha256(Path(yol).read_bytes()).hexdigest()

xesh = fayl_xeshi(SAQLASH / "quvur.joblib")
print(f"quvur.joblib SHA-256: {xesh[:32]}...")
(SAQLASH / "quvur.sha256").write_text(xesh, encoding="utf-8")

kutilgan = (SAQLASH / "quvur.sha256").read_text(encoding="utf-8").strip()
print(f"Yuklashdan oldin tekshiruv: {'mos' if fayl_xeshi(SAQLASH / 'quvur.joblib') == kutilgan else 'MOS EMAS — YUKLAMANG'}")

from skl2onnx import to_onnx
from skl2onnx.common.data_types import FloatTensorType, StringTensorType
import onnxruntime as ort

turlar = [(c, (StringTensorType if c in MATN else FloatTensorType)([None, 1]))
          for c in SON + MATN]
onx = to_onnx(quvur, initial_types=turlar,
              options={id(quvur.named_steps["model"]): {"zipmap": False}})
(SAQLASH / "quvur.onnx").write_bytes(onx.SerializeToString())

seans = ort.InferenceSession(str(SAQLASH / "quvur.onnx"),
                             providers=["CPUExecutionProvider"])
kirish = {c: (Xte[c].values.reshape(-1, 1).astype(np.float32) if c in SON
              else Xte[c].values.reshape(-1, 1).astype(object))
          for c in SON + MATN}
onnx_bashorat = seans.run(None, kirish)[0].ravel()
print(f"ONNX va sklearn bashoratlari mos: {float((onnx_bashorat == asl_bashorat).mean()):.4f}")

bitta = {c: v[:1] for c, v in kirish.items()}
t0 = time.perf_counter()
for _ in range(300):
    seans.run(None, bitta)
onnx_ms = (time.perf_counter() - t0) / 300 * 1000

bitta_df = Xte.iloc[:1]
t0 = time.perf_counter()
for _ in range(300):
    yuklangan.predict(bitta_df)
skl_ms = (time.perf_counter() - t0) / 300 * 1000

show(pd.DataFrame([
    {"format": "joblib (sklearn)", "hajmi (bayt)": (SAQLASH / "quvur.joblib").stat().st_size,
     "bitta bashorat (ms)": round(skl_ms, 3), "kod bajaradimi": "ha"},
    {"format": "ONNX", "hajmi (bayt)": (SAQLASH / "quvur.onnx").stat().st_size,
     "bitta bashorat (ms)": round(onnx_ms, 3), "kod bajaradimi": "yo'q"},
]))
print(f"ONNX tezlik ustunligi: {skl_ms / onnx_ms:.0f} barobar")

show(pd.DataFrame([
    {"Fayl": "quvur.joblib", "Nima uchun": "to'liq Pipeline — tayyorlash + model"},
    {"Fayl": "manifest.json", "Nima uchun": "versiyalar, ustunlar, metrikalar, sana"},
    {"Fayl": "quvur.sha256", "Nima uchun": "fayl almashtirilmaganini tekshirish"},
    {"Fayl": "quvur.onnx", "Nima uchun": "xizmat uchun tez va kodsiz format"},
    {"Fayl": "DATASHEET.md", "Nima uchun": "37-bob: ma'lumot qayerdan kelgan"},
    {"Fayl": "requirements.txt", "Nima uchun": "muhitni qayta tiklash"},
]))
print()
print("Hammasi bitta papkada: ish/modellar/")
for p in sorted(SAQLASH.iterdir()):
    if p.is_file():
        print(f"  {p.name:<22} {p.stat().st_size:>7} bayt")

muhit = f"""scikit-learn=={sklearn.__version__}
numpy=={np.__version__}
pandas=={pd.__version__}
joblib=={joblib.__version__}
"""
(SAQLASH / "requirements.txt").write_text(muhit, encoding="utf-8")
print(muhit)
print("Versiyalar QAT'IY qadalgan (==), chunki sklearn ichki formati versiyalar")
print("orasida o'zgaradi va eski model yangi versiyada jim buzilishi mumkin.")

show(pd.DataFrame([
    {"Qadam": "36. Ma'lumot yig'ildi", "Holati": "bajarildi"},
    {"Qadam": "37. Dataset qurildi", "Holati": "bajarildi"},
    {"Qadam": "38. Model moslashtirildi", "Holati": "bajarildi"},
    {"Qadam": "39. Model saqlandi", "Holati": "quvur + manifest + xesh + ONNX"},
    {"Qadam": "40. Xizmatga chiqarish", "Holati": "keyingi bob — Django API"},
]))

show(pd.DataFrame([
    {"Tushuncha": "Butun Pipeline saqlanadi", "Bitta jumla": "O'lchandi: faqat model jim noto'g'ri javob beradi"},
    {"Tushuncha": "Ustun tartibi", "Bitta jumla": "O'lchandi: DataFrame da muhim EMAS — nom bo'yicha tanlanadi"},
    {"Tushuncha": "Yetishmagan ustun", "Bitta jumla": "Baland ovozda ValueError — xavfsiz"},
    {"Tushuncha": "Nom almashishi", "Bitta jumla": "O'lchandi: JIM buziladi — shartnoma bilan ushlanadi"},
    {"Tushuncha": "Manifest", "Bitta jumla": "Versiya, ustun, metrika model yonida turadi"},
    {"Tushuncha": "Aylanma sinov", "Bitta jumla": "Saqlash ishlaganini isbotlaydi, taxmin qilmaydi"},
    {"Tushuncha": "pickle xavfsizligi", "Bitta jumla": "joblib.load kod bajaradi — begona fayl ochilmaydi"},
    {"Tushuncha": "ONNX", "Bitta jumla": "Kodsiz, portativ, bitta qatorda ancha tez"},
    {"Tushuncha": "Versiya qadash", "Bitta jumla": "== bilan, chunki ichki format o'zgaradi"},
]))