Django API: modelni xizmatga chiqarish

import json
import time
import joblib
import numpy as np
import pandas as pd
from pathlib import Path
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler

uy = pd.read_csv(DATA / "uy_ijara.csv")
SON = ["maydon_m2", "xonalar", "qavat", "metro_km", "qurilgan_yil"]
MATN = ["tuman"]
X, y = uy[SON + MATN], uy["ijaraga_berildi_30k"]
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)

quvur = Pipeline([
    ("tayyorlash", ColumnTransformer([
        ("son", Pipeline([("imp", SimpleImputer(strategy="median")),
                          ("sc", StandardScaler())]), SON),
        ("matn", OneHotEncoder(handle_unknown="ignore"), MATN)])),
    ("model", LogisticRegression(max_iter=2000)),
]).fit(Xtr, ytr)

MODEL_PAPKA = ISH / "xizmat" / "modellar"
MODEL_PAPKA.mkdir(parents=True, exist_ok=True)
joblib.dump(quvur, MODEL_PAPKA / "quvur.joblib")
print(f"Model saqlandi: {(MODEL_PAPKA / 'quvur.joblib').stat().st_size} bayt")
print(f"Aniqlik       : {accuracy_score(yte, quvur.predict(Xte)):.4f}")

bitta_soro = pd.DataFrame([{"maydon_m2": 52.0, "xonalar": 2, "qavat": 5,
                            "metro_km": 1.2, "qurilgan_yil": 2010.0,
                            "tuman": "Chilonzor"}])
yol = MODEL_PAPKA / "quvur.joblib"

t0 = time.perf_counter()
for _ in range(40):
    joblib.load(yol)
yuklash_ms = (time.perf_counter() - t0) / 40 * 1000

model_xotirada = joblib.load(yol)
t0 = time.perf_counter()
for _ in range(200):
    model_xotirada.predict(bitta_soro)
bashorat_ms = (time.perf_counter() - t0) / 200 * 1000

show(pd.DataFrame([
    {"yondashuv": "har so'rovda joblib.load", "bitta so'rov (ms)": round(yuklash_ms + bashorat_ms, 2)},
    {"yondashuv": "bir marta yuklab, xotirada saqlash", "bitta so'rov (ms)": round(bashorat_ms, 2)},
]))
print(f"Ortiqcha yuk: {(yuklash_ms + bashorat_ms) / bashorat_ms:.0f} barobar")
print(f"Kuniga 100 000 so'rovda farq: {yuklash_ms * 100000 / 1000:.0f} soniya bekorga sarflangan CPU")

"""Kiruvchi so'rovni tekshirish. Model ko'rmasdan oldin hamma narsa tekshiriladi."""

SHARTNOMA = {
    "maydon_m2":    {"tur": float, "min": 15,   "maks": 400},
    "xonalar":      {"tur": int,   "min": 1,    "maks": 10},
    "qavat":        {"tur": int,   "min": 1,    "maks": 50},
    "metro_km":     {"tur": float, "min": 0,    "maks": 50},
    "qurilgan_yil": {"tur": float, "min": 1900, "maks": 2030},
}
TUMANLAR = {"Yunusobod", "Chilonzor", "Mirzo Ulug'bek", "Shayxontohur",
            "Yakkasaroy", "Olmazor", "Sergeli", "Bektemir"}


def tekshir(malumot):
    """(tozalangan_dict, xatolar_royxati) qaytaradi.

    Xatolar bo'lsa, tozalangan dict ishlatilmaydi. Hech qachon
    "yarim to'g'ri" ma'lumot modelga o'tmaydi.
    """
    if not isinstance(malumot, dict):
        return None, ["tana JSON obyekt bo'lishi kerak"]

    xatolar, toza = [], {}
    for maydon, qoida in SHARTNOMA.items():
        if maydon not in malumot:
            xatolar.append(f"{maydon}: majburiy maydon yo'q")
            continue
        qiymat = malumot[maydon]
        if isinstance(qiymat, bool) or not isinstance(qiymat, (int, float)):
            xatolar.append(f"{maydon}: son bo'lishi kerak")
            continue
        if not (qoida["min"] <= qiymat <= qoida["maks"]):
            xatolar.append(
                f"{maydon}: {qoida['min']} va {qoida['maks']} orasida bo'lishi kerak")
            continue
        toza[maydon] = qoida["tur"](qiymat)

    tuman = malumot.get("tuman")
    if tuman is None:
        xatolar.append("tuman: majburiy maydon yo'q")
    elif tuman not in TUMANLAR:
        xatolar.append("tuman: ro'yxatda yo'q")      # ruxsat etilgan qiymatlar OCHILMAYDI
    else:
        toza["tuman"] = tuman

    ortiqcha = set(malumot) - set(SHARTNOMA) - {"tuman"}
    if ortiqcha:
        xatolar.append(f"notanish maydon: {sorted(ortiqcha)}")

    return (toza, []) if not xatolar else (None, xatolar)

import sys
sys.path.insert(0, str(ISH))
from xizmat.tekshiruv import tekshir

holatlar = [
    ("to'g'ri so'rov", {"maydon_m2": 52, "xonalar": 2, "qavat": 5, "metro_km": 1.2,
                        "qurilgan_yil": 2010, "tuman": "Chilonzor"}),
    ("maydon yetishmaydi", {"xonalar": 2, "qavat": 5, "metro_km": 1.2,
                            "qurilgan_yil": 2010, "tuman": "Chilonzor"}),
    ("diapazondan tashqari", {"maydon_m2": 5000, "xonalar": 2, "qavat": 5, "metro_km": 1.2,
                              "qurilgan_yil": 2010, "tuman": "Chilonzor"}),
    ("matn son o'rnida", {"maydon_m2": "katta", "xonalar": 2, "qavat": 5, "metro_km": 1.2,
                          "qurilgan_yil": 2010, "tuman": "Chilonzor"}),
    ("notanish tuman", {"maydon_m2": 52, "xonalar": 2, "qavat": 5, "metro_km": 1.2,
                        "qurilgan_yil": 2010, "tuman": "Toshkent shahri"}),
    ("ortiqcha maydon", {"maydon_m2": 52, "xonalar": 2, "qavat": 5, "metro_km": 1.2,
                         "qurilgan_yil": 2010, "tuman": "Chilonzor", "narx": 500}),
]
natija = []
for nom, m in holatlar:
    toza, xato = tekshir(m)
    natija.append({"holat": nom, "o'tdimi": "ha" if toza else "yo'q",
                   "birinchi xato": xato[0] if xato else "-"})
show(pd.DataFrame(natija))

"""Model xizmati — bitta faylli Django ilovasi.

Ishga tushirish:
    python xizmat/api.py runserver

Bitta fayl: o'quv maqsadida hamma narsa ko'rinib tursin. Katta loyihada
settings, urls va views alohida fayllarga ajratiladi.
"""
import json
import os
import sys
from pathlib import Path

import django
from django.conf import settings

HERE = Path(__file__).parent

if not settings.configured:
    settings.configure(
        DEBUG=False,                                   # ishlab chiqarishda HECH QACHON True emas
        SECRET_KEY=os.environ.get("MAXFIY_KALIT", "faqat-ishlab-chiqish-uchun"),
        ALLOWED_HOSTS=os.environ.get("RUXSAT_HOSTLAR", "127.0.0.1,localhost").split(","),
        ROOT_URLCONF=__name__,
        DATABASES={},
        INSTALLED_APPS=[],
        LOGGING_CONFIG=None,
    )
    django.setup()

from django.http import JsonResponse                      # noqa: E402
from django.urls import path                              # noqa: E402
from django.views.decorators.csrf import csrf_exempt      # noqa: E402

sys.path.insert(0, str(HERE.parent))
from xizmat.tekshiruv import tekshir                      # noqa: E402

import joblib                                             # noqa: E402
import pandas as pd                                       # noqa: E402

# Model BIR MARTA yuklanadi — modul import bo'lganda, so'rov kelganda emas.
MODEL_YOLI = HERE / "modellar" / "quvur.joblib"
MODEL = joblib.load(MODEL_YOLI)
USTUNLAR = ["maydon_m2", "xonalar", "qavat", "metro_km", "qurilgan_yil", "tuman"]


def javob(malumot=None, xato=None, kod=200):
    """Hamma javob bitta shaklda: muvaffaqiyat / malumot / xato."""
    return JsonResponse({"muvaffaqiyat": xato is None, "malumot": malumot, "xato": xato},
                        status=kod, json_dumps_params={"ensure_ascii": False})


@csrf_exempt
def bashorat(soro):
    if soro.method != "POST":
        return javob(xato="faqat POST", kod=405)
    try:
        tana = json.loads(soro.body.decode("utf-8"))
    except (ValueError, UnicodeDecodeError):
        return javob(xato="JSON o'qib bo'lmadi", kod=400)

    toza, xatolar = tekshir(tana)
    if xatolar:
        return javob(xato=xatolar, kod=400)

    try:
        kadr = pd.DataFrame([toza])[USTUNLAR]
        ehtimol = float(MODEL.predict_proba(kadr)[0][1])
    except Exception:
        # Ichki xato mijozga OCHILMAYDI — faqat serverda jurnalga yoziladi.
        return javob(xato="ichki xato", kod=500)

    return javob({"ijaraga_berilish_ehtimoli": round(ehtimol, 4),
                  "bashorat": int(ehtimol >= 0.5)})


def salomatlik(soro):
    """Monitoring uchun: xizmat tirikmi va model yuklanganmi."""
    return javob({"holat": "tirik", "model_yuklangan": MODEL is not None})


urlpatterns = [path("bashorat/", bashorat), path("salomatlik/", salomatlik)]

if __name__ == "__main__":
    from django.core.management import execute_from_command_line
    execute_from_command_line(sys.argv)

import importlib
sys.path.insert(0, str(ISH))
api = importlib.import_module("xizmat.api")
from django.test import Client

mijoz = Client()
r = mijoz.get("/salomatlik/")
print(f"GET /salomatlik/ -> {r.status_code}")
print(json.dumps(r.json(), ensure_ascii=False, indent=2))

togri = {"maydon_m2": 52, "xonalar": 2, "qavat": 5, "metro_km": 1.2,
         "qurilgan_yil": 2010, "tuman": "Chilonzor"}
r = mijoz.post("/bashorat/", data=json.dumps(togri), content_type="application/json")
print(f"POST /bashorat/ (to'g'ri) -> {r.status_code}")
print(json.dumps(r.json(), ensure_ascii=False, indent=2))

sinovlar = [
    ("bo'sh tana", "{}", "application/json"),
    ("buzuq JSON", "{bu json emas", "application/json"),
    ("maydon yetishmaydi", json.dumps({"xonalar": 2}), "application/json"),
    ("diapazondan tashqari", json.dumps({**togri, "maydon_m2": 9999}), "application/json"),
    ("notanish tuman", json.dumps({**togri, "tuman": "Yangi tuman"}), "application/json"),
]
qatorlar = []
for nom, tana, tur in sinovlar:
    r = mijoz.post("/bashorat/", data=tana, content_type=tur)
    j = r.json()
    xato = j["xato"]
    qatorlar.append({"sinov": nom, "kod": r.status_code,
                     "xato": (xato[0] if isinstance(xato, list) else xato)})
r = mijoz.get("/bashorat/")
qatorlar.append({"sinov": "GET yuborildi", "kod": r.status_code, "xato": r.json()["xato"]})
show(pd.DataFrame(qatorlar))

t0 = time.perf_counter()
N = 200
for _ in range(N):
    mijoz.post("/bashorat/", data=json.dumps(togri), content_type="application/json")
sorov_ms = (time.perf_counter() - t0) / N * 1000
print(f"To'liq so'rov (tekshiruv + bashorat + JSON): {sorov_ms:.2f} ms")
print(f"Faqat model bashorati                      : {bashorat_ms:.2f} ms")
print(f"Django va tekshiruv ortiqcha yuki          : {sorov_ms - bashorat_ms:.2f} ms")

show(pd.DataFrame([
    {"Qoida": "DEBUG = False", "Nega": "True bo'lsa xato sahifasi kod va sozlamalarni ko'rsatadi"},
    {"Qoida": "SECRET_KEY muhit o'zgaruvchisidan", "Nega": "Kodda yozilgan kalit Git ga tushadi"},
    {"Qoida": "ALLOWED_HOSTS cheklangan", "Nega": "Host sarlavhasi orqali hujumni to'xtatadi"},
    {"Qoida": "Kirish chegarada tekshiriladi", "Nega": "Model hech qachon tekshirilmagan ma'lumot ko'rmaydi"},
    {"Qoida": "Xato xabari ichkarini ochmaydi", "Nega": "Stack trace hujumchiga xarita beradi"},
    {"Qoida": "Foydalanuvchi faylini joblib.load QILMANG", "Nega": "39-bob: pickle kod bajaradi"},
    {"Qoida": "So'rov chegarasi (rate limit)", "Nega": "Bitta mijoz xizmatni to'ldirib qo'ymasin"},
    {"Qoida": "HTTPS va autentifikatsiya", "Nega": "Ochiq API — ochiq model"},
    {"Qoida": "Jurnalga shaxsiy ma'lumot yozilmaydi", "Nega": "36-bob: yig'ilmagan narsa sizib chiqmaydi"},
]))

show(pd.DataFrame([
    {"Bob": "36", "Qadam": "Ma'lumot yig'ish", "Natija": "xom/elonlar_xom.csv"},
    {"Bob": "37", "Qadam": "Dataset qurish", "Natija": "train/val/test + DATASHEET"},
    {"Bob": "38", "Qadam": "Model moslashtirish", "Natija": "baza o'lchandi, LoRA o'rganildi"},
    {"Bob": "39", "Qadam": "Saqlash", "Natija": "quvur + manifest + xesh + ONNX"},
    {"Bob": "40", "Qadam": "Xizmatga chiqarish", "Natija": "JSON API + tekshiruv + sinov"},
]))
print()
print("Yangi dataset ko'rganingizda shu zanjir butunligicha takrorlanadi.")
print("0-bobdagi 12 qadam protokoli — shu zanjirning qisqa kartasi.")

show(pd.DataFrame([
    {"Tushuncha": "Modelni bir marta yuklash", "Bitta jumla": "O'lchandi: har so'rovda yuklash bir necha barobar qimmat"},
    {"Tushuncha": "Chegarada tekshirish", "Bitta jumla": "Model tekshirilmagan ma'lumotni ko'rmaydi"},
    {"Tushuncha": "Bitta javob shakli", "Bitta jumla": "muvaffaqiyat / malumot / xato — har doim"},
    {"Tushuncha": "400 va 500 farqi", "Bitta jumla": "Mijoz xatosi va server xatosi aralashtirilmaydi"},
    {"Tushuncha": "Xato xabari", "Bitta jumla": "Foydali, lekin ichki tuzilmani ochmaydi"},
    {"Tushuncha": "Salomatlik endpointi", "Bitta jumla": "Monitoring xizmat tirikligini biladi"},
    {"Tushuncha": "Test client", "Bitta jumla": "Server ishga tushirmasdan API sinaladi"},
    {"Tushuncha": "runserver emas", "Bitta jumla": "gunicorn + nginx; worker soni xotiraga bog'liq"},
]))