Model formatlari: pickle xavfi va ONNX

import pickle

class ZararsizKorinadi:
    """Fayl ochilganda __reduce__ chaqiriladi va u ISTALGAN narsani qaytarishi mumkin."""
    def __reduce__(self):
        # zararli emas: shunchaki matn qaytaradigan funksiya chaqiriladi
        return (str.upper, ("men yuklanishda bajarildim",))

xom = pickle.dumps(ZararsizKorinadi())
print("fayl hajmi:", len(xom), "bayt")
print("yuklash natijasi:", pickle.loads(xom))
print()
print("=> pickle.loads() FUNKSIYA CHAQIRDI. Biz uni chaqirmadik.")

import pandas as pd
show(pd.DataFrame([
    {"holat": "O'z modelingiz, o'z serveringiz", "pickle xavfsizmi": "HA",
     "izoh": "Manbaga ishonasiz"},
    {"holat": "Hamkasb yuborgan model", "pickle xavfsizmi": "EHTIYOT",
     "izoh": "Uning mashinasi buzilgan bo'lishi mumkin"},
    {"holat": "Internetdan yuklangan model", "pickle xavfsizmi": "YO'Q",
     "izoh": "Hech qachon. ONNX yoki safetensors ishlating"},
    {"holat": "Foydalanuvchi yuklagan fayl", "pickle xavfsizmi": "YO'Q",
     "izoh": "To'g'ridan-to'g'ri masofaviy kod bajarish"},
]))

diagramma(
    {
        "joblib / pickle": (0, 0, 2.8, 1.1, "xizmat"),
        "ONNX": (3.6, 0, 2.8, 1.1, "saqlash"),
        "safetensors": (7.2, 0, 2.8, 1.1, "saqlash"),
        "Kod bajaradi\n(XAVF)": (0, 2.2, 2.8, 1.0, "oddiy"),
        "Faqat graf\n(xavfsiz)": (3.6, 2.2, 2.8, 1.0, "oddiy"),
        "Faqat vaznlar\n(xavfsiz)": (7.2, 2.2, 2.8, 1.0, "oddiy"),
    },
    [
        ("joblib / pickle", "Kod bajaradi\n(XAVF)"),
        ("ONNX", "Faqat graf\n(xavfsiz)"),
        ("safetensors", "Faqat vaznlar\n(xavfsiz)"),
    ],
    nom="model_formatlari.png", kenglik=12,
    izoh="ONNX va safetensors kod saqlamaydi — shuning uchun yuklash xavfsiz.",
)

"""Model formatlari: joblib va ONNX o'rtasida o'tkazish."""
from pathlib import Path
from typing import Any

import numpy as np


def onnx_ga(model, xususiyat_soni: int, yol: str | Path) -> Path:
    """sklearn modelini ONNX ga o'tkazadi."""
    from skl2onnx import to_onnx
    namuna = np.zeros((1, xususiyat_soni), dtype=np.float32)
    onx = to_onnx(model, namuna, options={id(model): {"zipmap": False}})
    p = Path(yol)
    p.write_bytes(onx.SerializeToString())
    return p


def onnx_sessiya(yol: str | Path):
    """ONNX modelini yuklaydi. Hech qanday kod bajarilmaydi."""
    import onnxruntime as ort
    s = ort.InferenceSession(str(yol), providers=["CPUExecutionProvider"])
    return s


def onnx_ehtimol(sessiya, X) -> np.ndarray:
    """Broker ehtimolini qaytaradi."""
    kirish = sessiya.get_inputs()[0].name
    chiqish = [o.name for o in sessiya.get_outputs()]
    natija = sessiya.run(chiqish, {kirish: np.asarray(X, dtype=np.float32)})
    ehtimollar = natija[-1]
    return np.asarray(ehtimollar)[:, 1]

python3 -c "
import sys, pathlib; sys.path.insert(0, 'src')
if not pathlib.Path('sinov_model.joblib').exists():
    import joblib, pandas as pd
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.model_selection import train_test_split
    from quvur.xususiyat import tayyorla
    d = pd.read_csv('../datasets/ijara.csv')
    x, y = tayyorla(d), d['broker']
    Xtr, _, ytr, _ = train_test_split(x, y, test_size=0.25, stratify=y, random_state=42)
    m = RandomForestClassifier(n_estimators=150, min_samples_leaf=3,
                               random_state=42, n_jobs=1).fit(Xtr, ytr)
    joblib.dump(m, 'sinov_model.joblib')
    print('model yaratildi (25-bobdagidek)')
else:
    print('model allaqachon bor')
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import joblib, pandas as pd
from quvur.format import onnx_ga
from quvur.xususiyat import XUSUSIYATLAR

m = joblib.load('sinov_model.joblib')
p = onnx_ga(m, len(XUSUSIYATLAR), 'model.onnx')
print('ONNX yozildi:', p.name, p.stat().st_size // 1024, 'KB')
print('joblib       :', __import__('pathlib').Path('sinov_model.joblib').stat().st_size // 1024, 'KB')
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import joblib, numpy as np, pandas as pd
from quvur.format import onnx_sessiya, onnx_ehtimol
from quvur.xususiyat import tayyorla

d = pd.read_csv('../datasets/ijara.csv').head(500)
x = tayyorla(d)

m = joblib.load('sinov_model.joblib')
sk = m.predict_proba(x)[:, 1]
s = onnx_sessiya('model.onnx')
on = onnx_ehtimol(s, x.values)

farq = np.abs(sk - on)
print(f'eng katta farq : {farq.max():.2e}')
print(f'o\'rtacha farq  : {farq.mean():.2e}')
print(f'bir xil qaror  : {int((( sk>0.5)==(on>0.5)).sum())} / {len(sk)}')
"

python3 -c "
import sys, time; sys.path.insert(0, 'src')
import joblib, numpy as np, pandas as pd
from quvur.format import onnx_sessiya, onnx_ehtimol
from quvur.xususiyat import tayyorla

d = pd.read_csv('../datasets/ijara.csv')
x = tayyorla(d)
m = joblib.load('sinov_model.joblib')
s = onnx_sessiya('model.onnx')
bitta = x.head(1); bitta_np = bitta.values.astype('float32')

def olch(f, n):
    f(); t0 = time.perf_counter()
    for _ in range(n): f()
    return (time.perf_counter() - t0) / n * 1000

print('BITTA qator (xizmat holati):')
print(f'  sklearn : {olch(lambda: m.predict_proba(bitta), 200):.3f} ms')
print(f'  ONNX    : {olch(lambda: onnx_ehtimol(s, bitta_np), 200):.3f} ms')
print()
toplam = x.head(2000); toplam_np = toplam.values.astype('float32')
print('2000 qator (to\'plam holati):')
print(f'  sklearn : {olch(lambda: m.predict_proba(toplam), 20):.2f} ms')
print(f'  ONNX    : {olch(lambda: onnx_ehtimol(s, toplam_np), 20):.2f} ms')
"

python3 -c "
import sys, time; sys.path.insert(0, 'src')
sys.path.insert(0, 'src')

def olch(f, n=8):
    f(); t0 = time.perf_counter()
    for _ in range(n): f()
    return (time.perf_counter() - t0) / n * 1000

import joblib
from quvur.format import onnx_sessiya
print(f'joblib yuklash : {olch(lambda: joblib.load(\"sinov_model.joblib\")):.1f} ms')
print(f'ONNX yuklash   : {olch(lambda: onnx_sessiya(\"model.onnx\")):.1f} ms')
"

import pandas as pd
show(pd.DataFrame([
    {"holat": "Tajriba, lokal ish", "format": "joblib", "sabab": "Eng oson, sklearn obyekti saqlanadi"},
    {"holat": "Xizmat (inference)", "format": "ONNX", "sabab": "Tez, xavfsiz, tildan mustaqil"},
    {"holat": "Model boshqa jamoaga beriladi", "format": "ONNX", "sabab": "Ular Python ishlatmasligi mumkin"},
    {"holat": "Chuqur o'rganish vaznlari", "format": "safetensors", "sabab": "Tez va xavfsiz yuklash"},
    {"holat": "Telefon/brauzerda ishlash", "format": "ONNX", "sabab": "onnxruntime-web, mobil qo'llab-quvvatlash"},
    {"holat": "Modelni qayta o'qitish kerak", "format": "joblib", "sabab": "ONNX faqat inference uchun"},
]))

python3 -c "
import sys; sys.path.insert(0, 'src')
import numpy as np, pandas as pd
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from quvur.format import onnx_ga, onnx_sessiya, onnx_ehtimol
from quvur.xususiyat import XUSUSIYATLAR, tayyorla

d = pd.read_csv('../datasets/ijara.csv')
x, y = tayyorla(d), d['broker']
Xtr, Xte, ytr, yte = train_test_split(x, y, test_size=0.25, stratify=y, random_state=42)
kv = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000, random_state=42)).fit(Xtr, ytr)

onnx_ga(kv, len(XUSUSIYATLAR), 'quvur.onnx')
s = onnx_sessiya('quvur.onnx')
a = kv.predict_proba(Xte.head(200))[:, 1]
b = onnx_ehtimol(s, Xte.head(200).values)
print('Pipeline (scaler + model) ONNX ga tushdi')
print(f'eng katta farq: {np.abs(a-b).max():.2e}')
print('=> StandardScaler ham grafga kirdi — xizmatda alohida kod kerak emas')
"