Yakuniy loyiha: hammasini bitta tizimga

import pandas as pd
show(pd.DataFrame([
    {"Talab": "Takrorlanadigan o'qitish", "Qaysi boblar": "2, 5, 24",
     "Nima kerak": "Seed, ma'lumot xeshi, bitta buyruq"},
    {"Talab": "Yomon model chiqmasin", "Qaysi boblar": "6, 24, 25, 31",
     "Nima kerak": "Sifat tekshiruvi, model testlari, darvoza"},
    {"Talab": "Qaysi model ishlayapti", "Qaysi boblar": "22, 23",
     "Nima kerak": "Kuzatuv, registr, alias"},
    {"Talab": "Buzilsa qaytsin", "Qaysi boblar": "27, 28, 30, 32",
     "Nima kerak": "Sog'liq, konteyner, avtomatik qaytarish"},
    {"Talab": "Eskirganini aytsin", "Qaysi boblar": "34, 35",
     "Nima kerak": "O'lchov, PSI, ogohlantirish"},
]))

diagramma(
    {
        "Ma'lumot\n(manifest)": (0, 2.6, 2.1, 1.0, "manba"),
        "Sifat\n(6-bob)": (2.8, 2.6, 1.9, 1.0, "qayta"),
        "O'qitish\n(24-bob)": (5.4, 2.6, 2.0, 1.0, "qayta"),
        "Darvoza\n(31-bob)": (8.1, 2.6, 2.0, 1.0, "kuzatuv"),
        "Registr\n(23-bob)": (10.8, 2.6, 2.0, 1.0, "saqlash"),
        "Servis\n(27-28)": (10.8, 0.9, 2.0, 1.0, "xizmat"),
        "Monitoring\n(34-35)": (8.1, 0.9, 2.0, 1.0, "kuzatuv"),
    },
    [
        ("Ma'lumot\n(manifest)", "Sifat\n(6-bob)"),
        ("Sifat\n(6-bob)", "O'qitish\n(24-bob)"),
        ("O'qitish\n(24-bob)", "Darvoza\n(31-bob)"),
        ("Darvoza\n(31-bob)", "Registr\n(23-bob)", "o'tdi"),
        ("Registr\n(23-bob)", "Servis\n(27-28)"),
        ("Servis\n(27-28)", "Monitoring\n(34-35)"),
        ("Monitoring\n(34-35)", "O'qitish\n(24-bob)", "drift"),
    ],
    nom="yakuniy_tizim.png", kenglik=14,
    izoh="Halqa yopiq: monitoring driftni sezsa, o'qitish qaytadan boshlanadi.",
)

"""Yakuniy quvur: ma'lumotdan monitoring darajasigacha."""
import hashlib
import json
import time
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Any

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score, precision_score, recall_score
from sklearn.model_selection import train_test_split

from .xususiyat import XUSUSIYATLAR, tayyorla


@dataclass(frozen=True, slots=True)
class Pasport:
    """2-bobdagi pasport + 5-bobdagi ma'lumot xeshi."""
    malumot_xesh: str
    qatorlar: int
    seed: int
    parametrlar: dict[str, Any]
    metrikalar: dict[str, float]
    vaqt: str


def malumot_xesh(yol: str | Path) -> str:
    h = hashlib.sha256()
    with open(yol, "rb") as f:
        for blok in iter(lambda: f.read(1 << 20), b""):
            h.update(blok)
    return h.hexdigest()[:16]


QOIDA = {
    "narx_mln": lambda s: s.between(0.5, 200),
    "maydon": lambda s: s.between(10, 500),
    "xonalar": lambda s: s.between(1, 10),
}


def sifat(d: pd.DataFrame) -> tuple[pd.DataFrame, int]:
    """6-bob: chegarada tekshirish."""
    yaxshi = pd.Series(True, index=d.index)
    for ustun, tekshir in QOIDA.items():
        yaxshi &= tekshir(d[ustun])
    return d[yaxshi], int((~yaxshi).sum())


def oqit(d: pd.DataFrame, seed: int = 42) -> tuple[Any, dict[str, float]]:
    """24-bob: bitta funksiya, takrorlanadigan."""
    x, y = tayyorla(d), d["broker"]
    Xtr, Xte, ytr, yte = train_test_split(
        x, y, test_size=0.25, stratify=y, random_state=seed)
    m = RandomForestClassifier(n_estimators=150, min_samples_leaf=3,
                               random_state=seed, n_jobs=1).fit(Xtr, ytr)
    p = m.predict(Xte)
    return m, {
        "f1": float(f1_score(yte, p)),
        "aniqlik": float(precision_score(yte, p)),
        "toliqlik": float(recall_score(yte, p)),
    }

python3 -c "
import sys; sys.path.insert(0, 'src')
import pandas as pd
from quvur.loyiha_oxiri import malumot_xesh, sifat

YOL = '../datasets/ijara.csv'
d = pd.read_csv(YOL)
xesh = malumot_xesh(YOL)
toza, rad = sifat(d)
print(f'1-BOSQICH: MALUMOT')
print(f'  malumot xeshi : {xesh}')
print(f'  kirgan qator  : {len(d):,}')
print(f'  rad etilgan   : {rad:,}')
print(f'  toza          : {len(toza):,}')
"

python3 -c "
import sys, json, time; sys.path.insert(0, 'src')
import joblib, pandas as pd
from quvur.loyiha_oxiri import Pasport, malumot_xesh, oqit, sifat
from quvur.darvoza import Chegara, asos_oqi, darvoza, hisobot
from dataclasses import asdict

YOL = '../datasets/ijara.csv'
d, _ = sifat(pd.read_csv(YOL))
m, metrikalar = oqit(d, seed=42)

p = Pasport(
    malumot_xesh=malumot_xesh(YOL), qatorlar=len(d), seed=42,
    parametrlar={'n_estimators': 150, 'min_samples_leaf': 3},
    metrikalar=metrikalar,
    vaqt=time.strftime('%Y-%m-%d %H:%M'),
)

ok, sabablar = darvoza(metrikalar, Chegara(), asos_oqi('ci_asos.json'))
print('2-BOSQICH: OQITISH VA DARVOZA')
print(hisobot(metrikalar, ok, sabablar))
if ok:
    joblib.dump(m, 'yakuniy_model.joblib')
    open('yakuniy_pasport.json','w').write(json.dumps(asdict(p), indent=1))
    print()
    print('  model va pasport saqlandi')
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import joblib
from quvur.sinov_yordam import ehtimol

m = joblib.load('yakuniy_model.joblib')

# 25-bob: minimal funksionallik
ega = ehtimol(m, telefon_takrorlanishi=1, kunlik_elonlar=1,
              tavsif_uzunligi=600, rasmlar_soni=15, egasi_tasdiqlangan=1)
brok = ehtimol(m, telefon_takrorlanishi=18, kunlik_elonlar=9,
               tavsif_uzunligi=60, rasmlar_soni=2, egasi_tasdiqlangan=0)

# 25-bob: yonalish
past = ehtimol(m, telefon_takrorlanishi=1)
yuqori = ehtimol(m, telefon_takrorlanishi=20)

print('3-BOSQICH: MODEL TESTLARI')
print(f'  aniq ega          : {ega:.4f}  {\"OK\" if ega < 0.3 else \"XATO\"}')
print(f'  aniq broker       : {brok:.4f}  {\"OK\" if brok > 0.7 else \"XATO\"}')
print(f'  telefon 1 -> 20   : {past:.4f} -> {yuqori:.4f}  {\"OK\" if yuqori > past else \"XATO\"}')
"

python3 -c "
import sys, json, pathlib; sys.path.insert(0, 'src')
import joblib
from quvur.xususiyat import XUSUSIYATLAR
from quvur.format import onnx_ga, onnx_sessiya, onnx_ehtimol
import numpy as np, pandas as pd
from quvur.xususiyat import tayyorla

m = joblib.load('yakuniy_model.joblib')
onnx_ga(m, len(XUSUSIYATLAR), 'yakuniy.onnx')

# 26-bob: ONNX bir xil javob beradimi
d = pd.read_csv('../datasets/ijara.csv').head(500)
x = tayyorla(d)
a = m.predict_proba(x)[:, 1]
s = onnx_sessiya('yakuniy.onnx')
b = onnx_ehtimol(s, x.values)
farq = float(np.max(np.abs(a - b)))
qaror = int(np.sum((a >= 0.5) == (b >= 0.5)))

print('4-BOSQICH: ONNX')
print(f'  joblib hajmi : {pathlib.Path(\"yakuniy_model.joblib\").stat().st_size/1024/1024:.1f} MB')
print(f'  onnx hajmi   : {pathlib.Path(\"yakuniy.onnx\").stat().st_size/1024/1024:.1f} MB')
print(f'  eng katta farq: {farq:.2e}')
print(f'  qaror mosligi : {qaror}/{len(d)}')
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import pandas as pd
from quvur.drift import hisobot, KATTA

eski = pd.read_csv('../datasets/ijara.csv')
yangi = pd.read_csv('../datasets/ijara_yangi.csv')
USTUN = ['tavsif_uzunligi','rasmlar_soni','telefon_takrorlanishi','kunlik_elonlar','narx_mln']
h = hisobot(eski, yangi, USTUN)

print('5-BOSQICH: DRIFT')
print(h.to_string(index=False))
ogoh = h[h['psi'] >= 0.10]
print()
if len(ogoh):
    print(f'  OGOHLANTIRISH: {len(ogoh)} ustunda siljish')
    print(f'  qaror: qayta oqitish kerak (35-bob)')
else:
    print('  siljish yoq - tegmang')
"

python3 -c "
import json, pathlib
p = json.load(open('yakuniy_pasport.json'))
print('=' * 50)
print('LOYIHA HISOBOTI')
print('=' * 50)
print(f'  malumot xeshi : {p[\"malumot_xesh\"]}')
print(f'  qatorlar      : {p[\"qatorlar\"]:,}')
print(f'  seed          : {p[\"seed\"]}')
print(f'  F1            : {p[\"metrikalar\"][\"f1\"]:.4f}')
print(f'  aniqlik       : {p[\"metrikalar\"][\"aniqlik\"]:.4f}')
print(f'  toliqlik      : {p[\"metrikalar\"][\"toliqlik\"]:.4f}')
print(f'  vaqt          : {p[\"vaqt\"]}')
print('-' * 50)
for f in ('yakuniy_model.joblib', 'yakuniy.onnx', 'yakuniy_pasport.json'):
    bor = pathlib.Path(f).exists()
    print(f'  {f:26s} {\"bor\" if bor else \"YOQ\"}')
print('=' * 50)
"

rm -f yakuniy_model.joblib yakuniy.onnx yakuniy_pasport.json
echo "vaqtinchalik fayllar tozalandi"