Ma'lumot sifati: yomon ma'lumot modelga yetib bormasin

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score

XUS = ["xonalar", "maydon", "qavat", "jami_qavat", "narx_mln", "tavsif_uzunligi",
       "rasmlar_soni", "telefon_takrorlanishi", "kunlik_elonlar", "egasi_tasdiqlangan"]

d = pd.read_csv(DATA / "ijara.csv")
Xtr, Xte, ytr, yte = train_test_split(d[XUS], d["broker"], test_size=0.25,
                                      stratify=d["broker"], random_state=42)

def buz(X, tur, ulush=0.25, seed=7):
    """Ma'lumotga haqiqiy hayotdagi nuqsonlarni kiritadi."""
    r = np.random.default_rng(seed)
    x = X.copy()
    idx = r.choice(x.index, int(len(x) * ulush), replace=False)
    if tur == "birlik":                       # mln so'm o'rniga so'm yozilgan
        x.loc[idx, "narx_mln"] = x.loc[idx, "narx_mln"] * 1_000_000
    elif tur == "nol":                        # yetishmagan qiymat 0 bilan to'ldirilgan
        x.loc[idx, "maydon"] = 0
    elif tur == "manfiy":                     # kiritish xatosi
        x.loc[idx, "rasmlar_soni"] = -1
    return x

MODELLAR = {
    "RandomForest (daraxt)": RandomForestClassifier(
        n_estimators=150, min_samples_leaf=3, random_state=42, n_jobs=1),
    "LogisticRegression (chiziqli)": make_pipeline(
        StandardScaler(), LogisticRegression(max_iter=2000, random_state=42)),
}
NUQSONLAR = [("birlik", "narx birligi almashgan (mln -> so'm)"),
             ("nol", "maydon = 0 (yetishmagan qiymat)"),
             ("manfiy", "rasmlar_soni = -1 (mumkin emas)")]

qatorlar = []
tayyor = {}
for nom, model in MODELLAR.items():
    m = model.fit(Xtr, ytr)
    tayyor[nom] = m
    qator = {"model": nom, "toza": round(f1_score(yte, m.predict(Xte)), 3)}
    for tur, _ in NUQSONLAR:
        qator[tur] = round(f1_score(yte, m.predict(buz(Xte, tur))), 3)
    qatorlar.append(qator)
show(pd.DataFrame(qatorlar))
for tur, izoh in NUQSONLAR:
    print(f"  {tur:8} = {izoh}")

diagramma(
    {
        "Tashqi manba\n(CSV, API, oqim)": (0, 1.2, 2.6, 1.1, "manba"),
        "TEKSHIRUV\n(sxema + qoidalar)": (3.4, 1.2, 2.6, 1.1, "qayta"),
        "Toza ma'lumot\n(ishonchli)": (6.8, 0, 2.6, 1.1, "saqlash"),
        "Karantin\n(rad etilganlar)": (6.8, 2.4, 2.6, 1.1, "xizmat"),
    },
    [
        ("Tashqi manba\n(CSV, API, oqim)", "TEKSHIRUV\n(sxema + qoidalar)"),
        ("TEKSHIRUV\n(sxema + qoidalar)", "Toza ma'lumot\n(ishonchli)", "o'tdi"),
        ("TEKSHIRUV\n(sxema + qoidalar)", "Karantin\n(rad etilganlar)", "rad etildi"),
    ],
    nom="tekshiruv_chegarasi.png", kenglik=12,
    izoh="Tekshiruv bitta joyda. Rad etilgan qatorlar yo'qolmaydi — karantinga tushadi va sababi yoziladi.",
)

"""Ma'lumot sxemasi va tekshiruv qoidalari.

Har ustun uchun: turi, ruxsat etilgan oraliq, bo'sh bo'lishi mumkinmi.
Bu "ma'lumot shartnomasi" (data contract) — manba va iste'molchi o'rtasidagi kelishuv.
"""
from dataclasses import dataclass, field
from typing import Any

import pandas as pd


@dataclass(frozen=True, slots=True)
class Ustun:
    nom: str
    tur: str                                  # "son" | "butun" | "matn" | "mantiqiy"
    eng_kam: float | None = None
    eng_ko_p: float | None = None
    bosh_bolishi_mumkin: bool = False
    ruxsat_etilgan: tuple[str, ...] = ()      # matn uchun: faqat shu qiymatlar


@dataclass(frozen=True, slots=True)
class Sxema:
    ustunlar: tuple[Ustun, ...]

    def nomlari(self) -> list[str]:
        return [u.nom for u in self.ustunlar]


def tekshir(df: pd.DataFrame, sxema: Sxema) -> tuple[pd.DataFrame, pd.DataFrame, list[str]]:
    """(toza, karantin, xabarlar) qaytaradi. Istisno OTMAYDI — qarorni chaqiruvchi qabul qiladi."""
    xabarlar: list[str] = []
    yetishmayotgan = set(sxema.nomlari()) - set(df.columns)
    if yetishmayotgan:
        raise ValueError(f"ustunlar yetishmaydi: {sorted(yetishmayotgan)}")

    yomon = pd.Series(False, index=df.index)
    sabablar = pd.Series("", index=df.index)

    def belgila(maska: pd.Series, sabab: str) -> None:
        nonlocal yomon, sabablar
        soni = int(maska.sum())
        if soni:
            xabarlar.append(f"{sabab}: {soni} qator")
            sabablar[maska & (sabablar == "")] = sabab
            yomon = yomon | maska

    for u in sxema.ustunlar:
        s = df[u.nom]
        if not u.bosh_bolishi_mumkin:
            belgila(s.isna(), f"{u.nom} bo'sh")
        if u.tur in {"son", "butun"}:
            son = pd.to_numeric(s, errors="coerce")
            belgila(son.isna() & s.notna(), f"{u.nom} son emas")
            if u.tur == "butun":
                belgila(son.notna() & (son % 1 != 0), f"{u.nom} butun emas")
            if u.eng_kam is not None:
                belgila(son.notna() & (son < u.eng_kam), f"{u.nom} < {u.eng_kam}")
            if u.eng_ko_p is not None:
                belgila(son.notna() & (son > u.eng_ko_p), f"{u.nom} > {u.eng_ko_p}")
        elif u.ruxsat_etilgan:
            belgila(s.notna() & ~s.isin(u.ruxsat_etilgan), f"{u.nom} ruxsat etilmagan qiymat")

    karantin = df[yomon].copy()
    if len(karantin):
        karantin["rad_sababi"] = sabablar[yomon]
    return df[~yomon].copy(), karantin, xabarlar


IJARA_SXEMASI = Sxema((
    Ustun("xonalar", "butun", eng_kam=1, eng_ko_p=10),
    Ustun("maydon", "son", eng_kam=10, eng_ko_p=500),
    Ustun("qavat", "butun", eng_kam=1, eng_ko_p=60),
    Ustun("jami_qavat", "butun", eng_kam=1, eng_ko_p=60),
    Ustun("narx_mln", "son", eng_kam=0.5, eng_ko_p=200),
    Ustun("tavsif_uzunligi", "butun", eng_kam=0, eng_ko_p=5000),
    Ustun("rasmlar_soni", "butun", eng_kam=0, eng_ko_p=100),
    Ustun("telefon_takrorlanishi", "butun", eng_kam=1, eng_ko_p=500),
    Ustun("kunlik_elonlar", "butun", eng_kam=1, eng_ko_p=200),
    Ustun("egasi_tasdiqlangan", "butun", eng_kam=0, eng_ko_p=1),
))

python3 -c "
import sys; sys.path.insert(0, 'src')
import pandas as pd
from quvur.sxema import tekshir, IJARA_SXEMASI

d = pd.read_csv('../datasets/ijara.csv')
toza, karantin, xabarlar = tekshir(d, IJARA_SXEMASI)
print(f'kirish   : {len(d):,} qator')
print(f'toza     : {len(toza):,}')
print(f'karantin : {len(karantin):,}')
print('xabarlar :', xabarlar or '(muammo yo\'q)')
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import numpy as np, pandas as pd
from quvur.sxema import tekshir, IJARA_SXEMASI

r = np.random.default_rng(7)
d = pd.read_csv('../datasets/ijara.csv')
n = len(d)
d.loc[r.choice(n, 300, replace=False), 'narx_mln'] *= 1_000_000   # birlik xatosi
d.loc[r.choice(n, 200, replace=False), 'maydon'] = 0              # yetishmagan -> 0
d.loc[r.choice(n, 150, replace=False), 'rasmlar_soni'] = -1       # manfiy
d.loc[r.choice(n, 100, replace=False), 'xonalar'] = np.nan        # bo'sh

toza, karantin, xabarlar = tekshir(d, IJARA_SXEMASI)
print(f'kirish   : {len(d):,} qator')
print(f'toza     : {len(toza):,}')
print(f'karantin : {len(karantin):,}')
print()
print('TOPILGAN MUAMMOLAR:')
for x in xabarlar:
    print(' ', x)
print()
print('karantin namunasi:')
print(karantin[['maydon','narx_mln','rasmlar_soni','rad_sababi']].head(4).to_string(index=False))
"

import sys
sys.path.insert(0, str(ISH / "src"))
from quvur.sxema import tekshir, IJARA_SXEMASI

r = np.random.default_rng(7)
buzuq = Xte.copy()
n = len(buzuq)
buzuq.loc[r.choice(buzuq.index, int(n * 0.10), replace=False), "narx_mln"] *= 1_000_000
buzuq.loc[r.choice(buzuq.index, int(n * 0.08), replace=False), "maydon"] = 0
buzuq.loc[r.choice(buzuq.index, int(n * 0.06), replace=False), "rasmlar_soni"] = -1

toza_x, karantin_x, _ = tekshir(buzuq, IJARA_SXEMASI)
y_toza = yte.loc[toza_x.index]

qatorlar = []
for nom, m in tayyor.items():
    qatorlar.append({
        "model": nom,
        "toza (ideal)": round(f1_score(yte, m.predict(Xte)), 3),
        "buzuq, tekshiruvsiz": round(f1_score(yte, m.predict(buzuq)), 3),
        "buzuq, tekshiruvdan keyin": round(f1_score(y_toza, m.predict(toza_x)), 3),
    })
show(pd.DataFrame(qatorlar))
print(f"karantinga tushdi: {len(karantin_x)} / {len(buzuq)} qator "
      f"({100 * len(karantin_x) / len(buzuq):.1f}%)")

import pandas as pd
show(pd.DataFrame([
    {"sabab": "Yuqori oqimdagi xatoni topadi",
     "izoh": "narx million barobar oshgan bo'lsa, manba tizimda BUG bor — model chidasa ham"},
    {"sabab": "Hisobot va tahlil buziladi",
     "izoh": "Model chidaydi, lekin o'rtacha narx grafigi butunlay noto'g'ri chiqadi"},
    {"sabab": "Keyingi model chidamasligi mumkin",
     "izoh": "Bugun daraxt, ertaga chiziqli yoki neyron tarmoq"},
    {"sabab": "Sabab ma'lum bo'ladi",
     "izoh": "Karantin 'nima noto'g'ri' degan savolga javob beradi"},
]))

import pandas as pd

def taqsimot_profili(df, ustunlar):
    """Har ustun uchun asosiy statistika — keyin solishtirish uchun."""
    return {u: {"o'rtacha": round(float(df[u].mean()), 3),
                "mediana": round(float(df[u].median()), 3),
                "std": round(float(df[u].std()), 3),
                "bo'sh_ulushi": round(float(df[u].isna().mean()), 4)}
            for u in ustunlar}

KUZATILADIGAN = ["narx_mln", "maydon", "rasmlar_soni", "telefon_takrorlanishi"]
asl = taqsimot_profili(d, KUZATILADIGAN)
yangi_d = pd.read_csv(DATA / "ijara_yangi.csv")
yangi = taqsimot_profili(yangi_d, KUZATILADIGAN)

qatorlar = []
for u in KUZATILADIGAN:
    e, y = asl[u]["o'rtacha"], yangi[u]["o'rtacha"]
    ozgarish = 100 * (y - e) / e if e else 0
    qatorlar.append({"ustun": u, "avgust o'rtacha": e, "sentabr o'rtacha": y,
                     "o'zgarish %": round(ozgarish, 1),
                     "ogohlantirish": "HA" if abs(ozgarish) > 10 else "-"})
show(pd.DataFrame(qatorlar))

import pandas as pd
show(pd.DataFrame([
    {"tur": "Sxema tekshiruvi", "savol": "Qiymat umuman mumkinmi?",
     "misol": "maydon = 0, rasmlar = -1", "qachon": "Har qatorda, darhol"},
    {"tur": "Taqsimot tekshiruvi", "savol": "Ma'lumot odatdagidek ko'rinadimi?",
     "misol": "O'rtacha 15% o'zgardi", "qachon": "To'plam bo'yicha, davriy"},
]))

import pandas as pd
show(pd.DataFrame([
    {"asbob": "Pandera", "kuchli tomoni": "pandas ga mos, dekorator bilan, tur ko'rsatkichlari"},
    {"asbob": "Great Expectations", "kuchli tomoni": "Boy qoidalar to'plami, HTML hisobot, ma'lumot hujjati"},
    {"asbob": "pydantic", "kuchli tomoni": "Bitta yozuv uchun ideal (API kirishi, 27-bob)"},
    {"asbob": "dbt tests", "kuchli tomoni": "SQL omborida, `not_null`, `unique`, `accepted_values`"},
]))

"""Ma'lumot sifati hisoboti — CI va kundalik kuzatuv uchun."""
from typing import Any

import pandas as pd

from .sxema import Sxema, tekshir


def hisobot(df: pd.DataFrame, sxema: Sxema) -> dict[str, Any]:
    toza, karantin, xabarlar = tekshir(df, sxema)
    ulush = len(toza) / len(df) if len(df) else 0.0
    return {
        "jami": len(df),
        "toza": len(toza),
        "karantin": len(karantin),
        "toza_ulushi": round(ulush, 4),
        "muammolar": xabarlar,
        "sabablar": (karantin["rad_sababi"].value_counts().to_dict() if len(karantin) else {}),
    }


def darvoza(h: dict[str, Any], eng_kam_ulush: float = 0.95) -> tuple[bool, str]:
    """CI uchun: sifat yetarlimi? (o'tdimi, sabab)"""
    if h["toza_ulushi"] < eng_kam_ulush:
        return False, (f"toza ulush {h['toza_ulushi']:.1%} < talab {eng_kam_ulush:.0%}")
    return True, f"toza ulush {h['toza_ulushi']:.1%}"

python3 -c "
import sys; sys.path.insert(0, 'src')
import numpy as np, pandas as pd
from quvur.sxema import IJARA_SXEMASI
from quvur.sifat import hisobot, darvoza

d = pd.read_csv('../datasets/ijara.csv')
ok, sabab = darvoza(hisobot(d, IJARA_SXEMASI))
print(f'TOZA ma\'lumot   -> o\'tdimi: {ok}  ({sabab})')

r = np.random.default_rng(7)
buzuq = d.copy()
buzuq.loc[r.choice(len(d), 1500, replace=False), 'maydon'] = 0
h = hisobot(buzuq, IJARA_SXEMASI)
ok, sabab = darvoza(h)
print(f'BUZUQ ma\'lumot  -> o\'tdimi: {ok}  ({sabab})')
print('sabablar:', h['sabablar'])
" ; echo "qaytish kodi: $?"