Fine-tuning: tayyor modelni moslashtirish

import os
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import time
import numpy as np
import pandas as pd

xabarlar = pd.read_csv(DATA / "mijoz_xabarlari.csv")
print(f"Xabarlar: {len(xabarlar)}")
print()
print(xabarlar["maqsad"].value_counts().to_string())
print()
print("Namunalar:")
for _, q in xabarlar.head(6).iterrows():
    belgi = "QIYIN" if q["qiyin"] else "oson"
    print(f"  [{q['maqsad']:<8}] [{belgi:<5}] {q['xabar']}")

from sklearn.dummy import DummyClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split

sinflar = sorted(xabarlar["maqsad"].unique())
sinf_raqami = {s: i for i, s in enumerate(sinflar)}
y = xabarlar["maqsad"].map(sinf_raqami).values
X = xabarlar["xabar"].values

tr, te = train_test_split(np.arange(len(xabarlar)), test_size=0.25,
                          random_state=42, stratify=y)
print(f"train {len(tr)}, test {len(te)}, {len(sinflar)} sinf: {sinflar}")
print()

baza = DummyClassifier(strategy="most_frequent").fit(X[tr], y[tr])
tfidf = make_pipeline(TfidfVectorizer(ngram_range=(1, 2), min_df=1),
                      LogisticRegression(max_iter=2000)).fit(X[tr], y[tr])

t0 = time.perf_counter()
tfidf_vaqt = time.perf_counter() - t0
print(f"Baza (eng ko'p sinf)  : {baza.score(X[te], y[te]):.4f}")
print(f"TF-IDF + LogReg       : {tfidf.score(X[te], y[te]):.4f}")

qiyin_maska = xabarlar["qiyin"].values[te].astype(bool)
print("TF-IDF ni qismlarga bo'lib tekshiramiz:")
print(f"  OSON misollarda : {tfidf.score(X[te][~qiyin_maska], y[te][~qiyin_maska]):.4f}  ({(~qiyin_maska).sum()} misol)")
print(f"  QIYIN misollarda: {tfidf.score(X[te][qiyin_maska], y[te][qiyin_maska]):.4f}  ({qiyin_maska.sum()} misol)")
print()
print("Mening taxminim: TF-IDF qiyin misollarda yiqiladi.")
print("O'lchov: yiqilmadi.")

show(pd.DataFrame([
    {"Yondashuv": "Noldan o'qitish", "Ma'lumot": "millionlab", "Narxi": "minglab GPU-soat",
     "Qachon": "Deyarli hech qachon — sizning vazifangiz emas"},
    {"Yondashuv": "Fine-tuning (to'liq)", "Ma'lumot": "minglab", "Narxi": "soatlar",
     "Qachon": "Ko'p ma'lumot bor, vazifa pretraining dan uzoq"},
    {"Yondashuv": "LoRA / QLoRA", "Ma'lumot": "yuzlab-minglab", "Narxi": "daqiqalar",
     "Qachon": "Odatiy tanlov — arzon va deyarli shunday yaxshi"},
    {"Yondashuv": "Faqat bosh qatlam", "Ma'lumot": "yuzlab", "Narxi": "daqiqalar",
     "Qachon": "Juda kam ma'lumot"},
    {"Yondashuv": "Prompt / RAG", "Ma'lumot": "0", "Narxi": "yo'q",
     "Qachon": "Bilim kerak, uslub emas — o'qitish shart emas"},
]))

from transformers import AutoTokenizer

namunalar = ["Bugun to'lasam narx o'zgaradimi?",
             "Konditsioner buzilgan, tuzatib bering",
             "Bronni bekor qilmoqchiman"]
tok_jadval = []
for nom in ["distilroberta-base", "distilbert-base-multilingual-cased"]:
    tok = AutoTokenizer.from_pretrained(nom)
    tokenlar = [len(tok.tokenize(s)) for s in namunalar]
    belgilar = [len(s) for s in namunalar]
    tok_jadval.append({"model": nom, "lug'at hajmi": f"{tok.vocab_size:,}",
                       "tokenlar": str(tokenlar),
                       "token/belgi": round(sum(tokenlar) / sum(belgilar), 3)})
show(pd.DataFrame(tok_jadval))

for nom in ["distilroberta-base", "distilbert-base-multilingual-cased"]:
    tok = AutoTokenizer.from_pretrained(nom)
    print(f"{nom}:")
    print(f"  {tok.tokenize(namunalar[0])}")

"""Matn klassifikatsiyasi uchun fine-tuning: to'liq va LoRA."""
import time

import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from transformers import AutoModelForSequenceClassification, AutoTokenizer

MODEL = "distilbert-base-multilingual-cased"
MAKS_UZUNLIK = 48


def tayyorla(matnlar, model_nomi=MODEL, maks=MAKS_UZUNLIK):
    tok = AutoTokenizer.from_pretrained(model_nomi)
    return tok(list(matnlar), truncation=True, max_length=maks,
               padding="max_length", return_tensors="pt")


def oqit(enc, y, train_idx, test_idx, n_sinf, lora=False, epoxalar=3,
         lr=None, seed=0, model_nomi=MODEL):
    """Modelni o'qitadi va (aniqlik, o'qitiladigan, jami, vaqt, model) qaytaradi.

    lora=True bo'lsa, asosiy vaznlar MUZLATILADI va faqat kichik
    adapter matritsalari o'qitiladi.
    """
    torch.manual_seed(seed)
    model = AutoModelForSequenceClassification.from_pretrained(model_nomi, num_labels=n_sinf)
    if lora:
        from peft import LoraConfig, get_peft_model
        model = get_peft_model(model, LoraConfig(
            task_type="SEQ_CLS", r=8, lora_alpha=16, lora_dropout=0.05,
            target_modules=["q_lin", "v_lin"]))      # faqat attention Q va V

    oqitiladigan = sum(p.numel() for p in model.parameters() if p.requires_grad)
    jami = sum(p.numel() for p in model.parameters())

    def bolak(idx):
        return TensorDataset(enc["input_ids"][idx], enc["attention_mask"][idx],
                             torch.tensor(y[idx]))

    opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad],
                            lr=lr or (5e-4 if lora else 3e-5))
    yuklovchi = DataLoader(bolak(train_idx), batch_size=16, shuffle=True,
                           generator=torch.Generator().manual_seed(seed))
    t0 = time.perf_counter()
    model.train()
    for _ in range(epoxalar):
        for ids, maska, nishon in yuklovchi:
            opt.zero_grad()
            model(input_ids=ids, attention_mask=maska, labels=nishon).loss.backward()
            opt.step()
    vaqt = time.perf_counter() - t0

    model.eval()
    togri = 0
    with torch.no_grad():
        for ids, maska, nishon in DataLoader(bolak(test_idx), batch_size=32):
            bashorat = model(input_ids=ids, attention_mask=maska).logits.argmax(-1)
            togri += int((bashorat == nishon).sum())
    return togri / len(test_idx), oqitiladigan, jami, vaqt, model

import sys
sys.path.insert(0, str(ISH))
from moslashtirish.oqit import tayyorla, oqit

enc = tayyorla(X)
print(f"Tokenizatsiya tayyor: {tuple(enc['input_ids'].shape)}")
print()
natijalar = []
modellar = {}
for nom, lora in [("to'liq fine-tuning", False), ("LoRA (r=8)", True)]:
    aniqlik, oqitiladigan, jami, vaqt, model = oqit(enc, y, tr, te, len(sinflar), lora=lora)
    modellar[nom] = model
    natijalar.append({"usul": nom, "aniqlik": round(aniqlik, 4),
                      "o'qitiladigan": oqitiladigan, "jami parametr": jami,
                      "ulushi": f"{oqitiladigan / jami:.2%}",
                      "vaqt (s)": round(vaqt, 1)})
show(pd.DataFrame(natijalar))

import shutil
from pathlib import Path

print(f"{'usul':<22} {'diskda':>12}")
print("-" * 36)
for nom, model in modellar.items():
    papka = ISH / ("saqlangan_" + ("lora" if "LoRA" in nom else "toliq"))
    shutil.rmtree(papka, ignore_errors=True)
    model.save_pretrained(papka)
    hajm = sum(p.stat().st_size for p in Path(papka).rglob("*") if p.is_file())
    print(f"{nom:<22} {hajm / 1e6:>9.2f} MB")
print()
print("LoRA da faqat adapter saqlanadi — asosiy model o'zgarmagan, qayta saqlash shart emas.")

KALIT = "o'qitiladigan"
taqqoslash = pd.DataFrame([
    {"usul": "baza (eng ko'p sinf)", "aniqlik": round(baza.score(X[te], y[te]), 4),
     "o'qitilgan parametr": "0", "izoh": "hech narsa o'rganmaydi"},
    {"usul": "TF-IDF + LogReg", "aniqlik": round(tfidf.score(X[te], y[te]), 4),
     "o'qitilgan parametr": "minglab", "izoh": "bir soniyada o'qitiladi"},
    {"usul": natijalar[0]["usul"], "aniqlik": natijalar[0]["aniqlik"],
     "o'qitilgan parametr": f"{natijalar[0][KALIT]:,}", "izoh": "hamma vazn o'zgaradi"},
    {"usul": natijalar[1]["usul"], "aniqlik": natijalar[1]["aniqlik"],
     "o'qitilgan parametr": f"{natijalar[1][KALIT]:,}", "izoh": "adapter — bir necha MB"},
])
show(taqqoslash)

show(pd.DataFrame([
    {"Tuzoq": "lr juda katta", "Belgisi": "loss NaN yoki aniqlik tasodifiy darajada",
     "Yechim": "to'liq FT uchun 2e-5..5e-5; LoRA uchun 1e-4..1e-3"},
    {"Tuzoq": "Katastrofik unutish", "Belgisi": "model yangi vazifada yaxshi, boshqa hammasida yomon",
     "Yechim": "kamroq epoxa, kichik lr, yoki LoRA ishlatish"},
    {"Tuzoq": "Kam ma'lumotda ko'p epoxa", "Belgisi": "train aniqlik 1.0, test pastda qoladi",
     "Yechim": "early stopping, validation kuzatiladi"},
    {"Tuzoq": "Test da sozlash", "Belgisi": "qog'ozda yaxshi, amalda yomon",
     "Yechim": "validation da sozlanadi — 5 va 23-bob"},
    {"Tuzoq": "Nomutanosib sinflar", "Belgisi": "kichik sinf umuman topilmaydi",
     "Yechim": "class weight yoki F1 metrikasi — 7 va 8-bob"},
    {"Tuzoq": "Tasodifiy seed", "Belgisi": "har ishga tushirishda boshqa natija",
     "Yechim": "3 seed, eng yomonini ham yozish — 34-bob"},
]))

show(pd.DataFrame([
    {"Holat": "Kalit so'zlar sinfni aniqlaydi", "To'g'ri yechim": "TF-IDF + logistic regression",
     "Sabab": "shu bobda o'lchandi — arzonroq va yuqoriroq"},
    {"Holat": "Bilim kerak, uslub emas", "To'g'ri yechim": "RAG (qidirish + LLM)",
     "Sabab": "yangi fakt qo'shish uchun o'qitish shart emas"},
    {"Holat": "Ma'lumot 100 dan kam", "To'g'ri yechim": "prompt yoki few-shot",
     "Sabab": "fine-tuning shovqinga moslashadi"},
    {"Holat": "Talab tez-tez o'zgaradi", "To'g'ri yechim": "prompt",
     "Sabab": "har o'zgarishda qayta o'qitish qimmat"},
    {"Holat": "Chiqish formati kerak", "To'g'ri yechim": "fine-tuning YOKI struktura majburlash",
     "Sabab": "uslub va format — fine-tuning kuchli tomoni"},
    {"Holat": "Maxsus domen tili", "To'g'ri yechim": "fine-tuning",
     "Sabab": "tibbiy, huquqiy matn — pretraining dan uzoq"},
]))

show(pd.DataFrame([
    {"Qadam": "36. Ma'lumot yig'ildi", "Holati": "bajarildi"},
    {"Qadam": "37. Dataset qurildi", "Holati": "bajarildi — guruh bo'yicha split"},
    {"Qadam": "38. Baza o'lchandi", "Holati": "TF-IDF yetarli ekani ANIQLANDI"},
    {"Qadam": "38. Fine-tuning o'rganildi", "Holati": "to'liq va LoRA — mexanizm ma'lum"},
    {"Qadam": "39. Modelni saqlash", "Holati": "keyingi bob"},
    {"Qadam": "40. Xizmatga chiqarish", "Holati": "Django API"},
]))

show(pd.DataFrame([
    {"Tushuncha": "Baza avval", "Bitta jumla": "O'lchandi: TF-IDF transformerdan yuqori chiqdi"},
    {"Tushuncha": "Fine-tuning", "Bitta jumla": "Tayyor vaznlardan boshlab o'z ma'lumotida davom etish"},
    {"Tushuncha": "LoRA", "Bitta jumla": "Asosiy vazn muzlatiladi, kichik adapter o'qitiladi"},
    {"Tushuncha": "LoRA afzalligi", "Bitta jumla": "Xotira, saqlash, almashtirish — aniqlik emas"},
    {"Tushuncha": "Tokenizatsiya", "Bitta jumla": "O'lchandi: ikki model ham o'zbek so'zini maydalaydi"},
    {"Tushuncha": "Ko'p tilli model", "Bitta jumla": "O'zbek matni uchun ingliz modelidan afzal"},
    {"Tushuncha": "Katastrofik unutish", "Bitta jumla": "Ko'p epoxa eski bilimni buzadi"},
    {"Tushuncha": "Qachon qilmaslik", "Bitta jumla": "Kalit so'z yetarli bo'lsa, RAG yoki prompt bo'lsa"},
]))