chunker.py — matnni bo'laklash

[==========]
       [==========]
              [==========]
       ^^^^^ ustma-ust

Paragraf 1  \
Paragraf 2   > 1-chunk (limitga sig'guncha)
Paragraf 3  /

Paragraf 4  \
Paragraf 5   > 2-chunk

Paragraf 5: "Nihol Premium tarifi eng qimmat variant."
--- chunk chegarasi ---
Paragraf 6: "Narxi oyiga 450 000 so'm."

from app.config import settings

MAX_TOKENS = 2000
CHARS_PER_TOKEN = 3


def _estimate_tokens(text: str) -> int:
    """Token sonini TAXMINIY hisoblaydi."""
    return len(text) // CHARS_PER_TOKEN


def _overlap_parts(parts: list[str], overlap_tokens: int) -> list[str]:
    """Yopilgan chunk oxiridan keyingi chunk uchun paragraf tanlaydi."""
    if overlap_tokens <= 0 or not parts:
        return []

    tanlangan, jami = [], 0
    for p in reversed(parts):              # oxiridan orqaga
        t = _estimate_tokens(p)
        if jami + t > overlap_tokens and tanlangan:
            break
        tanlangan.insert(0, p)
        jami += t

    # HAMMASINI qaytarmaymiz — aks holda sikl oldinga siljimaydi
    if len(tanlangan) >= len(parts):
        tanlangan = tanlangan[1:]
    return tanlangan


def chunk_text(text: str, title: str = "",
               max_tokens: int = MAX_TOKENS,
               overlap: int | None = None) -> list[str]:
    if overlap is None:
        overlap = settings.chunk_overlap   # .env dagi CHUNK_OVERLAP

    # Overlap chunk dan katta bo'lsa — CHEKSIZ SIKL. Chegaralaymiz.
    overlap = max(0, min(overlap, max_tokens // 2))

    if not text or not text.strip():
        return []

    prefix = title + "\n\n" if title else ""
    full_text = prefix + text

    if _estimate_tokens(full_text) <= max_tokens:
        return [full_text]

    paragraphs = text.split("\n\n")
    chunk_limit = max_tokens - _estimate_tokens(prefix)

    chunks = []
    current_parts = []
    current_tokens = 0

    for paragraph in paragraphs:
        paragraph = paragraph.strip()
        if not paragraph:
            continue

        para_tokens = _estimate_tokens(paragraph)
        would_exceed = (current_tokens + para_tokens) > chunk_limit

        if would_exceed and current_parts:
            chunks.append(prefix + "\n\n".join(current_parts))
            # OVERLAP: oxirgi paragraf(lar)ni keyingi chunk ga ko'chiramiz
            current_parts = _overlap_parts(current_parts, overlap)
            current_tokens = sum(_estimate_tokens(p) for p in current_parts)

        current_parts.append(paragraph)
        current_tokens += para_tokens

    if current_parts:
        chunks.append(prefix + "\n\n".join(current_parts))

    return chunks

return len(text) // CHARS_PER_TOKEN      # 3 ga bo'lamiz

prefix = title + "\n\n" if title else ""

# parcha
if would_exceed and current_parts:

if current_parts:
    chunks.append(prefix + "\n\n".join(current_parts))

matn = "\n\n".join(f"{i}-paragraf. Matn {i} haqida. " * 8 for i in range(20))

for ov in (0, 50, 200):
    c = chunk_text(matn, title="Sinov", max_tokens=400, overlap=ov)
    ust = 0
    for a, b in zip(c, c[1:]):
        pa = set(a.split("\n\n")[1:]); pb = set(b.split("\n\n")[1:])
        ust += len(pa & pb)
    print(f"overlap={ov:<4} chunk={len(c):<3} ustma-ust paragraf={ust}")

overlap=0    chunk=4   ustma-ust paragraf=0
overlap=50   chunk=5   ustma-ust paragraf=4
overlap=200  chunk=6   ustma-ust paragraf=10

# overlap chunk dan KATTA — cheksiz sikl bo'lmasligi kerak
c = chunk_text(matn, title="T", max_tokens=100, overlap=999)
print(f"{len(c)} chunk — sikl tugadi")

print(chunk_text(""))                      # []
print(len(chunk_text("Salom", title="T"))) # 1

# matn yo'qolmadimi
asl = set(matn.split())
yig = set(" ".join(chunk_text(matn, max_tokens=400, overlap=50)).split())
print("yo'qolgan:", asl - yig or "yo'q")

matn = "\n\n".join(f"Bu {i}-paragraf. " * 20 for i in range(30))
b = chunk_text(matn, title="Sinov hujjati", max_tokens=500)

print(f"chunk soni : {len(b)}")
print(f"o'lchamlar : {[len(x) for x in b]}")
print(f"tokenlar   : {[_estimate_tokens(x) for x in b]}")
print(f"sarlavha bormi: {all(x.startswith('Sinov hujjati') for x in b)}")

asl = set(matn.split())
yigilgan = set(" ".join(b).split())
print("yo'qolgan so'zlar:", asl - yigilgan)

python -c "
from scraper.chunker import chunk_text, _estimate_tokens
m = open('hujjat.txt').read()
b = chunk_text(m, title='Hujjat')
print(f'{len(m)} belgi -> {len(b)} chunk')
for i, c in enumerate(b[:3], 1):
    print(f'  {i}: {_estimate_tokens(c)} token, {len(c)} belgi')
"