Retrieval (Qidirish)

import numpy as np

def cosine_similarity(a: list[float], b: list[float]) -> float:
    a, b = np.array(a), np.array(b)
    return float(a @ b / (np.linalg.norm(a) * np.linalg.norm(b)))


q, _ = generate_query_embedding("Ucell loyihasi")
d1, _ = generate_query_embedding("Ucell uchun billing tizimi modernizatsiyasi")
d2, _ = generate_query_embedding("Bugun ob-havo issiq bo'ladi")

print(f"Tegishli    : {cosine_similarity(q, d1):.4f}")
print(f"Tegishli emas: {cosine_similarity(q, d2):.4f}")

session.execute(text(
    "CREATE INDEX IF NOT EXISTS chunks_embedding_hnsw "
    "ON chunks USING hnsw (embedding vector_cosine_ops) "
    "WITH (m = 16, ef_construction = 64)"
))
session.commit()

plan = session.execute(text(
    "EXPLAIN ANALYZE "
    "SELECT id FROM chunks "
    "ORDER BY embedding <=> CAST(:e AS vector) LIMIT 5"
), {"e": str(q)}).scalars().all()

for line in plan:
    print(line)

def search_with_threshold(session, question: str, top_k: int = 5,
                          min_score: float = 0.35) -> list[dict]:
    emb, _ = generate_query_embedding(question)
    rows = session.execute(text(SEARCH_SQL),
                           {"e": str(emb), "k": top_k}).mappings().all()
    return [dict(r) for r in rows if r["score"] >= min_score]


hits = search_with_threshold(session, "kosmik raketa dvigateli")
print(f"Topildi: {len(hits)} ta chunk")

KEYWORD_SQL = (
    "SELECT id, title, content, "
    "       ts_rank(to_tsvector('simple', content), "
    "               plainto_tsquery('simple', :q)) AS score "
    "FROM chunks "
    "WHERE to_tsvector('simple', content) @@ plainto_tsquery('simple', :q) "
    "ORDER BY score DESC "
    "LIMIT :k"
)

def keyword_search(session, question: str, top_k: int = 20) -> list[dict]:
    rows = session.execute(text(KEYWORD_SQL),
                           {"q": question, "k": top_k}).mappings().all()
    return [dict(r) for r in rows]

def rrf_fuse(vector_hits: list[dict], keyword_hits: list[dict],
             k: int = 60, top_k: int = 10) -> list[dict]:
    scores: dict[int, float] = {}
    store: dict[int, dict] = {}

    for rank, hit in enumerate(vector_hits, start=1):
        scores[hit["id"]] = scores.get(hit["id"], 0.0) + 1.0 / (k + rank)
        store[hit["id"]] = hit

    for rank, hit in enumerate(keyword_hits, start=1):
        scores[hit["id"]] = scores.get(hit["id"], 0.0) + 1.0 / (k + rank)
        store.setdefault(hit["id"], hit)

    ordered = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
    out = []
    for chunk_id, score in ordered[:top_k]:
        row = dict(store[chunk_id])
        row["rrf_score"] = score
        out.append(row)
    return out

def hybrid_search(session, question: str, top_k: int = 10) -> list[dict]:
    emb, _ = generate_query_embedding(question)
    vector_hits = retrieve_chunks(session, emb, top_k=20)
    keyword_hits = keyword_search(session, question, top_k=20)
    return rrf_fuse(vector_hits, keyword_hits, top_k=top_k)


question = "MPLS-TE konfiguratsiyasi"
for i, c in enumerate(hybrid_search(session, question, top_k=5), 1):
    print(f"{i}. {c['rrf_score']:.5f}  {c['title'][:45]}")

REWRITE = (
    "Quyidagi suhbat asosida oxirgi savolni to'liq, mustaqil savolga aylantir. "
    "Olmoshlarni (u, bu, o'sha) aniq nom bilan almashtir. "
    "Faqat yangi savolni qaytar, izoh yozma.\n\n"
    "Suhbat:\n{history}\n\nOxirgi savol: {question}"
)

def rewrite_query(question: str, history: list[dict]) -> str:
    if not history:
        return question
    lines = "\n".join(f"{m['role']}: {m['content'][:200]}" for m in history[-4:])
    out = _ollama_chat(
        [{"role": "user", "content": REWRITE.format(history=lines, question=question)}],
        temperature=0.0, max_tokens=80,
    )
    return (out or question).strip()


history = [
    {"role": "user", "content": "Ucell billing loyihasi haqida gapirib bering"},
    {"role": "assistant", "content": "Nihol Ucell uchun billing tizimini yangilagan..."},
]
print(rewrite_query("u qancha vaqt oldi?", history))

STEP_BACK = (
    "Quyidagi savoldan bir pog'ona umumiyroq savol yoz. "
    "Maqsad — kerakli fon ma'lumotini topish. "
    "Faqat savolni qaytar.\n\nSavol: {question}"
)

def step_back_query(question: str) -> str:
    out = _ollama_chat(
        [{"role": "user", "content": STEP_BACK.format(question=question)}],
        temperature=0.2, max_tokens=60,
    )
    return (out or question).strip()


q = "Ucell billing modulida qaysi to'lov shlyuzi ishlatilgan?"
print("Tor  :", q)
print("Keng :", step_back_query(q))

DECOMPOSE = (
    "Quyidagi savolni mustaqil qidiriladigan kichik savollarga ajrat. "
    "Har birini yangi qatorda yoz, raqamlama, izoh yozma. "
    "Agar savol allaqachon oddiy bo'lsa — o'zini qaytar.\n\nSavol: {question}"
)

def decompose_query(question: str, max_parts: int = 4) -> list[str]:
    out = _ollama_chat(
        [{"role": "user", "content": DECOMPOSE.format(question=question)}],
        temperature=0.2, max_tokens=150,
    )
    parts = [p.strip(" -•") for p in (out or "").split("\n") if len(p.strip()) > 10]
    return parts[:max_parts] or [question]


for p in decompose_query("Ucell va Sharq Telekom loyihalarini narx bo'yicha solishtiring"):
    print("-", p)

VARIANTS = (
    "Quyidagi savolni 4 xil usulda qayta yoz. "
    "Har biri boshqa so'zlar va boshqa uslub ishlatsin, lekin ma'no bir xil qolsin. "
    "Har birini yangi qatorda yoz, raqamlama.\n\nSavol: {question}"
)

def generate_variants(question: str, n: int = 4) -> list[str]:
    out = _ollama_chat(
        [{"role": "user", "content": VARIANTS.format(question=question)}],
        temperature=0.7, max_tokens=200,
    )
    variants = [v.strip(" -•0123456789.") for v in (out or "").split("\n")]
    variants = [v for v in variants if len(v) > 10]
    return [question] + variants[:n]


for v in generate_variants("Ucell uchun nima qilingan?"):
    print("-", v)

def multi_query_retrieve(session, question: str, top_k: int = 10) -> list[dict]:
    variants = generate_variants(question)

    scores: dict[int, float] = {}
    store: dict[int, dict] = {}
    hit_count: dict[int, int] = {}

    for variant in variants:
        emb, err = generate_query_embedding(variant)
        if err or emb is None:
            continue
        for rank, hit in enumerate(retrieve_chunks(session, emb, top_k=10), start=1):
            cid = hit["id"]
            scores[cid] = scores.get(cid, 0.0) + 1.0 / (60 + rank)
            hit_count[cid] = hit_count.get(cid, 0) + 1
            store[cid] = hit

    ordered = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
    out = []
    for cid, score in ordered[:top_k]:
        row = dict(store[cid])
        row["rrf_score"] = score
        row["found_in"] = hit_count[cid]
        out.append(row)
    return out


for i, c in enumerate(multi_query_retrieve(session, "Ucell uchun nima qilingan?", 5), 1):
    print(f"{i}. {c['rrf_score']:.5f}  ({c['found_in']}/5 variantda)  {c['title'][:40]}")

from sqlalchemy.orm import Session
from app.services.chat import _ollama_chat, generate_query_embedding
from app.services.retrieval import retrieve_chunks

HYDE_PROMPT = (
    "Quyidagi savolga javob bo'la oladigan qisqa, faktik paragraf yoz. "
    "Bilmasang ham 'bilmayman' deb yozma — ishonarli javob yozaver. "
    "Faqat paragrafni qaytar, boshqa hech narsa yozma.

"
    "Savol: {question}"
)

def generate_hypothetical_answer(question: str) -> str:
    prompt = HYDE_PROMPT.format(question=question)
    answer = _ollama_chat(
        [{"role": "user", "content": prompt}],
        temperature=0.3,
        max_tokens=200,
    )
    return (answer or "").strip()


q = "Nihol kompaniyasi Ucell uchun qanday loyiha qilgan?"
print(generate_hypothetical_answer(q))

def hyde_retrieve(session: Session, question: str, top_k: int = 10) -> list[dict]:
    hypothetical = generate_hypothetical_answer(question)
    if not hypothetical:
        embedding, _ = generate_query_embedding(question)
        return retrieve_chunks(session, embedding, top_k)

    embedding, err = generate_query_embedding(hypothetical)
    if err or embedding is None:
        embedding, _ = generate_query_embedding(question)

    return retrieve_chunks(session, embedding, top_k)

def compare_hyde(session: Session, question: str, top_k: int = 5):
    plain_emb, _ = generate_query_embedding(question)
    plain = retrieve_chunks(session, plain_emb, top_k)
    hyde = hyde_retrieve(session, question, top_k)

    print(f"SAVOL: {question}
")
    print(f"{'#':<3} {'ODDIY':<42} {'HyDE':<42}")
    print("-" * 90)
    for i in range(top_k):
        a = plain[i]["title"][:40] if i < len(plain) else "-"
        b = hyde[i]["title"][:40] if i < len(hyde) else "-"
        print(f"{i+1:<3} {a:<42} {b:<42}")

    plain_ids = {c["id"] for c in plain}
    hyde_ids = {c["id"] for c in hyde}
    overlap = len(plain_ids & hyde_ids) / max(1, len(plain_ids))
    print(f"
Ust-ust tushishi: {overlap:.0%}")


compare_hyde(session, "Ucell uchun nima qilingan?")

import json

SCHEMA = {
    "year": "integer — loyiha yili, masalan 2023",
    "sector": "string — soha: bank, telekom, davlat, ta'lim, tibbiyot",
    "status": "string — holat: tugagan, jarayonda",
}

PARSE = (
    "Savoldan qidiruv qismini va filtrlarni ajrat.\n"
    "Mavjud maydonlar:\n{schema}\n\n"
    "JSON qaytar, boshqa hech narsa yozma:\n"
    '{{"query": "semantik qism", "filters": {{"maydon": ["operator", qiymat]}}}}\n'
    "Operatorlar: eq, gt, gte, lt, lte\n"
    "Filtr yo'q bo'lsa filters bo'sh obyekt bo'lsin.\n\n"
    "Savol: {question}"
)

def parse_self_query(question: str) -> dict:
    schema_text = "\n".join(f"- {k}: {v}" for k, v in SCHEMA.items())
    out = _ollama_chat(
        [{"role": "user", "content": PARSE.format(schema=schema_text, question=question)}],
        temperature=0.0, max_tokens=200,
    )
    try:
        start, end = out.index("{"), out.rindex("}") + 1
        parsed = json.loads(out[start:end])
    except (ValueError, json.JSONDecodeError):
        return {"query": question, "filters": {}}

    clean = {k: v for k, v in parsed.get("filters", {}).items() if k in SCHEMA}
    return {"query": parsed.get("query") or question, "filters": clean}


print(parse_self_query("2023 yildan keyingi tugagan bank loyihalari"))

OPERATORS = {"eq": "=", "gt": ">", "gte": ">=", "lt": "<", "lte": "<="}

def build_where(filters: dict) -> tuple[str, dict]:
    clauses, params = [], {}
    for i, (field, rule) in enumerate(filters.items()):
        if field not in SCHEMA or not isinstance(rule, list) or len(rule) != 2:
            continue
        op, value = rule
        if op not in OPERATORS:
            continue
        key = f"p{i}"
        clauses.append(f"{field} {OPERATORS[op]} :{key}")
        params[key] = value
    return (" AND ".join(clauses) or "TRUE"), params

def self_query_search(session, question: str, top_k: int = 5) -> list[dict]:
    parsed = parse_self_query(question)
    where, params = build_where(parsed["filters"])

    emb, err = generate_query_embedding(parsed["query"])
    if err or emb is None:
        return []

    sql = (
        "SELECT id, title, content, year, sector, "
        "       1 - (embedding <=> CAST(:e AS vector)) AS score "
        "FROM chunks "
        f"WHERE {where} "
        "ORDER BY embedding <=> CAST(:e AS vector) "
        "LIMIT :k"
    )
    params.update({"e": str(emb), "k": top_k})
    rows = session.execute(text(sql), params).mappings().all()
    return [dict(r) for r in rows]


for c in self_query_search(session, "2023 yildan keyingi bank loyihalari"):
    print(f"{c['year']}  {c['sector']:<10}  {c['score']:.3f}  {c['title'][:40]}")

HYPE_PROMPT = (
    "Quyidagi matn qaysi savollarga javob beradi? "
    "{n} ta aniq savol yoz. Har birini yangi qatorda, raqamlamasdan. "
    "Savollar matndagi turli faktlarga tegsin.\n\nMatn: {chunk}"
)

def generate_hypothetical_prompts(chunk: str, n: int = 4) -> list[str]:
    out = _ollama_chat(
        [{"role": "user", "content": HYPE_PROMPT.format(n=n, chunk=chunk)}],
        temperature=0.4, max_tokens=300,
    )
    items = [q.strip(" -•0123456789.") for q in (out or "").split("\n")]
    return [q for q in items if q.endswith("?") and len(q) > 15][:n]

HYPE_SQL = (
    "INSERT INTO hype_index (chunk_id, question, chunk_text, embedding) "
    "VALUES (:cid, :q, :body, :e)"
)

def index_hype(session, title: str, body: str) -> dict:
    stats = {"chunks": 0, "questions": 0}

    for chunk in chunk_text(body, size=800):
        chunk_id = session.execute(
            text(PARENT_SQL), {"t": title, "c": chunk}
        ).scalar_one()
        stats["chunks"] += 1

        for question in generate_hypothetical_prompts(chunk):
            emb, err = generate_query_embedding(question)
            if err or emb is None:
                continue
            session.execute(text(HYPE_SQL), {
                "cid": chunk_id, "q": question, "body": chunk, "e": str(emb),
            })
            stats["questions"] += 1

    session.commit()
    return stats


body = open("nihol_corpus.txt", encoding="utf-8").read()
print(index_hype(session, "Nihol korpusi", body))

HYPE_SEARCH = (
    "SELECT chunk_id, question, chunk_text, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM hype_index "
    "ORDER BY embedding <=> CAST(:e AS vector) "
    "LIMIT :k"
)

def hype_retrieve(session, question: str, top_k: int = 5) -> list[dict]:
    emb, err = generate_query_embedding(question)
    if err or emb is None:
        return []

    rows = session.execute(text(HYPE_SEARCH),
                           {"e": str(emb), "k": top_k * 4}).mappings().all()

    seen, out = set(), []
    for r in rows:
        if r["chunk_id"] in seen:
            continue
        seen.add(r["chunk_id"])
        out.append(dict(r))
        if len(out) == top_k:
            break
    return out


for r in hype_retrieve(session, "To'lov shlyuzi qachon ulangan?", top_k=3):
    print(f"{r['score']:.3f}  mos savol: {r['question'][:55]}")

import numpy as np

def measure_redundancy(chunks: list[dict]) -> float:
    vectors = []
    for c in chunks:
        emb, err = generate_query_embedding(c["content"][:500])
        if not err and emb:
            vectors.append(emb)

    pairs = []
    for i in range(len(vectors)):
        for j in range(i + 1, len(vectors)):
            pairs.append(cosine_similarity(vectors[i], vectors[j]))
    return sum(pairs) / len(pairs) if pairs else 0.0


emb, _ = generate_query_embedding("Ucell loyihasi haqida")
plain = retrieve_chunks(session, emb, top_k=5)
print(f"Oddiy top-5 o'zaro o'xshashlik: {measure_redundancy(plain):.3f}")

def dartboard_select(question: str, candidates: list[dict],
                     top_k: int = 5, lam: float = 0.4) -> list[dict]:
    q_emb, err = generate_query_embedding(question)
    if err or q_emb is None:
        return candidates[:top_k]

    vectors = []
    for c in candidates:
        emb, e = generate_query_embedding(c["content"][:500])
        vectors.append(emb if not e and emb else None)

    relevance = [
        cosine_similarity(q_emb, v) if v is not None else 0.0
        for v in vectors
    ]

    selected, chosen_idx = [], []
    remaining = set(range(len(candidates)))

    while remaining and len(selected) < top_k:
        best_idx, best_score = None, -1e9

        for i in remaining:
            if vectors[i] is None:
                continue
            penalty = 0.0
            if chosen_idx:
                penalty = max(
                    cosine_similarity(vectors[i], vectors[j]) for j in chosen_idx
                )
            score = relevance[i] - lam * penalty
            if score > best_score:
                best_idx, best_score = i, score

        if best_idx is None:
            break

        row = dict(candidates[best_idx])
        row["dartboard_score"] = round(best_score, 4)
        row["relevance"] = round(relevance[best_idx], 4)
        selected.append(row)
        chosen_idx.append(best_idx)
        remaining.discard(best_idx)

    return selected

candidates = retrieve_chunks(session, emb, top_k=20)
picked = dartboard_select("Ucell loyihasi haqida", candidates, top_k=5, lam=0.4)

print(f"{'rel':<8}{'ball':<9}{'sarlavha'}")
for c in picked:
    print(f"{c['relevance']:<8.3f}{c['dartboard_score']:<9.3f}{c['title'][:45]}")

print(f"\nDartboard o'zaro o'xshashlik: {measure_redundancy(picked):.3f}")

for lam in (0.0, 0.2, 0.4, 0.6, 0.8):
    result = dartboard_select("Ucell loyihasi haqida", candidates, 5, lam)
    avg_rel = sum(c["relevance"] for c in result) / len(result)
    print(f"lambda={lam}  o'rtacha relevantlik={avg_rel:.3f}  "
          f"ortiqchalik={measure_redundancy(result):.3f}")

DOC_SUMMARY = (
    "Quyidagi hujjatning mavzusini 3-4 gapda yoz. "
    "Asosiy mavzular, mijoz nomi va yo'nalishni ko'rsat. "
    "Tafsilotlarga kirma.\n\n{body}"
)

SUMMARY_SQL = (
    "INSERT INTO doc_summaries (doc_id, title, summary, embedding) "
    "VALUES (:d, :t, :s, :e)"
)

def index_summaries(session, documents: list[dict]) -> int:
    saved = 0
    for doc in documents:
        summary = _ollama_chat(
            [{"role": "user", "content": DOC_SUMMARY.format(body=doc["body"][:4000])}],
            temperature=0.3, max_tokens=200,
        )
        summary = (summary or "").strip()
        if len(summary) < 50:
            continue
        emb, err = generate_query_embedding(summary)
        if err or emb is None:
            continue
        session.execute(text(SUMMARY_SQL), {
            "d": doc["id"], "t": doc["title"], "s": summary, "e": str(emb),
        })
        saved += 1
    session.commit()
    return saved

SUMMARY_SEARCH = (
    "SELECT doc_id, title, 1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM doc_summaries "
    "ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)

SCOPED_SEARCH = (
    "SELECT id, doc_id, title, content, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM chunks WHERE doc_id = ANY(:docs) "
    "ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)

def hierarchical_retrieve(session, question: str,
                          n_docs: int = 3, top_k: int = 5) -> list[dict]:
    emb, err = generate_query_embedding(question)
    if err or emb is None:
        return []

    docs = session.execute(text(SUMMARY_SEARCH),
                           {"e": str(emb), "k": n_docs}).mappings().all()
    if not docs:
        return retrieve_chunks(session, emb, top_k)

    doc_ids = [d["doc_id"] for d in docs]
    print("Tanlangan hujjatlar:")
    for d in docs:
        print(f"  {d['score']:.3f}  {d['title'][:50]}")

    rows = session.execute(text(SCOPED_SEARCH),
                           {"e": str(emb), "docs": doc_ids, "k": top_k}).mappings().all()
    return [dict(r) for r in rows]


hits = hierarchical_retrieve(session, "billing tizimidagi to'lov integratsiyasi")
print()
for c in hits:
    print(f"  {c['score']:.3f}  {c['content'][:60]}...")

emb, _ = generate_query_embedding("billing tizimidagi to'lov integratsiyasi")
flat = retrieve_chunks(session, emb, top_k=5)

flat_docs = {c.get("doc_id") for c in flat}
hier_docs = {c.get("doc_id") for c in hits}

print(f"Bir bosqichli : {len(flat_docs)} xil hujjatdan")
print(f"Ikki bosqichli: {len(hier_docs)} xil hujjatdan")

DOC_SCORES = (
    "SELECT id, chunk_index, content, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM chunks WHERE doc_id = :d "
    "ORDER BY chunk_index"
)

def score_document(session, question: str, doc_id: int) -> list[dict]:
    emb, err = generate_query_embedding(question)
    if err or emb is None:
        return []
    rows = session.execute(text(DOC_SCORES),
                           {"e": str(emb), "d": doc_id}).mappings().all()
    return [dict(r) for r in rows]


scored = score_document(session, "to'lov shlyuzi integratsiyasi", doc_id=12)
for c in scored[:8]:
    bar = "#" * int(c["score"] * 40)
    print(f"{c['chunk_index']:>3}  {c['score']:.3f}  {bar}")

def find_segments(scored: list[dict], threshold: float = 0.4,
                  max_len: int = 5, max_segments: int = 2) -> list[dict]:
    values = [c["score"] - threshold for c in scored]
    segments = []

    for start in range(len(values)):
        total = 0.0
        for end in range(start, min(start + max_len, len(values))):
            total += values[end]
            if total > 0:
                segments.append({"start": start, "end": end, "value": total})

    segments.sort(key=lambda s: s["value"], reverse=True)

    picked, used = [], set()
    for seg in segments:
        span = set(range(seg["start"], seg["end"] + 1))
        if span & used:
            continue
        used |= span
        picked.append(seg)
        if len(picked) == max_segments:
            break
    return picked


for seg in find_segments(scored):
    print(f"chunk {seg['start']}-{seg['end']}  qiymat={seg['value']:.3f}")

def rse_retrieve(session, question: str, doc_ids: list[int],
                 threshold: float = 0.4) -> list[dict]:
    results = []
    for doc_id in doc_ids:
        scored = score_document(session, question, doc_id)
        if not scored:
            continue
        for seg in find_segments(scored, threshold=threshold):
            pieces = scored[seg["start"]: seg["end"] + 1]
            results.append({
                "doc_id": doc_id,
                "range": f"{seg['start']}-{seg['end']}",
                "chunks": len(pieces),
                "value": round(seg["value"], 3),
                "content": " ".join(p["content"] for p in pieces),
            })
    results.sort(key=lambda r: r["value"], reverse=True)
    return results


for seg in rse_retrieve(session, "to'lov shlyuzi integratsiyasi", [12, 18, 24]):
    print(f"hujjat {seg['doc_id']}  chunk {seg['range']}  "
          f"({seg['chunks']} ta, {len(seg['content'])} belgi)")

COMPRESS = (
    "Quyidagi matndan FAQAT savolga tegishli gaplarni ko'chir.\n"
    "Qoidalar:\n"
    "- Gaplarni o'zgartirmasdan, aynan ko'chir\n"
    "- Hech narsa qo'shma, xulosa yozma\n"
    "- Tegishli gap bo'lmasa — BO'SH qaytar\n\n"
    "Savol: {question}\n\nMatn: {chunk}"
)

def compress_chunk(question: str, chunk: str, min_len: int = 40) -> str:
    out = _ollama_chat(
        [{"role": "user", "content": COMPRESS.format(question=question, chunk=chunk)}],
        temperature=0.0, max_tokens=400,
    )
    compressed = (out or "").strip()
    if len(compressed) < min_len:
        return ""
    if len(compressed) > len(chunk):
        return chunk
    return compressed

def compressed_retrieve(session, question: str, top_k: int = 5) -> dict:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=top_k)

    original_len = sum(len(c["content"]) for c in chunks)
    kept = []

    for c in chunks:
        compressed = compress_chunk(question, c["content"])
        if not compressed:
            continue
        row = dict(c)
        row["original"] = c["content"]
        row["content"] = compressed
        kept.append(row)

    new_len = sum(len(c["content"]) for c in kept)
    return {
        "chunks": kept,
        "before": original_len,
        "after": new_len,
        "dropped": len(chunks) - len(kept),
        "ratio": new_len / original_len if original_len else 0,
    }


r = compressed_retrieve(session, "To'lov shlyuzi qachon ulangan?")
print(f"Oldin    : {r['before']} belgi ({len(r['chunks']) + r['dropped']} chunk)")
print(f"Keyin    : {r['after']} belgi ({len(r['chunks'])} chunk)")
print(f"Tashlandi: {r['dropped']} ta chunk")
print(f"Siqilish : {r['ratio']:.0%}")

for c in r["chunks"]:
    print(f"--- {c['title'][:45]}")
    print(f"    asl      : {len(c['original'])} belgi")
    print(f"    siqilgan : {c['content'][:110]}")

def smart_context(session, question: str, top_k: int = 5,
                  limit: int = 3000) -> str:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=top_k)
    total = sum(len(c["content"]) for c in chunks)

    if total <= limit:
        print(f"Siqish kerak emas ({total} belgi)")
        return "\n\n".join(c["content"] for c in chunks)

    print(f"Siqilmoqda ({total} belgi > {limit})")
    result = compressed_retrieve(session, question, top_k)
    return "\n\n".join(c["content"] for c in result["chunks"])