Advanced RAG Patterns

GRADE = (
    "Quyidagi hujjat bo'lagi savolga javob berish uchun foydalimi? "
    "0 dan 1 gacha faqat bitta son yoz (masalan 0.8). Boshqa hech narsa yozma.\n\n"
    "Savol: {question}\n\nBo'lak: {chunk}"
)

def grade_chunk(question: str, chunk: str) -> float:
    out = _ollama_chat(
        [{"role": "user", "content": GRADE.format(question=question, chunk=chunk[:900])}],
        temperature=0.0, max_tokens=8,
    )
    try:
        return max(0.0, min(1.0, float((out or "0").strip().split()[0])))
    except (ValueError, IndexError):
        return 0.0


def grade_context(question: str, chunks: list[dict]) -> tuple[float, list[dict]]:
    graded = []
    for c in chunks:
        row = dict(c)
        row["grade"] = grade_chunk(question, c["content"])
        graded.append(row)
    best = max((g["grade"] for g in graded), default=0.0)
    return best, sorted(graded, key=lambda x: x["grade"], reverse=True)

REFINE = (
    "Quyidagi matndan FAQAT savolga tegishli gaplarni ko'chir. "
    "Hech narsa qo'shma, o'zgartirma. Tegishli gap bo'lmasa bo'sh qaytar.\n\n"
    "Savol: {question}\n\nMatn: {chunk}"
)

def refine_knowledge(question: str, chunks: list[dict]) -> str:
    kept = []
    for c in chunks:
        out = _ollama_chat(
            [{"role": "user", "content": REFINE.format(question=question,
                                                        chunk=c["content"])}],
            temperature=0.0, max_tokens=300)
        cleaned = (out or "").strip()
        if len(cleaned) > 40:
            kept.append(cleaned)
    return "\n\n".join(kept)

def crag_answer(session, question: str, max_rounds: int = 2) -> dict:
    current = question
    trail = []

    for round_no in range(1, max_rounds + 1):
        emb, _ = generate_query_embedding(current)
        chunks = retrieve_chunks(session, emb, top_k=5)
        best, graded = grade_context(current, chunks)
        trail.append({"round": round_no, "query": current, "best_grade": round(best, 2)})

        if best > 0.7:
            good = [g for g in graded if g["grade"] > 0.5]
            context = refine_knowledge(current, good)
            answer = _ollama_chat([
                {"role": "system", "content": SYSTEM},
                {"role": "user", "content": f"Kontekst:\n{context}\n\nSavol: {question}"},
            ], temperature=0.2)
            return {"answer": answer, "action": "CORRECT", "trail": trail}

        if best < 0.3:
            break

        current = step_back_query(current)

    return {
        "answer": "Bazada bu savolga yetarli ma'lumot topilmadi.",
        "action": "INCORRECT",
        "trail": trail,
    }


result = crag_answer(session, "Ucell uchun qanday to'lov shlyuzi ishlatilgan?")
for step in result["trail"]:
    print(f"{step['round']}-urinish  ball={step['best_grade']}  savol: {step['query'][:55]}")
print(f"\nQaror: {result['action']}\n{result['answer'][:200]}")

def needs_retrieval(question: str) -> bool:
    prompt = (
        "Bu savolga javob berish uchun tashqi ma'lumot bazasi kerakmi? "
        "Salomlashish, matematika, umumiy bilim uchun KERAK EMAS. "
        "Faqat HA yoki YOQ deb yoz.\n\nSavol: " + question
    )
    out = _ollama_chat([{"role": "user", "content": prompt}],
                       temperature=0.0, max_tokens=5)
    return "HA" in (out or "").upper()


for q in ["Salom, qalaysiz?", "Ucell loyihasi haqida ayting", "15 ni 3 ga bo'ling"]:
    print(f"{needs_retrieval(q)!s:<6} {q}")

def is_relevant(question: str, chunk: str) -> bool:
    prompt = (
        "Bu hujjat bo'lagi savolga javob berishga yordam beradimi? "
        "Faqat HA yoki YOQ.\n\n"
        f"Savol: {question}\n\nBo'lak: {chunk[:800]}"
    )
    out = _ollama_chat([{"role": "user", "content": prompt}],
                       temperature=0.0, max_tokens=5)
    return "HA" in (out or "").upper()


def is_supported(answer: str, context: str) -> bool:
    prompt = (
        "Bu javobdagi barcha faktlar kontekstda bormi? "
        "Bitta fakt ham kontekstdan tashqarida bo'lsa YOQ deb yoz. "
        "Faqat HA yoki YOQ.\n\n"
        f"Kontekst:\n{context}\n\nJavob: {answer}"
    )
    out = _ollama_chat([{"role": "user", "content": prompt}],
                       temperature=0.0, max_tokens=5)
    return "HA" in (out or "").upper()

def self_rag(session, question: str, top_k: int = 5) -> dict:
    if not needs_retrieval(question):
        answer = _ollama_chat([{"role": "user", "content": question}], temperature=0.5)
        return {"answer": answer, "path": "retrieval-siz", "supported": None}

    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=top_k)
    relevant = [c for c in chunks if is_relevant(question, c["content"])]

    if not relevant:
        return {"answer": "Bazada tegishli ma'lumot topilmadi.",
                "path": "tegishli chunk yo'q", "supported": None}

    context = "\n\n".join(c["content"] for c in relevant)
    answer = _ollama_chat([
        {"role": "system", "content": SYSTEM},
        {"role": "user", "content": f"Kontekst:\n{context}\n\nSavol: {question}"},
    ], temperature=0.2)

    supported = is_supported(answer, context)
    if not supported:
        return {"answer": "Ishonchli javob shakllantira olmadim.",
                "path": f"{len(relevant)}/{len(chunks)} chunk, tasdiqlanmadi",
                "supported": False}

    return {"answer": answer,
            "path": f"{len(relevant)}/{len(chunks)} chunk tegishli",
            "supported": True}


for q in ["Salom!", "Ucell uchun nima qilingan?", "Nihol Marsda ofis ochganmi?"]:
    r = self_rag(session, q)
    print(f"[{r['path']}]  {r['answer'][:70]}")

CLASSIFY = (
    "Savolni bitta toifaga ajrat. Faqat toifa nomini yoz:\n"
    "FACTUAL — aniq bitta fakt so'ralgan (sana, raqam, nom)\n"
    "ANALYTICAL — tahlil, taqqoslash, umumlashtirish kerak\n"
    "OPINION — fikr, baho, tavsiya so'ralgan\n"
    "CONTEXTUAL — oldingi suhbatga bog'liq\n\n"
    "Savol: {question}"
)

def classify_query(question: str) -> str:
    out = _ollama_chat(
        [{"role": "user", "content": CLASSIFY.format(question=question)}],
        temperature=0.0, max_tokens=10,
    )
    text_out = (out or "").upper()
    for label in ("FACTUAL", "ANALYTICAL", "OPINION", "CONTEXTUAL"):
        if label in text_out:
            return label
    return "FACTUAL"


for q in ["Kompaniya qachon tashkil topgan?",
          "Bank va telekom loyihalarini solishtiring",
          "Qaysi yondashuvni tavsiya qilasiz?"]:
    print(f"{classify_query(q):<12} {q}")

def retrieve_factual(session, question: str) -> list[dict]:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=15)
    chunks = [c for c in chunks if c["score"] >= 0.45]
    return search_and_rerank(session, question, candidates=15, top_n=3) if chunks else []


def retrieve_analytical(session, question: str) -> list[dict]:
    seen, out = set(), []
    for sub in decompose_query(question):
        emb, err = generate_query_embedding(sub)
        if err or emb is None:
            continue
        for c in retrieve_chunks(session, emb, top_k=5):
            if c["id"] not in seen:
                seen.add(c["id"])
                out.append(c)
    return out[:12]


def retrieve_opinion(session, question: str) -> list[dict]:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=25)
    return filter_by_diversity(chunks, max_similarity=0.85, top_k=8)


def retrieve_contextual(session, question: str, history: list[dict]) -> list[dict]:
    rewritten = rewrite_query(question, history)
    emb, _ = generate_query_embedding(rewritten)
    return retrieve_chunks(session, emb, top_k=6)

STRATEGIES = {
    "FACTUAL":    retrieve_factual,
    "ANALYTICAL": retrieve_analytical,
    "OPINION":    retrieve_opinion,
}

def adaptive_retrieve(session, question: str, history: list[dict] | None = None) -> dict:
    category = classify_query(question)

    if category == "CONTEXTUAL":
        chunks = retrieve_contextual(session, question, history or [])
    else:
        chunks = STRATEGIES[category](session, question)

    return {"category": category, "chunks": chunks, "count": len(chunks)}


for q in ["Kompaniya qachon tashkil topgan?",
          "Bank va telekom loyihalarini solishtiring",
          "Qaysi texnologiyani tavsiya qilasiz?"]:
    r = adaptive_retrieve(session, q)
    print(f"{r['category']:<12} {r['count']:>2} chunk   {q[:45]}")

import numpy as np
from sklearn.mixture import GaussianMixture

def embed_all(texts: list[str]) -> np.ndarray:
    vectors = []
    for t in texts:
        emb, err = generate_query_embedding(t)
        vectors.append(emb if not err and emb else [0.0] * 768)
    return np.array(vectors)


def cluster_texts(vectors: np.ndarray, max_clusters: int = 10) -> np.ndarray:
    n = len(vectors)
    if n <= 2:
        return np.zeros(n, dtype=int)

    k = min(max_clusters, max(2, n // 5))
    gmm = GaussianMixture(n_components=k, random_state=42, covariance_type="diag")
    return gmm.fit_predict(vectors)

SUMMARIZE = (
    "Quyidagi matnlarning umumiy mazmunini 3-4 gapda yoz. "
    "Asosiy mavzular va faktlarni saqlab qol. "
    "Faqat xulosani yoz.\n\n{texts}"
)

def summarize_cluster(texts: list[str]) -> str:
    joined = "\n\n---\n\n".join(t[:900] for t in texts[:8])
    out = _ollama_chat(
        [{"role": "user", "content": SUMMARIZE.format(texts=joined)}],
        temperature=0.3, max_tokens=250,
    )
    return (out or "").strip()

def build_raptor_tree(chunks: list[str], max_levels: int = 3) -> dict[int, list[str]]:
    tree = {0: chunks}
    current = chunks

    for level in range(1, max_levels + 1):
        if len(current) <= 3:
            break

        vectors = embed_all(current)
        labels = cluster_texts(vectors)

        summaries = []
        for label in sorted(set(labels)):
            members = [current[i] for i in range(len(current)) if labels[i] == label]
            summary = summarize_cluster(members)
            if summary:
                summaries.append(summary)

        if not summaries or len(summaries) >= len(current):
            break

        tree[level] = summaries
        current = summaries
        print(f"{level}-daraja: {len(summaries)} ta xulosa")

    return tree


chunks = chunk_text(open("nihol_corpus.txt", encoding="utf-8").read(), size=800)
tree = build_raptor_tree(chunks)

TREE_SQL = (
    "INSERT INTO chunks (title, content, level, embedding) "
    "VALUES (:t, :c, :l, :e)"
)

def index_tree(session, tree: dict[int, list[str]]) -> int:
    total = 0
    for level, texts in tree.items():
        for t in texts:
            emb, err = generate_query_embedding(t)
            if err or emb is None:
                continue
            session.execute(text(TREE_SQL), {
                "t": f"level-{level}",
                "c": t,
                "l": level,
                "e": str(emb),
            })
            total += 1
    session.commit()
    return total


print(f"Indekslandi: {index_tree(session, tree)} ta yozuv")

for q in ["Ucell billing moduli qachon yangilandi?",
          "Kompaniyaning umumiy yo'nalishi nima?"]:
    emb, _ = generate_query_embedding(q)
    top = retrieve_chunks(session, emb, top_k=3)
    levels = [c.get("level") for c in top]
    print(f"\n{q}\n  topilgan darajalar: {levels}")

STEPBACK = (
    "Quyidagi aniq savoldan bir pog'ona umumiyroq savol yoz. "
    "Maqsad — javob uchun kerak bo'ladigan fon ma'lumotini topish.\n"
    "Misol:\n"
    "  Aniq: 'X modulida qaysi versiyada webhook qo'shilgan?'\n"
    "  Umumiy: 'X moduli qanday rivojlangan?'\n\n"
    "Faqat savolni yoz.\n\nAniq savol: {question}"
)

def step_back_query(question: str) -> str:
    out = _ollama_chat(
        [{"role": "user", "content": STEPBACK.format(question=question)}],
        temperature=0.2, max_tokens=60,
    )
    return (out or question).strip()

def step_back_answer(session, question: str) -> dict:
    broad_question = step_back_query(question)

    narrow_emb, _ = generate_query_embedding(question)
    broad_emb, _ = generate_query_embedding(broad_question)

    narrow = retrieve_chunks(session, narrow_emb, top_k=4)
    broad = retrieve_chunks(session, broad_emb, top_k=4)

    seen = {c["id"] for c in narrow}
    broad = [c for c in broad if c["id"] not in seen]

    context = (
        "UMUMIY KONTEKST:\n"
        + "\n\n".join(c["content"] for c in broad)
        + "\n\nANIQ KONTEKST:\n"
        + "\n\n".join(c["content"] for c in narrow)
    )

    answer = _ollama_chat([
        {"role": "system", "content": SYSTEM},
        {"role": "user", "content": f"{context}\n\nSavol: {question}"},
    ], temperature=0.2)

    return {
        "answer": answer,
        "broad_question": broad_question,
        "chunks": len(narrow) + len(broad),
    }


r = step_back_answer(session, "Ucell billing modulida to'lov shlyuzi qachon ulangan?")
print("Keng savol :", r["broad_question"])
print("Chunk soni :", r["chunks"])
print("\nJavob:\n" + r["answer"][:300])

import fitz
import base64

def extract_images(pdf_path: str, min_size: int = 120) -> list[dict]:
    doc = fitz.open(pdf_path)
    images = []
    for page_no in range(len(doc)):
        for img in doc[page_no].get_images(full=True):
            pix = fitz.Pixmap(doc, img[0])
            if pix.width < min_size or pix.height < min_size:
                continue
            if pix.n - pix.alpha >= 4:
                pix = fitz.Pixmap(fitz.csRGB, pix)
            images.append({
                "page": page_no + 1,
                "bytes": pix.tobytes("png"),
                "size": f"{pix.width}x{pix.height}",
            })
    doc.close()
    return images


images = extract_images("nihol_arxitektura.pdf")
print(f"Topilgan rasmlar: {len(images)}")

CAPTION = (
    "Bu texnik rasmni batafsil tasvirla. Quyidagilarni albatta yoz:\n"
    "1. Rasm turi (diagramma, jadval, grafik, skrinshot)\n"
    "2. Rasmdagi BARCHA matn va raqamlar\n"
    "3. Elementlar orasidagi bog'lanishlar\n"
    "4. Asosiy xulosa\n"
    "O'zbek tilida yoz."
)

def caption_image(image_bytes: bytes) -> str:
    b64 = base64.b64encode(image_bytes).decode()
    out = _ollama_chat(
        [{"role": "user", "content": CAPTION, "images": [b64]}],
        model="llava:13b",
        temperature=0.2,
        max_tokens=400,
    )
    return (out or "").strip()


print(caption_image(images[3]["bytes"])[:400])

IMAGE_SQL = (
    "INSERT INTO chunks (title, content, source_type, page, embedding) "
    "VALUES (:t, :c, 'image', :p, :e)"
)

def index_images(session, pdf_path: str, doc_title: str) -> int:
    saved = 0
    for img in extract_images(pdf_path):
        caption = caption_image(img["bytes"])
        if len(caption) < 60:
            continue
        emb, err = generate_query_embedding(caption)
        if err or emb is None:
            continue
        session.execute(text(IMAGE_SQL), {
            "t": f"{doc_title} — {img['page']}-sahifa rasmi",
            "c": caption,
            "p": img["page"],
            "e": str(emb),
        })
        saved += 1
    session.commit()
    return saved


print(f"Indekslandi: {index_images(session, 'nihol_arxitektura.pdf', 'Arxitektura')} ta rasm")

emb, _ = generate_query_embedding("billing tizimi qanday komponentlardan iborat?")
for c in retrieve_chunks(session, emb, top_k=3):
    print(f"{c['score']:.3f}  {c['title'][:50]}")

EXPLAIN = (
    "Bu hujjat bo'lagi berilgan savolga qanday aloqador? "
    "Bir jumlada tushuntir. Aloqasi bo'lmasa — nega yo'qligini ayt.\n\n"
    "Savol: {question}\n\nBo'lak: {chunk}"
)

def explain_chunk(question: str, chunk: str) -> str:
    out = _ollama_chat(
        [{"role": "user", "content": EXPLAIN.format(question=question,
                                                     chunk=chunk[:800])}],
        temperature=0.2, max_tokens=80,
    )
    return (out or "").strip()

def explainable_retrieve(session, question: str, top_k: int = 5) -> list[dict]:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k=top_k)

    out = []
    for c in chunks:
        row = dict(c)
        row["explanation"] = explain_chunk(question, c["content"])
        out.append(row)
    return out


for i, c in enumerate(explainable_retrieve(session, "Ucell loyihasi haqida"), 1):
    print(f"{i}. [{c['score']:.3f}] {c['title'][:44]}")
    print(f"   -> {c['explanation'][:110]}\n")

NEGATIVE = ("bog'liq emas", "aloqasi yo'q", "aloqador emas",
            "ma'lumot yo'q", "tegishli emas", "emas,")

def filter_by_explanation(chunks: list[dict]) -> list[dict]:
    kept = []
    for c in chunks:
        low = c.get("explanation", "").lower()
        if any(marker in low for marker in NEGATIVE):
            continue
        kept.append(c)
    return kept


explained = explainable_retrieve(session, "Ucell loyihasi haqida")
kept = filter_by_explanation(explained)
print(f"{len(explained)} -> {len(kept)} chunk")
for c in kept:
    print(f"  {c['title'][:50]}")

import json

LIGHT_EXTRACT = (
    "Matndan obyektlar va munosabatlarni ajrat. JSON qaytar:\n"
    '{{"entities": [{{"name": "...", "type": "...", "description": "..."}}],\n'
    ' "relations": [{{"from": "...", "to": "...", "description": "..."}}]}}\n'
    "description — qisqa, bir jumla.\n\nMatn: {chunk}"
)

def extract_light(chunk: str) -> dict:
    out = _ollama_chat(
        [{"role": "user", "content": LIGHT_EXTRACT.format(chunk=chunk[:1200])}],
        temperature=0.0, max_tokens=600,
    )
    try:
        start, end = out.index("{"), out.rindex("}") + 1
        return json.loads(out[start:end])
    except (ValueError, json.JSONDecodeError):
        return {"entities": [], "relations": []}

ENT_SQL = (
    "INSERT INTO light_entities (name, type, description, chunk_id, embedding) "
    "VALUES (:n, :t, :d, :c, :e) "
    "ON CONFLICT (name) DO UPDATE SET description = EXCLUDED.description"
)
REL_SQL = (
    "INSERT INTO light_relations (src, dst, description, chunk_id, embedding) "
    "VALUES (:s, :d, :desc, :c, :e)"
)

def index_lightrag(session, chunks: list[str]) -> dict:
    stats = {"entities": 0, "relations": 0}

    for chunk_id, chunk in enumerate(chunks):
        data = extract_light(chunk)

        for ent in data.get("entities", []):
            name = normalize(ent.get("name", ""))
            desc = ent.get("description") or name
            if not name:
                continue
            emb, err = generate_query_embedding(f"{ent['name']}: {desc}")
            if err or emb is None:
                continue
            session.execute(text(ENT_SQL), {
                "n": name, "t": ent.get("type", "?"), "d": desc,
                "c": chunk_id, "e": str(emb)})
            stats["entities"] += 1

        for rel in data.get("relations", []):
            src, dst = normalize(rel.get("from", "")), normalize(rel.get("to", ""))
            desc = rel.get("description") or f"{src} - {dst}"
            if not src or not dst:
                continue
            emb, err = generate_query_embedding(desc)
            if err or emb is None:
                continue
            session.execute(text(REL_SQL), {
                "s": src, "d": dst, "desc": desc,
                "c": chunk_id, "e": str(emb)})
            stats["relations"] += 1

    session.commit()
    return stats


print(index_lightrag(session, chunks[:60]))

KEYWORDS = (
    "Savoldan ikki xil kalit so'z ajrat. JSON qaytar:\n"
    '{{"local": ["aniq obyekt nomlari"], "global": ["umumiy mavzular"]}}\n\n'
    "Savol: {question}"
)

def extract_keywords_two_level(question: str) -> dict:
    out = _ollama_chat(
        [{"role": "user", "content": KEYWORDS.format(question=question)}],
        temperature=0.0, max_tokens=150,
    )
    try:
        start, end = out.index("{"), out.rindex("}") + 1
        data = json.loads(out[start:end])
        return {"local": data.get("local", []), "global": data.get("global", [])}
    except (ValueError, json.JSONDecodeError):
        return {"local": [question], "global": [question]}


print(extract_keywords_two_level(
    "Ucell billing tizimida to'lov integratsiyasi qanday qilingan?"))

ENT_SEARCH = (
    "SELECT name, description, chunk_id, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM light_entities ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)
REL_SEARCH = (
    "SELECT src, dst, description, chunk_id, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM light_relations ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)

def lightrag_retrieve(session, question: str, top_k: int = 5) -> dict:
    keys = extract_keywords_two_level(question)
    entities, relations, chunk_ids = [], [], set()

    for kw in keys["local"]:
        emb, err = generate_query_embedding(kw)
        if err or emb is None:
            continue
        for r in session.execute(text(ENT_SEARCH),
                                 {"e": str(emb), "k": 3}).mappings().all():
            entities.append(dict(r))
            chunk_ids.add(r["chunk_id"])

    for kw in keys["global"]:
        emb, err = generate_query_embedding(kw)
        if err or emb is None:
            continue
        for r in session.execute(text(REL_SEARCH),
                                 {"e": str(emb), "k": 3}).mappings().all():
            relations.append(dict(r))
            chunk_ids.add(r["chunk_id"])

    return {"keywords": keys, "entities": entities[:top_k],
            "relations": relations[:top_k], "chunk_ids": sorted(chunk_ids)}


r = lightrag_retrieve(session, "Ucell billing tizimida to'lov integratsiyasi qanday qilingan?")
print("Obyektlar:")
for e in r["entities"][:3]:
    print(f"  {e['score']:.3f}  {e['name']}: {e['description'][:55]}")
print("Munosabatlar:")
for rel in r["relations"][:3]:
    print(f"  {rel['score']:.3f}  {rel['src']} -> {rel['dst']}: {rel['description'][:45]}")
print(f"\nJami {len(r['chunk_ids'])} ta chunk")

MEMORIZE = (
    "Quyidagi matnlardan asosiy mavzularni ajrat. "
    "Har birini shu formatda yoz:\n"
    "MAVZU: qisqa tavsif\n"
    "5-8 ta mavzu yoz, boshqa hech narsa yozma.\n\n{texts}"
)

def memorize(texts: list[str]) -> list[tuple[str, str]]:
    joined = "\n\n---\n\n".join(t[:700] for t in texts[:12])
    out = _ollama_chat(
        [{"role": "user", "content": MEMORIZE.format(texts=joined)}],
        temperature=0.3, max_tokens=400,
    )
    pairs = []
    for line in (out or "").split("\n"):
        if ":" not in line:
            continue
        key, _, value = line.partition(":")
        key, value = key.strip(" -•"), value.strip()
        if len(key) > 2 and len(value) > 10:
            pairs.append((key, value))
    return pairs

MEM_SQL = (
    "INSERT INTO memory_store (topic, description, embedding) "
    "VALUES (:t, :d, :e)"
)

def build_memory(session, chunks: list[str], batch: int = 12) -> int:
    saved = 0
    for i in range(0, len(chunks), batch):
        for topic, desc in memorize(chunks[i: i + batch]):
            emb, err = generate_query_embedding(f"{topic}: {desc}")
            if err or emb is None:
                continue
            session.execute(text(MEM_SQL),
                            {"t": topic, "d": desc, "e": str(emb)})
            saved += 1
    session.commit()
    return saved


print(f"Xotirada: {build_memory(session, chunks)} ta yozuv")

rows = session.execute(text(
    "SELECT topic, description FROM memory_store LIMIT 5")).all()
for topic, desc in rows:
    print(f"  {topic}: {desc[:60]}")

CLUES = (
    "Quyidagi xotira asosida savolga javob topish uchun "
    "3 ta aniq qidiruv so'rovi yoz. "
    "Xotiradagi aniq nomlar va atamalardan foydalan. "
    "Har birini yangi qatorda, raqamlamasdan.\n\n"
    "Xotira:\n{memory}\n\nSavol: {question}"
)

MEM_SEARCH = (
    "SELECT topic, description, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM memory_store ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)

def generate_clues(session, question: str, n_memory: int = 8) -> list[str]:
    emb, err = generate_query_embedding(question)
    if err or emb is None:
        return [question]

    rows = session.execute(text(MEM_SEARCH),
                           {"e": str(emb), "k": n_memory}).mappings().all()
    memory = "\n".join(f"{r['topic']}: {r['description']}" for r in rows)

    out = _ollama_chat(
        [{"role": "user", "content": CLUES.format(memory=memory, question=question)}],
        temperature=0.4, max_tokens=200,
    )
    clues = [c.strip(" -•0123456789.") for c in (out or "").split("\n")]
    return [c for c in clues if len(c) > 10][:3] or [question]


vague = "Bizda shunga o'xshash tajriba bormidi?"
for c in generate_clues(session, vague):
    print("-", c)

def memorag_retrieve(session, question: str, top_k: int = 8) -> list[dict]:
    clues = generate_clues(session, question)

    seen, out = set(), []
    for clue in clues:
        emb, err = generate_query_embedding(clue)
        if err or emb is None:
            continue
        for c in retrieve_chunks(session, emb, top_k=5):
            if c["id"] in seen:
                continue
            seen.add(c["id"])
            row = dict(c)
            row["found_by"] = clue
            out.append(row)

    out.sort(key=lambda c: c["score"], reverse=True)
    return out[:top_k]


for c in memorag_retrieve(session, "Bizda shunga o'xshash tajriba bormidi?")[:4]:
    print(f"{c['score']:.3f}  {c['title'][:40]}")
    print(f"        topildi: {c['found_by'][:50]}")

from dataclasses import dataclass, field
from sentence_transformers import SentenceTransformer
import hashlib, json

_model = SentenceTransformer("intfloat/multilingual-e5-base")

def local_embed(body: str) -> list[float]:
    return _model.encode(f"query: {body}", normalize_embeddings=True).tolist()


@dataclass
class Citation:
    doc_id: int
    chunk_index: int
    sentence: str


@dataclass
class Relationship:
    src: str
    dst: str
    kind: str
    citation: Citation
    rel_id: str = field(default="")

    def __post_init__(self):
        raw = f"{self.src}|{self.kind}|{self.dst}|{self.citation.sentence}"
        self.rel_id = hashlib.md5(raw.encode()).hexdigest()[:12]

LOCAL_EXTRACT = (
    "Matndan munosabatlarni ajrat. Har biri uchun uni tasdiqlaydigan "
    "AYNAN o'sha jumlani ham ko'chir. JSON qaytar:\n"
    '[{{"from": "...", "to": "...", "kind": "...", "sentence": "..."}}]\n'
    "sentence — matndan aynan ko'chirilgan jumla bo'lsin.\n\nMatn: {chunk}"
)

def extract_with_citation(chunk: str, doc_id: int,
                          chunk_index: int) -> list[Relationship]:
    out = _ollama_chat(
        [{"role": "user", "content": LOCAL_EXTRACT.format(chunk=chunk[:1200])}],
        temperature=0.0, max_tokens=700,
    )
    try:
        start, end = out.index("["), out.rindex("]") + 1
        items = json.loads(out[start:end])
    except (ValueError, json.JSONDecodeError):
        return []

    rels = []
    for item in items:
        sentence = (item.get("sentence") or "").strip()
        if not sentence or sentence not in chunk:
            continue
        rels.append(Relationship(
            src=normalize(item.get("from", "")),
            dst=normalize(item.get("to", "")),
            kind=item.get("kind", "BOGLIQ"),
            citation=Citation(doc_id, chunk_index, sentence),
        ))
    return rels

REL_STORE = (
    "INSERT INTO local_relations "
    "(rel_id, src, dst, kind, doc_id, chunk_index, sentence, embedding) "
    "VALUES (:id, :s, :d, :k, :doc, :ci, :sent, :e) "
    "ON CONFLICT (rel_id) DO NOTHING"
)

def index_local_graph(session, documents: list[dict]) -> dict:
    stats = {"relations": 0, "rejected": 0}

    for doc in documents:
        for chunk_index, chunk in enumerate(chunk_text(doc["body"], size=800)):
            rels = extract_with_citation(chunk, doc["id"], chunk_index)
            for rel in rels:
                if not rel.src or not rel.dst:
                    stats["rejected"] += 1
                    continue
                text_form = f"{rel.src} {rel.kind} {rel.dst}: {rel.citation.sentence}"
                session.execute(text(REL_STORE), {
                    "id": rel.rel_id, "s": rel.src, "d": rel.dst, "k": rel.kind,
                    "doc": rel.citation.doc_id, "ci": rel.citation.chunk_index,
                    "sent": rel.citation.sentence, "e": str(local_embed(text_form)),
                })
                stats["relations"] += 1

    session.commit()
    return stats


print(index_local_graph(session, documents))

LOCAL_SEARCH = (
    "SELECT src, dst, kind, doc_id, chunk_index, sentence, "
    "       1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM local_relations "
    "ORDER BY embedding <=> CAST(:e AS vector) LIMIT :k"
)

def attributed_answer(session, question: str, top_k: int = 6) -> dict:
    emb = local_embed(question)
    rows = session.execute(text(LOCAL_SEARCH),
                           {"e": str(emb), "k": top_k}).mappings().all()
    facts = [dict(r) for r in rows]
    if not facts:
        return {"answer": "Ma'lumot topilmadi.", "citations": []}

    context = "\n".join(
        f"[{i + 1}] {f['sentence']}" for i, f in enumerate(facts)
    )
    answer = _ollama_chat([
        {"role": "system", "content":
         "Javobni faqat berilgan faktlardan yoz. Har bir da'vodan keyin "
         "manba raqamini kvadrat qavsda ko'rsat, masalan [1]."},
        {"role": "user", "content": f"Faktlar:\n{context}\n\nSavol: {question}"},
    ], temperature=0.1)

    return {"answer": answer, "citations": facts}


r = attributed_answer(session, "Ucell billing tizimi qaysi texnologiyalarda qurilgan?")
print(r["answer"])
print("\nManbalar:")
for i, c in enumerate(r["citations"][:3], 1):
    print(f"  [{i}] hujjat {c['doc_id']}, chunk {c['chunk_index']}")
    print(f"      \"{c['sentence'][:75]}\"")