RAG Turlari

from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from app.config import get_settings
from app.services.chat import _ollama_chat, generate_query_embedding

settings = get_settings()
engine = create_engine(settings.database_url)
Session = sessionmaker(bind=engine)
session = Session()

def chunk_text(body: str, size: int = 800, overlap: int = 100) -> list[str]:
    body = " ".join(body.split())
    chunks, start = [], 0
    while start < len(body):
        end = start + size
        piece = body[start:end]
        if end < len(body):
            dot = piece.rfind(". ")
            if dot > size * 0.5:
                piece = piece[:dot + 1]
                end = start + dot + 1
        piece = piece.strip()
        if len(piece) > 50:
            chunks.append(piece)
        start = max(end - overlap, start + 1)
    return chunks

INSERT_SQL = (
    "INSERT INTO chunks (title, content, chunk_index, embedding) "
    "VALUES (:t, :c, :i, :e)"
)

def index_document(session, doc_title: str, body: str) -> int:
    saved = 0
    for i, chunk in enumerate(chunk_text(body)):
        embedding, err = generate_query_embedding(chunk)
        if err or embedding is None:
            continue
        session.execute(text(INSERT_SQL),
                        {"t": doc_title, "c": chunk, "i": i, "e": str(embedding)})
        saved += 1
    session.commit()
    return saved


body = open("ucell.txt", encoding="utf-8").read()
print(index_document(session, "Ucell billing loyihasi", body))

SEARCH_SQL = (
    "SELECT title, content, 1 - (embedding <=> CAST(:e AS vector)) AS score "
    "FROM chunks "
    "ORDER BY embedding <=> CAST(:e AS vector) "
    "LIMIT :k"
)

def retrieve_chunks(session, embedding: list[float], top_k: int = 5) -> list[dict]:
    rows = session.execute(
        text(SEARCH_SQL), {"e": str(embedding), "k": top_k}
    ).mappings().all()
    return [dict(r) for r in rows]

SYSTEM = (
    "Sen yordamchi assistentsan. Javobni FAQAT berilgan kontekstdan yoz. "
    "Kontekstda javob bo'lmasa — 'Ma'lumot topilmadi' deb yoz. "
    "Hech narsa o'ylab topma."
)

def ask(session, question: str, top_k: int = 5) -> str:
    embedding, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, embedding, top_k)
    context = "\n\n".join(f"[{c['title']}]\n{c['content']}" for c in chunks)
    return _ollama_chat([
        {"role": "system", "content": SYSTEM},
        {"role": "user", "content": f"Kontekst:\n{context}\n\nSavol: {question}"},
    ], temperature=0.2)


print(ask(session, "Ucell uchun qanday loyiha qilingan?"))

import json
import networkx as nx

EXTRACT = (
    "Quyidagi matndan obyektlar va ular orasidagi munosabatlarni ajrat.\n"
    "Obyekt turlari: KOMPANIYA, LOYIHA, TEXNOLOGIYA, SHAXS, SANA\n"
    "Faqat JSON qaytar:\n"
    '{{"entities": [{{"name": "...", "type": "..."}}], '
    '"relations": [{{"from": "...", "to": "...", "type": "..."}}]}}\n\n'
    "Matn: {chunk}"
)

def extract_graph(chunk: str) -> dict:
    out = _ollama_chat(
        [{"role": "user", "content": EXTRACT.format(chunk=chunk[:1200])}],
        temperature=0.0, max_tokens=500,
    )
    try:
        start, end = out.index("{"), out.rindex("}") + 1
        return json.loads(out[start:end])
    except (ValueError, json.JSONDecodeError):
        return {"entities": [], "relations": []}


sample = "Nihol kompaniyasi Ucell uchun billing tizimini FastAPI va PostgreSQL da qurgan."
print(json.dumps(extract_graph(sample), ensure_ascii=False, indent=1))

def normalize(name: str) -> str:
    name = name.lower().strip()
    for suffix in (" kompaniyasi", " mchj", " ltd", " loyihasi", " tizimi"):
        if name.endswith(suffix):
            name = name[: -len(suffix)]
    return name.strip()


def build_graph(chunks: list[str]) -> nx.DiGraph:
    G = nx.DiGraph()
    for i, chunk in enumerate(chunks):
        data = extract_graph(chunk)

        for ent in data.get("entities", []):
            key = normalize(ent.get("name", ""))
            if not key:
                continue
            if G.has_node(key):
                G.nodes[key]["chunks"].append(i)
            else:
                G.add_node(key, type=ent.get("type", "?"),
                           label=ent["name"], chunks=[i])

        for rel in data.get("relations", []):
            a, b = normalize(rel.get("from", "")), normalize(rel.get("to", ""))
            if a and b and a != b:
                G.add_edge(a, b, type=rel.get("type", "BOGLIQ"))
    return G


graph = build_graph(chunks[:60])
print(f"Obyektlar : {graph.number_of_nodes()}")
print(f"Bog'lanish: {graph.number_of_edges()}")

def graph_retrieve(session, question: str, graph: nx.DiGraph,
                   hops: int = 2, top_k: int = 8) -> list[dict]:
    found = extract_graph(question)
    seeds = [normalize(e["name"]) for e in found.get("entities", [])]
    seeds = [s for s in seeds if graph.has_node(s)]
    if not seeds:
        emb, _ = generate_query_embedding(question)
        return retrieve_chunks(session, emb, top_k)

    undirected = graph.to_undirected()
    reached = set(seeds)
    for seed in seeds:
        reached |= set(nx.single_source_shortest_path_length(
            undirected, seed, cutoff=hops).keys())

    chunk_ids = set()
    for node in reached:
        chunk_ids.update(graph.nodes[node].get("chunks", []))

    print(f"Boshlang'ich obyektlar: {seeds}")
    print(f"{hops} qadamda topildi : {len(reached)} obyekt, {len(chunk_ids)} chunk")

    rows = session.execute(text(
        "SELECT id, title, content FROM chunks WHERE chunk_index = ANY(:ids) LIMIT :k"
    ), {"ids": list(chunk_ids), "k": top_k}).mappings().all()
    return [dict(r) for r in rows]


hits = graph_retrieve(session, "Ucell loyihasida qaysi texnologiyalar ishlatilgan?", graph)

import subprocess, os, yaml

WORKDIR = "graphrag_nihol"

def init_graphrag(corpus_dir: str) -> None:
    os.makedirs(f"{WORKDIR}/input", exist_ok=True)
    for name in os.listdir(corpus_dir):
        if name.endswith(".txt"):
            src = os.path.join(corpus_dir, name)
            dst = f"{WORKDIR}/input/{name}"
            with open(src, encoding="utf-8") as fh_in, \
                 open(dst, "w", encoding="utf-8") as fh_out:
                fh_out.write(fh_in.read())

    subprocess.run(["graphrag", "init", "--root", WORKDIR], check=True)


init_graphrag("nihol_corpus")

def configure_local(model: str = "qwen2.5:7b") -> None:
    path = f"{WORKDIR}/settings.yaml"
    with open(path, encoding="utf-8") as fh:
        cfg = yaml.safe_load(fh)

    cfg["llm"] = {
        "type": "openai_chat",
        "model": model,
        "api_base": "http://localhost:11434/v1",
        "api_key": "ollama",
        "max_tokens": 2000,
        "concurrent_requests": 2,
    }
    cfg["embeddings"]["llm"] = {
        "type": "openai_embedding",
        "model": "nomic-embed-text",
        "api_base": "http://localhost:11434/v1",
        "api_key": "ollama",
    }

    with open(path, "w", encoding="utf-8") as fh:
        yaml.safe_dump(cfg, fh, allow_unicode=True)


configure_local()

def build_index() -> None:
    result = subprocess.run(
        ["graphrag", "index", "--root", WORKDIR],
        capture_output=True, text=True,
    )
    print(result.stdout[-1500:])
    if result.returncode != 0:
        print("XATO:", result.stderr[-800:])


build_index()

def ask_graphrag(question: str, method: str = "local") -> str:
    result = subprocess.run(
        ["graphrag", "query", "--root", WORKDIR,
         "--method", method, "--query", question],
        capture_output=True, text=True,
    )
    out = result.stdout
    marker = "SUCCESS: "
    return out[out.index(marker) + len(marker):].strip() if marker in out else out


print("--- LOCAL ---")
print(ask_graphrag("Ucell billing loyihasida nima qilingan?", "local")[:250])
print("\n--- GLOBAL ---")
print(ask_graphrag("Kompaniyaning asosiy yo'nalishlari qanday?", "global")[:250])

def tool_search_docs(question: str, top_k: int = 5) -> str:
    emb, _ = generate_query_embedding(question)
    chunks = retrieve_chunks(session, emb, top_k)
    if not chunks:
        return "Hech narsa topilmadi."
    return "\n\n".join(f"[{c['title']}] {c['content'][:400]}" for c in chunks)


def tool_count_projects(sector: str = "") -> str:
    sql = "SELECT COUNT(DISTINCT title) FROM chunks"
    params = {}
    if sector:
        sql += " WHERE sector = :s"
        params["s"] = sector
    n = session.execute(text(sql), params).scalar_one()
    return f"Loyihalar soni: {n}"


def tool_list_sectors() -> str:
    rows = session.execute(text(
        "SELECT DISTINCT sector FROM chunks WHERE sector IS NOT NULL"
    )).scalars().all()
    return "Sohalar: " + ", ".join(rows)


TOOLS = {
    "search_docs": {
        "fn": tool_search_docs,
        "desc": "Hujjatlardan matn qidiradi. Parametr: question (savol matni).",
    },
    "count_projects": {
        "fn": tool_count_projects,
        "desc": "Loyihalar sonini sanaydi. Parametr: sector (ixtiyoriy soha nomi).",
    },
    "list_sectors": {
        "fn": tool_list_sectors,
        "desc": "Mavjud sohalar ro'yxatini qaytaradi. Parametr yo'q.",
    },
}

AGENT_PROMPT = (
    "Sen ma'lumot izlovchi agentsan. Quyidagi vositalar mavjud:\n{tools}\n\n"
    "Har qadamda FAQAT JSON qaytar:\n"
    '  Vosita kerak bo\'lsa: {{"tool": "nom", "args": {{...}}}}\n'
    '  Javob tayyor bo\'lsa: {{"answer": "javob matni"}}\n\n'
    "Javobni faqat vositalardan olingan ma'lumotga asosla.\n\n"
    "Savol: {question}\n\nOldingi qadamlar:\n{history}"
)

def run_agent(question: str, max_steps: int = 5) -> dict:
    tools_text = "\n".join(f"- {name}: {t['desc']}" for name, t in TOOLS.items())
    history, trail = [], []

    for step in range(1, max_steps + 1):
        prompt = AGENT_PROMPT.format(
            tools=tools_text,
            question=question,
            history="\n".join(history) or "(hali yo'q)",
        )
        raw = _ollama_chat([{"role": "user", "content": prompt}],
                           temperature=0.0, max_tokens=400)
        try:
            start, end = raw.index("{"), raw.rindex("}") + 1
            decision = json.loads(raw[start:end])
        except (ValueError, json.JSONDecodeError):
            break

        if "answer" in decision:
            return {"answer": decision["answer"], "steps": step, "trail": trail}

        name = decision.get("tool")
        if name not in TOOLS:
            break

        args = decision.get("args") or {}
        try:
            result = TOOLS[name]["fn"](**args)
        except TypeError as exc:
            result = f"Xato: {exc}"

        trail.append(f"{name}({args})")
        history.append(f"Vosita {name} natijasi: {result[:600]}")

    return {"answer": "Javob topilmadi.", "steps": max_steps, "trail": trail}


r = run_agent("Telekom sohasida nechta loyiha bor va ulardan biri haqida gapiring")
for t in r["trail"]:
    print(" →", t)
print(f"\n({r['steps']} qadam)\n{r['answer'][:250]}")