Loyihalar — 7 ta amaliy ish, kod bilan

ocr_book/projects/
  p1_cli/        ocr_cli.py  evaluate.py
  p2_receipt/    receipt.py
  p3_bot/        ocr_service.py  bot.py
  p4_table/      table2xlsx.py
  p5_benchmark/  benchmark.py  translit.py
  p6_archive/    archive.py
  p7_capstone/   db.py  ingest.py  search.py  app.py
Har fayl boshida:  sys.path.insert(0, ...parents[2])  ->  ocrlib.py ni topadi

# ocr_cli.py — 1-loyiha: rasm yoki PDF -> matn + confidence. Buyruq qatori vositasi.
#
#   python ocr_cli.py ../../samples/05_telefon_soya.jpg --lang uzb --preprocess adaptive --boxes
#   python ocr_cli.py ../../samples/09_aralash.pdf --lang rus+uzb
#
# Chiqish: out/<fayl>.txt (matn), out/<fayl>.json (sahifa, method, confidence, gate),
#          --boxes bo'lsa out/<fayl>_boxes.png (so'z ramkalari, qizil = conf <= 60)
import argparse
import json
import sys
import time
from pathlib import Path

import cv2

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))          # ocrlib.py shu yerda
from ocrlib import clean_ocr_text, ocr_data, ocr_with_confidence, pdf_page_count, pdf_page_text, preprocess, render_pdf_page, to_gray  # noqa: E402

MIN_CHARS = 100          # text layer bor deb hisoblash chegarasi (7-bob)


def gate(method, conf):
    """8-bob quality gate."""
    if method == "text_layer":
        return "accept"
    if conf >= 80:
        return "accept"
    if conf >= 60:
        return "accept_with_flag"
    return "manual_review"


def prepare(img, args):
    if args.preprocess == "none":
        return to_gray(img)
    return preprocess(img, args.preprocess, do_deskew=not args.no_deskew)


def draw_boxes(gray, lang, psm, out_path):
    canvas = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
    d = ocr_data(gray, lang=lang, psm=psm)
    for x, y, w, h, c, t in zip(d["left"], d["top"], d["width"], d["height"], d["conf"], d["text"]):
        c = int(float(c))
        if c <= 0 or not t.strip():
            continue
        cv2.rectangle(canvas, (x, y), (x + w, y + h), (0, 180, 0) if c > 60 else (0, 0, 255), 3)
    cv2.imwrite(str(out_path), canvas)


def ocr_page(img, args):
    prep = prepare(img, args)
    text, conf = ocr_with_confidence(prep, lang=args.lang, psm=args.psm)
    return (clean_ocr_text(text) if not args.raw else text), conf, prep


def process_file(path, args):
    pages = []
    if path.suffix.lower() == ".pdf":
        for i in range(1, pdf_page_count(path) + 1):
            layer = pdf_page_text(path, i).strip()
            if len(layer) >= MIN_CHARS:
                pages.append({"page": i, "text": layer, "method": "text_layer", "confidence": 100.0})
                continue
            img = render_pdf_page(path, i, dpi=args.dpi)
            text, conf, _ = ocr_page(img, args)
            pages.append({"page": i, "text": text, "method": "ocr_tesseract", "confidence": round(conf, 1)})
    else:
        text, conf, prep = ocr_page(path, args)
        pages.append({"page": 1, "text": text, "method": "ocr_tesseract", "confidence": round(conf, 1)})
        if args.boxes:
            draw_boxes(prep, args.lang, args.psm, args.out / f"{path.stem}_boxes.png")
    for p in pages:
        p["gate"] = gate(p["method"], p["confidence"])
    return pages


def main():
    ap = argparse.ArgumentParser(description="OCR CLI: rasm/PDF -> matn + confidence")
    ap.add_argument("inputs", nargs="+", type=Path)
    ap.add_argument("--lang", default="rus+uzb")
    ap.add_argument("--psm", type=int, default=3)
    ap.add_argument("--preprocess", choices=["none", "otsu", "adaptive"], default="otsu")
    ap.add_argument("--no-deskew", action="store_true")
    ap.add_argument("--dpi", type=int, default=300, help="PDF sahifasini rasmga aylantirish DPI")
    ap.add_argument("--raw", action="store_true", help="clean_ocr_text qo'llanmasin")
    ap.add_argument("--boxes", action="store_true", help="so'z ramkalarini rasmga chizish")
    ap.add_argument("--out", type=Path, default=Path("out"))
    args = ap.parse_args()
    args.out.mkdir(exist_ok=True)

    for path in args.inputs:
        t0 = time.perf_counter()
        pages = process_file(path, args)
        (args.out / f"{path.stem}.txt").write_text("\n\n---\n\n".join(p["text"] for p in pages), encoding="utf-8")
        (args.out / f"{path.stem}.json").write_text(
            json.dumps([{k: v for k, v in p.items() if k != "text"} for p in pages], ensure_ascii=False, indent=1),
            encoding="utf-8")
        print(f"{path.name}: {len(pages)} sahifa, {time.perf_counter() - t0:.1f} s")
        for p in pages:
            print(f"  sahifa {p['page']}: {p['method']:14s} conf={p['confidence']:6.1f} -> {p['gate']:17s} "
                  f"'{p['text'][:50].replace(chr(10), ' ')}...'")


if __name__ == "__main__":
    main()

05_telefon_soya.jpg: 1 sahifa, 1.9 s
  sahifa 1: ocr_tesseract  conf=  95.8 -> accept            'SHARTNOMA М: 45 Toshkent shahri, 2024-yil 15-yanva...'
09_aralash.pdf: 3 sahifa, 2.5 s
  sahifa 1: text_layer     conf= 100.0 -> accept            'SHARTNOMA № 45 Toshkent shahri, 2024-yil 15-yanvar...'
  sahifa 2: ocr_tesseract  conf=  95.2 -> accept            'ДОГОВОР № 45 г. Ташкент, 15 января 2024 г.  ООО "Н...'
  sahifa 3: ocr_tesseract  conf=  92.7 -> accept            'ЗНАВТМОМА № 45  Toshkent shahri, 2024-yil 15-ya  "...'
05_telefon_soya.json
05_telefon_soya.txt
05_telefon_soya_boxes.png
09_aralash.json
09_aralash.txt

# evaluate.py — 1-loyiha: ground_truth/ ustida preprocessing variantlarini solishtirish.
#
#   python evaluate.py --preprocess none otsu adaptive
#   python evaluate.py --preprocess otsu --psm 6 --only chek
#
# Har rasm uchun CER/WER va o'rtacha; --csv bilan jadval faylga.
import argparse
import csv
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import ocr_text, preprocess, score, to_gray  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
S, G = ROOT / "samples", ROOT / "ground_truth"


def lang_for(name):
    if "kirill" in name or "gazeta" in name:
        return "uzb_cyrl"
    if "rus" in name:
        return "rus"
    return "uzb"


def variant(path, mode, deskew):
    return to_gray(path) if mode == "none" else preprocess(path, mode, do_deskew=deskew)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--preprocess", nargs="+", default=["none", "otsu", "adaptive"])
    ap.add_argument("--psm", type=int, default=3)
    ap.add_argument("--no-deskew", action="store_true")
    ap.add_argument("--only", default="", help="fayl nomida shu so'z bo'lganlar")
    ap.add_argument("--csv", type=Path)
    args = ap.parse_args()

    files = sorted(p for p in S.iterdir() if p.suffix in (".png", ".jpg") and (G / (p.stem + ".txt")).exists()
                   and args.only in p.name)
    table = {p.name: {} for p in files}
    for mode in args.preprocess:
        t0, cers, wers = time.perf_counter(), [], []
        for p in files:
            truth = (G / (p.stem + ".txt")).read_text(encoding="utf-8")
            sc = score(truth, ocr_text(variant(p, mode, not args.no_deskew), lang=lang_for(p.name), psm=args.psm))
            table[p.name][mode] = sc
            cers.append(sc["cer"]); wers.append(sc["wer"])
        print(f"{mode:10s} o'rtacha CER={sum(cers)/len(cers):.3f} WER={sum(wers)/len(wers):.3f} ({time.perf_counter()-t0:.0f} s)")

    print(f"\n{'rasm':22s}" + "".join(f"{m:>14s}" for m in args.preprocess))
    for name, row in table.items():
        print(f"{name:22s}" + "".join(f"{row[m]['cer']:7.3f}/{row[m]['wer']:4.2f} " for m in args.preprocess))
    if args.csv:
        with open(args.csv, "w", newline="", encoding="utf-8") as f:
            w = csv.writer(f)
            w.writerow(["file"] + [f"{m}_{k}" for m in args.preprocess for k in ("cer", "wer")])
            for name, row in table.items():
                w.writerow([name] + [f"{row[m][k]:.4f}" for m in args.preprocess for k in ("cer", "wer")])
        print("CSV:", args.csv)


if __name__ == "__main__":
    main()

otsu       o'rtacha CER=0.109 WER=0.257 (2 s)
adaptive   o'rtacha CER=0.041 WER=0.196 (2 s)

rasm                            otsu      adaptive
10_chek_1.jpg           0.022/0.14   0.045/0.22 
10_chek_2.jpg           0.293/0.51   0.048/0.22 
10_chek_3.jpg           0.012/0.12   0.031/0.15 
CSV: out/chek.csv

# receipt.py — 2-loyiha: chek surati -> JSON {sana, summa, inn, mahsulotlar}. Ikki usul yonma-yon:
#   A) Tesseract (PSM 6) + regex     B) VLM + JSON prompt (OLLAMA_HOST ochiq bo'lsa)
# Baholash: ground_truth/receipts.json bilan har maydon bo'yicha to'g'ri/noto'g'ri.
#
#   python receipt.py                 # 3 ta chek, ikkala usul, natija jadvali
import json
import os
import re
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import ocr_text, preprocess  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
S, G = ROOT / "samples", ROOT / "ground_truth"

DATE = re.compile(r"(\d{2})[./-](\d{2})[./-](\d{4})")
INN = re.compile(r"(?:INN|ИНН|STIR)\s*[:№]?\s*(\d{9})", re.I)
TOTAL = re.compile(r"(?:JAMI|ИТОГО|ИТОГ|TOTAL)\s*[:：]?\s*((?:\d[\d ]*\d)|\d)", re.I)
# "Non oq 400g 1 x 4 500": nom ichida raqam bo'lishi mumkin (400g), shuning uchun nom = .+? (eng qisqa),
# keyin miqdor, x (lotin yoki kirill х — OCR ikkalasini beradi), narx
ITEM = re.compile(r"^(?P<name>.+?)\s+(?P<qty>\d{1,2})\s*[xхX×]\s*(?P<price>\d[\d ]*\d)\s*$")


def num(s):
    return int(re.sub(r"\D", "", s))


def extract_regex(text):
    """OCR matnidan maydonlar. Topilmasa None (taxmin qilmaymiz)."""
    d = DATE.search(text)
    inn = INN.search(text)
    total = TOTAL.search(text)
    items = []
    for ln in text.splitlines():
        m = ITEM.match(ln.strip())
        if m:
            items.append([m["name"].strip(), int(m["qty"]), num(m["price"])])
    return {"sana": f"{d[1]}.{d[2]}.{d[3]}" if d else None,
            "inn": inn[1] if inn else None,
            "summa": num(total[1]) if total else None,
            "mahsulotlar": items}


def extract_vlm(image_path, model=os.environ.get("VLM_MODEL", "qwen2.5vl:7b")):
    """6-bob: prompt bilan maydon ajratish. Faqat JSON so'raymiz, topilmagan -> null, taxmin taqiqlanadi."""
    import base64
    from openai import OpenAI
    client = OpenAI(base_url=f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/v1", api_key="ollama")
    b64 = base64.b64encode(Path(image_path).read_bytes()).decode()
    prompt = ('Bu do\'kon cheki. Faqat JSON qaytar: {"sana": "DD.MM.YYYY", "inn": "9 raqam", "summa": butun son, '
              '"mahsulotlar": [["nomi", miqdor, narx], ...]}. Topilmagan maydon uchun null yoz. Taxmin qilma. '
              'JSON dan boshqa hech narsa yozma.')
    r = client.chat.completions.create(model=model, temperature=0, messages=[{"role": "user", "content": [
        {"type": "text", "text": prompt},
        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}]}])
    raw = r.choices[0].message.content
    m = re.search(r"\{.*\}", raw, re.S)
    try:
        return json.loads(m.group(0)) if m else {"_xato": raw[:120]}
    except json.JSONDecodeError:
        return {"_xato": raw[:120]}


def compare(pred, truth):
    """Har maydon: to'g'ri (1) yoki yo'q (0). Mahsulotlar: nechtasi nomi va narxi bilan mos."""
    res = {"sana": pred.get("sana") == truth["sana"],
           "inn": str(pred.get("inn")) == truth["inn"],
           "summa": pred.get("summa") == truth["summa"]}
    # mahsulotlar: qat'iy (nom + narx) va yumshoq (faqat narx) — OCR "1L" ni "IL" deb o'qisa nom mos kelmaydi, narx mos keladi
    t_items = {(n.lower(), p) for n, q, p in truth["mahsulotlar"]}
    p_items = {(str(n).lower(), p) for n, q, p in (pred.get("mahsulotlar") or []) if isinstance(p, int)}
    t_prices = sorted(p for _, p in t_items)
    p_prices = sorted(p for _, p in p_items)
    res["mahsulotlar"] = f"{len(t_items & p_items)}/{len(t_items)} (narx: {sum(1 for p in t_prices if p in p_prices)}/{len(t_prices)})"
    return res


def vlm_available(model=os.environ.get("VLM_MODEL", "qwen2.5vl:7b")):
    """Server ochiq VA model yuklangan bo'lsagina True (aks holda 404 bilan yiqilardi)."""
    try:
        import json as _json
        import urllib.request
        tags = _json.load(urllib.request.urlopen(f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/api/tags", timeout=3))
        return any(m["name"] == model for m in tags["models"])
    except Exception:
        return False


if __name__ == "__main__":
    truths = json.loads((G / "receipts.json").read_text(encoding="utf-8"))
    use_vlm = vlm_available()
    print(f"VLM: {'ochiq' if use_vlm else 'yopiq (faqat regex)'}\n")
    totals = {"regex": {"sana": 0, "inn": 0, "summa": 0}, "vlm": {"sana": 0, "inn": 0, "summa": 0}}
    for t in truths:
        img = S / t["file"]
        mode = "adaptive" if "2" in t["file"] else "otsu"                # 2-chek telefon surati
        t0 = time.perf_counter()
        text = ocr_text(preprocess(img, mode, do_deskew=False), lang="uzb", psm=6)
        pr = extract_regex(text)
        cr = compare(pr, t)
        print(f"{t['file']}  [regex, {time.perf_counter()-t0:.1f} s]  sana={pr['sana']} inn={pr['inn']} summa={pr['summa']} "
              f"mahsulot={len(pr['mahsulotlar'])} -> {cr}")
        for k in ("sana", "inn", "summa"):
            totals["regex"][k] += int(cr[k])
        if use_vlm:
            t0 = time.perf_counter()
            pv = extract_vlm(img)
            cv = compare(pv, t) if "_xato" not in pv else {"xato": pv["_xato"]}
            print(f"{'':14s}  [vlm,   {time.perf_counter()-t0:.1f} s]  sana={pv.get('sana')} inn={pv.get('inn')} "
                  f"summa={pv.get('summa')} mahsulot={len(pv.get('mahsulotlar') or [])} -> {cv}")
            for k in ("sana", "inn", "summa"):
                totals["vlm"][k] += int(bool(cv.get(k)))
    print("\nMaydon aniqligi (3 chekdan):", totals["regex"], "| vlm:", totals["vlm"] if use_vlm else "-")

# ocr_service.py — 3-loyiha: botdan ajratilgan "toza" funksiya. Telegram siz ham test qilinadi.
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import clean_ocr_text, ocr_with_confidence, preprocess  # noqa: E402

GOOD, OK = 80, 60


def process_image(path, lang="uzb"):
    """Surat -> {'text', 'conf', 'advice'}. Quality gate (8-bob) foydalanuvchi xabariga aylanadi."""
    prep = preprocess(path, "adaptive")                 # telefon surati: notekis yorug'lik
    text, conf = ocr_with_confidence(prep, lang=lang)
    text = clean_ocr_text(text)
    if not text:
        advice = "Matn topilmadi. Hujjatni to'liq kadrga oling va yorug'roq joyda suratga oling."
    elif conf < OK:
        advice = (f"Sifat past ({conf:.0f}%). Qayta suratga oling: yorug'lik bir tekis bo'lsin, "
                  "telefonni qog'ozga parallel tuting, soyadan qoching.")
    elif conf < GOOD:
        advice = f"O'rtacha sifat ({conf:.0f}%). Raqam va sanalarni tekshirib oling."
    else:
        advice = f"Yaxshi ({conf:.0f}%)."
    return {"text": text, "conf": round(conf, 1), "advice": advice, "words": len(text.split())}


if __name__ == "__main__":
    root = Path(__file__).resolve().parents[2] / "samples"
    for name in ("05_telefon_soya.jpg", "12_dogli.png", "10_chek_2.jpg"):
        r = process_image(root / name)
        print(f"{name}: conf={r['conf']} so'zlar={r['words']} -> {r['advice']}")
        print("   ", r["text"][:80].replace("\n", " "), "...")

# bot.py — 3-loyiha: Telegram bot (aiogram 3). Surat -> matn + confidence + maslahat.
#
#   pip install aiogram
#   BOT_TOKEN=123456:ABC... python bot.py
#
# Xavfsizlik: token faqat muhit o'zgaruvchisidan; fayl vaqtinchalik papkada va ishlagach o'chadi;
# foydalanuvchi uchun daqiqasiga RATE_LIMIT surat; 10 MB dan katta fayl qabul qilinmaydi.
import asyncio
import os
import tempfile
import time
from collections import defaultdict
from pathlib import Path

from aiogram import Bot, Dispatcher, F
from aiogram.filters import Command
from aiogram.types import Message

from ocr_service import process_image

RATE_LIMIT = 5                 # surat / daqiqa / foydalanuvchi
MAX_BYTES = 10 * 1024 * 1024
LANGS = {"uz": "uzb", "ru": "rus", "uzk": "uzb_cyrl", "uzru": "rus+uzb"}

dp = Dispatcher()
user_lang = {}                              # user_id -> tesseract lang
hits = defaultdict(list)                    # user_id -> [timestamp, ...]


def rate_limited(user_id):
    now = time.time()
    hits[user_id] = [t for t in hits[user_id] if now - t < 60]
    if len(hits[user_id]) >= RATE_LIMIT:
        return True
    hits[user_id].append(now)
    return False


@dp.message(Command("start"))
async def on_start(message: Message):
    await message.answer("Hujjat suratini yuboring, matnini qaytaraman.\n"
                         "Til: /lang uz (lotin), /lang uzk (kirill), /lang ru, /lang uzru")


@dp.message(Command("lang"))
async def on_lang(message: Message):
    code = (message.text.split(maxsplit=1) + [""])[1].strip().lower()
    if code not in LANGS:
        await message.answer("Variantlar: " + ", ".join(LANGS))
        return
    user_lang[message.from_user.id] = LANGS[code]
    await message.answer(f"Til: {LANGS[code]}")


@dp.message(F.photo)
async def on_photo(message: Message, bot: Bot):
    uid = message.from_user.id
    if rate_limited(uid):
        await message.answer(f"Daqiqasiga {RATE_LIMIT} ta surat. Biroz kuting.")
        return
    photo = message.photo[-1]                             # eng katta o'lcham
    if photo.file_size and photo.file_size > MAX_BYTES:
        await message.answer("Fayl 10 MB dan katta.")
        return
    with tempfile.TemporaryDirectory() as tmp:            # chiqishda papka va fayl o'chadi
        path = Path(tmp) / "in.jpg"
        await bot.download(photo, destination=path)
        result = await asyncio.to_thread(process_image, path, user_lang.get(uid, "uzb"))   # OCR event loop ni bloklamasin
    text = result["text"][:3500] or "(bo'sh)"
    await message.answer(f"{text}\n\n— {result['advice']}")


async def main():
    token = os.environ.get("BOT_TOKEN")
    if not token:
        raise SystemExit("BOT_TOKEN muhit o'zgaruvchisi kerak")
    bot = Bot(token)
    await dp.start_polling(bot)


if __name__ == "__main__":
    asyncio.run(main())

05_telefon_soya.jpg: conf=95.8 so'zlar=143 -> Yaxshi (96%).
    SHARTNOMA М: 45 Toshkent shahri, 2024-yil 15-yanvar  "Nihol" MChJ nomidan direkt ...
12_dogli.png: conf=83.7 so'zlar=143 -> Yaxshi (84%).
    “SHARTNOMA Ne 45 Toshkent shahri, 2024-yil.15-yanvar  "Nihol" MChJ nomidan Anvar ...
10_chek_2.jpg: conf=89.1 so'zlar=38 -> Yaxshi (89%).
    MAKRO MARKET INN 305987123  Sana: 02.03.2024 09:15  Guruch lazer 1kg Yog' IL Kar ...
bot.py import OK, handlers: 3

pip install aiogram
export BOT_TOKEN=123456789:AAF...        # @BotFather dan
python bot.py                             # /start, /lang uz|uzk|ru|uzru, surat yuboring

# table2xlsx.py — 4-loyiha: jadvalli skan -> .xlsx. Ikki yo'l:
#   A) Tesseract so'z koordinatalari + ustun chegaralarini o'zimiz topamiz (bo'shliq bo'yicha)
#   B) VLM -> Markdown jadval -> pandas (OLLAMA_HOST ochiq bo'lsa)
# Baholash: katak aniqligi = to'g'ri kataklar / jami kataklar (ground_truth/07_jadval.txt)
import os
import re
import sys
import time
from pathlib import Path

import cv2
import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import ocr_data, preprocess  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
S, G, OUT = ROOT / "samples", ROOT / "ground_truth", Path("out")
OUT.mkdir(exist_ok=True)


# ---------- A) Tesseract: avval jadval chiziqlarini topamiz, keyin so'zlarni kataklarga joylaymiz ----------
def grid_lines(binary, frac=12):
    """Uzun gorizontal/vertikal chiziqlar (morfologiya). Qaytaradi: chiziqlar maskasi, ustun x lari, qator y lari."""
    inv = 255 - binary                                   # matn va chiziqlar oq
    h, w = inv.shape
    horiz = cv2.morphologyEx(inv, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (w // frac, 1)))
    vert = cv2.morphologyEx(inv, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (1, h // frac)))

    def positions(profile, min_run=3):
        pos, start = [], None
        for i, v in enumerate(profile):
            if v and start is None:
                start = i
            elif not v and start is not None:
                if i - start >= min_run:
                    pos.append((start + i) // 2)
                start = None
        return pos
    xs = positions((vert.sum(axis=0) > 0.3 * 255 * h))     # ustun chiziqlari
    ys = positions((horiz.sum(axis=1) > 0.3 * 255 * w))    # qator chiziqlari
    return cv2.bitwise_or(horiz, vert), xs, ys


def tesseract_table(binary, lang="uzb"):
    """1) chiziqlarni olib tashlaymiz (Tesseract ularni harf deb o'ylaydi)  2) so'zlar -> (qator, ustun) katak."""
    grid, xs, ys = grid_lines(binary)
    no_lines = cv2.bitwise_or(binary, cv2.dilate(grid, np.ones((3, 3), np.uint8)))   # chiziq o'rni oq bo'ladi
    d = ocr_data(no_lines, lang=lang, psm=11)                   # siyrak matn: har so'z alohida
    if len(xs) < 2 or len(ys) < 2:
        return [], no_lines
    cells = {}
    for i, t in enumerate(d["text"]):
        if int(float(d["conf"][i])) <= 0 or not t.strip():
            continue
        cx, cy = d["left"][i] + d["width"][i] / 2, d["top"][i] + d["height"][i] / 2
        col = sum(1 for x in xs if cx > x) - 1
        row = sum(1 for y in ys if cy > y) - 1
        if 0 <= col < len(xs) - 1 and 0 <= row < len(ys) - 1:
            cells.setdefault((row, col), []).append((d["left"][i], t))
    table = []
    for r in range(len(ys) - 1):
        table.append([" ".join(t for _, t in sorted(cells.get((r, c), []))) for c in range(len(xs) - 1)])
    return table, no_lines


# ---------- B) VLM -> Markdown ----------
def markdown_table_to_rows(md):
    rows = []
    for ln in md.splitlines():
        ln = ln.strip()
        if not ln.startswith("|") or re.match(r"^\|?\s*:?-{2,}", ln):
            continue
        rows.append([c.strip() for c in ln.strip("|").split("|")])
    return rows


def vlm_table(image_path, model=os.environ.get("VLM_MODEL", "qwen2.5vl:7b")):
    import base64
    from openai import OpenAI
    client = OpenAI(base_url=f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/v1", api_key="ollama")
    b64 = base64.b64encode(Path(image_path).read_bytes()).decode()
    r = client.chat.completions.create(model=model, temperature=0, messages=[{"role": "user", "content": [
        {"type": "text", "text": "Rasmdagi jadvalni Markdown jadval qilib qaytar. Sarlavha qatorini ham. Boshqa hech narsa yozma."},
        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}])
    return r.choices[0].message.content


# ---------- umumiy ----------
def rows_to_xlsx(rows, path):
    width = max(len(r) for r in rows)
    rows = [r + [""] * (width - len(r)) for r in rows]
    pd.DataFrame(rows[1:], columns=[c or f"col{i}" for i, c in enumerate(rows[0])]).to_excel(path, index=False)


def norm(s):
    return re.sub(r"\s+", "", str(s)).lower().replace("'", "").replace("’", "")


def cell_accuracy(pred, truth):
    """Har haqiqiy katak bashoratda (o'z qatorida) bormi. Raqamli kataklar alohida sanaladi."""
    total = num_total = hit = num_hit = 0
    for i, trow in enumerate(truth):
        prow = pred[i] if i < len(pred) else []
        pcells = {norm(c) for c in prow}
        for c in trow:
            if not c:
                continue
            total += 1
            is_num = bool(re.fullmatch(r"[\d ]+", c))
            num_total += is_num
            if norm(c) in pcells:
                hit += 1
                num_hit += is_num
    return hit, total, num_hit, num_total


if __name__ == "__main__":
    truth = [ln.split(" | ") for ln in (G / "07_jadval.txt").read_text(encoding="utf-8").splitlines()]
    img = preprocess(S / "07_jadval.png", "otsu", do_deskew=False)

    t0 = time.perf_counter()
    rows_a, no_lines = tesseract_table(img)
    cv2.imwrite(str(OUT / "jadval_chiziqsiz.png"), no_lines)
    rows_to_xlsx(rows_a, OUT / "jadval_tesseract.xlsx")
    h, t, nh, nt = cell_accuracy(rows_a, truth)
    print(f"A) Tesseract + chiziqlar bo'yicha kataklar: {len(rows_a)} qator, kataklar {h}/{t}, raqamli {nh}/{nt}  ({time.perf_counter()-t0:.1f} s)")
    for r in rows_a[:4]:
        print("   ", r)

    try:
        import urllib.request
        urllib.request.urlopen(f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/api/tags", timeout=3)
        t0 = time.perf_counter()
        md = vlm_table(S / "07_jadval.png")
        rows_b = markdown_table_to_rows(md)
        rows_to_xlsx(rows_b, OUT / "jadval_vlm.xlsx")
        h, t, nh, nt = cell_accuracy(rows_b, truth)
        print(f"B) VLM Markdown:               {len(rows_b)} qator, kataklar {h}/{t}, raqamli {nh}/{nt}  ({time.perf_counter()-t0:.1f} s)")
        for r in rows_b[:4]:
            print("   ", r)
    except Exception as e:
        print(f"B) VLM o'tkazib yuborildi: {type(e).__name__}")
    print("Excel fayllar:", sorted(p.name for p in OUT.glob("*.xlsx")))

# translit.py — 5-loyiha: o'zbek kirill -> lotin. Qoidaga asoslangan, lug'atsiz.
# Nozik joylar: е so'z boshida/unlidan keyin "ye", ц -> "ts", ъ -> ' (tutuq belgisi), ь tashlanadi.
import re

VOWELS = "аеёиоуўэюяАЕЁИОУЎЭЮЯ"
MAP = {
    "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "ё": "yo", "ж": "j", "з": "z", "и": "i", "й": "y",
    "к": "k", "л": "l", "м": "m", "н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u",
    "ф": "f", "х": "x", "ц": "ts", "ч": "ch", "ш": "sh", "щ": "sh", "ъ": "'", "ь": "", "э": "e", "ю": "yu",
    "я": "ya", "ў": "o'", "қ": "q", "ғ": "g'", "ҳ": "h",
}


IOTATED = "еёюяЕЁЮЯ"


def _one(ch, prev, nxt):
    lower = ch.lower()
    if lower == "е":
        # so'z boshida, unlidan keyin va ъ/ь dan keyin "ye": етти -> yetti, объект -> obyekt
        out = "ye" if (prev is None or prev in VOWELS or prev.lower() in "ъь" or not prev.isalpha()) else "e"
    elif lower == "ъ" and nxt is not None and nxt in IOTATED:
        out = ""                       # объект -> obyekt: tutuq belgisi yozilmaydi, е o'zi "ye" bo'ladi
    else:
        out = MAP.get(lower, ch)
    return out.capitalize() if ch.isupper() and out else out


def cyr2lat(text):
    out = []
    for i, ch in enumerate(text):
        prev = text[i - 1] if i > 0 else None
        nxt = text[i + 1] if i + 1 < len(text) else None
        out.append(_one(ch, prev, nxt))
    return "".join(out)


if __name__ == "__main__":
    tests = [("Тошкент шаҳри", "Toshkent shahri"), ("ўзбек тили", "o'zbek tili"), ("Ғафур Ғулом", "G'afur G'ulom"),
             ("етказиб бериш", "yetkazib berish"), ("шартнома", "shartnoma"), ("Қуйидагилар ҳақида", "Quyidagilar haqida"),
             ("объект", "obyekt"), ("санъат", "san'at"), ("цирк", "tsirk"), ("Ёзилган", "Yozilgan")]
    for src, want in tests:
        got = cyr2lat(src)
        print(("OK " if got == want else "!! ") + f"{src} -> {got}" + ("" if got == want else f"  (kutilgan: {want})"))

OK Тошкент шаҳри -> Toshkent shahri
OK ўзбек тили -> o'zbek tili
OK Ғафур Ғулом -> G'afur G'ulom
OK етказиб бериш -> yetkazib berish
OK шартнома -> shartnoma
OK Қуйидагилар ҳақида -> Quyidagilar haqida
OK объект -> obyekt
OK санъат -> san'at
OK цирк -> tsirk
OK Ёзилган -> Yozilgan

# benchmark.py — 5-loyiha: o'zbek lotin va kirill uchun uch dvigatelni raqam bilan solishtirish.
#   Tesseract (uzb / uzb_cyrl), EasyOCR (en / ru — o'zbek yo'q), VLM (OLLAMA_HOST ochiq bo'lsa)
# Chiqish: CER/WER/vaqt matritsasi, eng ko'p adashadigan belgilar, kirill OCR + translit tajribasi.
import difflib
import os
import sys
import time
from collections import Counter
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import normalize, ocr_text, preprocess, score  # noqa: E402
from translit import cyr2lat  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
S, G = ROOT / "samples", ROOT / "ground_truth"

LATIN = ["03_uzb_lotin.png", "05_telefon_soya.jpg", "06_qiyshiq.png", "11_past_dpi.png", "12_dogli.png"]
CYRIL = ["04_uzb_kirill.png", "14_gazeta_1.png", "14_gazeta_3.png"]


def truth_of(name):
    return (G / (Path(name).stem + ".txt")).read_text(encoding="utf-8")


def mode_for(name):
    return "adaptive" if "telefon" in name else "otsu"


# ---------- dvigatellar ----------
def tesseract_engine(script):
    lang = "uzb" if script == "lotin" else "uzb_cyrl"
    return lambda name: ocr_text(preprocess(S / name, mode_for(name)), lang=lang)


def easyocr_engine(script):
    import easyocr
    reader = easyocr.Reader(["en"] if script == "lotin" else ["ru"], gpu=False, verbose=False)
    return lambda name: "\n".join(r[1] for r in reader.readtext(str(S / name)) if r[2] > 0.3)


def vlm_engine(model=os.environ.get("VLM_MODEL", "qwen2.5vl:7b")):
    import base64
    import io
    from openai import OpenAI
    from PIL import Image
    client = OpenAI(base_url=f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/v1", api_key="ollama")

    def run(name):
        im = Image.open(S / name).convert("RGB")
        im.thumbnail((1600, 1600))
        buf = io.BytesIO()
        im.save(buf, format="PNG")
        b64 = base64.b64encode(buf.getvalue()).decode()
        # 11-bob 6-lab: yumshoq prompt bilan model lotinni kirillga o'giradi. Qat'iy prompt + ``` qavslarni olib tashlash.
        prompt = ("Rasmdagi matnni AYNAN yozilganidek ko'chir: qaysi alifboda bo'lsa (lotin yoki kirill) shu alifboda, "
                  "tarjima qilma, alifboni o'zgartirma, imloni tuzatma. Qatorlarni saqla. Markdown, ``` qavslar, izoh qo'shma.")
        r = client.chat.completions.create(model=model, temperature=0, messages=[{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}])
        import re as _re
        return _re.sub(r"^```[a-z]*\s*|\s*```$", "", r.choices[0].message.content.strip(), flags=_re.M)
    return run


def confusions(truth, pred, top=8):
    """Qaysi belgi qaysi belgiga almashgan (SequenceMatcher 'replace' bloklari, 1-2 belgili)."""
    t, p = normalize(truth), normalize(pred)
    c = Counter()
    for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, t, p, autojunk=False).get_opcodes():
        if tag == "replace" and i2 - i1 <= 2 and j2 - j1 <= 2:
            c[(t[i1:i2], p[j1:j2])] += 1
    return c.most_common(top)


def evaluate(engine, files, label):
    cers, wers, t0, conf = [], [], time.perf_counter(), Counter()
    for name in files:
        pred = engine(name)
        sc = score(truth_of(name), pred)
        cers.append(sc["cer"]); wers.append(sc["wer"])
        for k, v in confusions(truth_of(name), pred):
            conf[k] += v
    dt = (time.perf_counter() - t0) / len(files)
    print(f"  {label:22s} CER={sum(cers)/len(cers):.3f}  WER={sum(wers)/len(wers):.3f}  {dt:5.1f} s/sahifa   "
          f"xatolar: {', '.join(f'{a}->{b}({n})' for (a, b), n in conf.most_common(4))}")
    print("  " + " " * 22 + "sahifalar: " + "  ".join(f"{n[:12]}={c:.3f}" for n, c in zip(files, cers)))
    return cers


def available(kind):
    if kind == "easyocr":
        try:
            import easyocr  # noqa: F401
            return True
        except ImportError:
            return False
    if kind == "vlm":
        try:
            import json
            import urllib.request
            tags = json.load(urllib.request.urlopen(f"{os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}/api/tags", timeout=3))
            return any(m["name"] == os.environ.get("VLM_MODEL", "qwen2.5vl:7b") for m in tags["models"])
        except Exception:
            return False


if __name__ == "__main__":
    for script, files in (("lotin", LATIN), ("kirill", CYRIL)):
        print(f"\n== {script} ({len(files)} sahifa)")
        evaluate(tesseract_engine(script), files, "Tesseract")
        if available("easyocr"):
            evaluate(easyocr_engine(script), files, "EasyOCR (en/ru)")
        if available("vlm"):
            evaluate(vlm_engine(), files, "VLM")

    print("\n== Translit tajribasi: kirill OCR -> lotin, lotin haqiqat bilan solishtirish")
    # 04_uzb_kirill va 03_uzb_lotin bir xil shartnoma: kirill OCR ni lotinga o'girib, lotin haqiqatga solishtiramiz
    pred = cyr2lat(ocr_text(preprocess(S / "04_uzb_kirill.png", "otsu"), lang="uzb_cyrl"))
    truth_lat = truth_of("03_uzb_lotin.png")
    sc = score(truth_lat, pred)
    print(f"  kirill OCR + translit vs lotin haqiqat: CER={sc['cer']:.3f} WER={sc['wer']:.3f}")
    sc2 = score(truth_lat, ocr_text(preprocess(S / "03_uzb_lotin.png", "otsu"), lang="uzb"))
    print(f"  to'g'ridan-to'g'ri lotin OCR:           CER={sc2['cer']:.3f} WER={sc2['wer']:.3f}")
    print("  farq: imlo (йил/yil, 2024 йил/2024-yil) va translit qoidalari, OCR emas —", confusions(truth_lat, pred, 5))

# archive.py — 6-loyiha: eski gazeta (sarg'aygan, ikki ustun, orqa tomon ko'rinadi) -> toza matn.
#   1) preprocessing zanjiri, har qadamdan keyin CER    2) ustunlarni o'zimiz ajratamiz (proyeksiya)
#   3) sahifalar orasida takrorlangan qatorlar (kolontitul) olib tashlanadi   4) txt + metadata.json
import json
import re
import sys
from pathlib import Path

import cv2
import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ocrlib import clean_ocr_text, deskew, ocr_text, ocr_with_confidence, score, to_gray  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
S, G, OUT = ROOT / "samples", ROOT / "ground_truth", Path("out")
OUT.mkdir(exist_ok=True)
LANG = "uzb_cyrl"
PAGES = ["14_gazeta_1.png", "14_gazeta_2.png", "14_gazeta_3.png"]


# ---------- 1) zanjir ----------
def chain(gray):
    """Har qadam (nom, rasm) qaytaradi — qaysi qadam foyda berganini o'lchash uchun."""
    steps = [("kulrang", gray)]
    med = cv2.medianBlur(gray, 3)
    steps.append(("+ medianBlur(3)", med))
    _, otsu = cv2.threshold(med, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    steps.append(("+ Otsu", otsu))
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2, 2))
    opened = cv2.morphologyEx(otsu, cv2.MORPH_CLOSE, kernel)     # oq fon ichidagi mayda qora nuqtalar (bleed qoldig'i)
    steps.append(("+ morph CLOSE 2x2", opened))
    steps.append(("+ deskew", deskew(opened)))
    return steps


# ---------- 2) ustunlar ----------
def split_columns(binary, min_gap=40, margin=0.15, header=0.08, footer=0.92):
    """Sahifa: sarlavha tasmasi (ikki ustunga cho'zilgan) + tana (ustunlar) + sahifa raqami.
    Tana uchun vertikal proyeksiya: har x da nechta qora piksel; o'rtadagi eng keng oq bo'shliq = ustunlar orasi.
    Qaytaradi: [sarlavha, chap ustun, o'ng ustun] (yoki ustun topilmasa [sarlavha, tana])."""
    h, w = binary.shape
    head, body = binary[:int(h * header), :], binary[int(h * header):int(h * footer), :]
    ink = (body < 128).sum(axis=0)
    lo, hi = int(w * margin), int(w * (1 - margin))
    cur_start, gaps = None, []
    for x in range(lo, hi):
        if ink[x] == 0:
            cur_start = x if cur_start is None else cur_start
        elif cur_start is not None:
            gaps.append((x - cur_start, cur_start, x))
            cur_start = None
    if not gaps or max(gaps)[0] < min_gap:
        return [head, body]
    _, a, b = max(gaps)
    mid = (a + b) // 2
    return [head, body[:, :mid], body[:, mid:]]


def ocr_columns(binary):
    parts = split_columns(binary)
    texts = [ocr_text(p, lang=LANG, psm=6) for p in parts]          # har bo'lak: bitta yaxlit blok
    return "\n\n".join(t.strip() for t in texts if t.strip()), len(parts) - 1


# ---------- 3) kolontitul ----------
def remove_repeated_lines(pages_text, min_pages=2):
    norm = lambda s: re.sub(r"\W+", "", s).lower()
    count = {}
    for text in pages_text:
        for ln in {norm(l) for l in text.splitlines() if l.strip()}:
            count[ln] = count.get(ln, 0) + 1
    repeated = {k for k, v in count.items() if v >= min_pages and len(k) > 10}
    out = []
    for text in pages_text:
        kept = [l for l in text.splitlines() if norm(l) not in repeated]
        out.append("\n".join(kept))
    return out, repeated


if __name__ == "__main__":
    print("1) Preprocessing zanjiri, har qadamdan keyingi CER (PSM 3, avtomatik layout):")
    print(f"   {'qadam':20s}" + "".join(f"{p[:-4]:>14s}" for p in PAGES) + "    o'rtacha")
    chains = {p: chain(to_gray(S / p)) for p in PAGES}
    means = []
    for i, (name, _) in enumerate(chains[PAGES[0]]):
        row = [score((G / (p[:-4] + ".txt")).read_text(encoding="utf-8"), ocr_text(chains[p][i][1], lang=LANG))["cer"]
               for p in PAGES]
        means.append(sum(row) / len(row))
        print(f"   {name:20s}" + "".join(f"{c:14.3f}" for c in row) + f"{means[-1]:12.3f}")
    best = min(range(len(means)), key=means.__getitem__)
    print(f"   -> eng yaxshi qadam: '{chains[PAGES[0]][best][0]}'. Undan keyingi qadamlar zarar qildi, ularni tashlaymiz.")
    finals = {p: chains[p][best][1] for p in PAGES}

    print("\n2) Ustunlarni proyeksiya bilan ajratish (PSM 6 har ustunga) vs avtomatik (PSM 3):")
    pages_text, meta = [], []
    for p in PAGES:
        truth = (G / (p[:-4] + ".txt")).read_text(encoding="utf-8")
        auto = ocr_text(finals[p], lang=LANG)
        manual, ncol = ocr_columns(finals[p])
        _, conf = ocr_with_confidence(finals[p], lang=LANG)
        print(f"   {p}: auto CER={score(truth, auto)['cer']:.3f}   ustunlar={ncol} CER={score(truth, manual)['cer']:.3f}   conf={conf:.1f}")
        pages_text.append(manual)
        meta.append({"file": p, "columns": ncol, "confidence": round(conf, 1)})

    print("\n3) Takrorlangan qatorlar (kolontitul) va sahifa raqamlari:")
    cleaned, repeated = remove_repeated_lines(pages_text)
    print("   olib tashlandi:", [r[:40] for r in repeated])
    for p, text, m in zip(PAGES, cleaned, meta):
        final = clean_ocr_text(text)
        (OUT / (p[:-4] + ".txt")).write_text(final, encoding="utf-8")
        m["chars"] = len(final)
    (OUT / "metadata.json").write_text(json.dumps(meta, ensure_ascii=False, indent=1), encoding="utf-8")
    print("   yozildi:", sorted(f.name for f in OUT.iterdir()))
    print("   1-sahifa boshi:", repr(clean_ocr_text(cleaned[0])[:100]))

# db.py — 7-loyiha: SQLite + FTS5. pages jadvali (matn + metadata) va to'liq matnli indeks.
import sqlite3
from datetime import datetime

SCHEMA = """
CREATE TABLE IF NOT EXISTS documents (
    id INTEGER PRIMARY KEY, filename TEXT UNIQUE, pages INTEGER, created TEXT);
CREATE TABLE IF NOT EXISTS pages (
    id INTEGER PRIMARY KEY, doc_id INTEGER REFERENCES documents(id), page_num INTEGER,
    text TEXT, method TEXT, confidence REAL, gate TEXT, review_status TEXT DEFAULT 'none');
CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
    text, content='pages', content_rowid='id', tokenize='unicode61 remove_diacritics 0');
CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
    INSERT INTO pages_fts(rowid, text) VALUES (new.id, new.text); END;
CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
    INSERT INTO pages_fts(pages_fts, rowid, text) VALUES ('delete', old.id, old.text); END;
"""


def connect(path="archive.db"):
    con = sqlite3.connect(path)
    con.row_factory = sqlite3.Row
    con.executescript(SCHEMA)
    return con


def add_document(con, filename, n_pages):
    con.execute("DELETE FROM pages WHERE doc_id IN (SELECT id FROM documents WHERE filename=?)", (filename,))
    con.execute("DELETE FROM documents WHERE filename=?", (filename,))
    cur = con.execute("INSERT INTO documents(filename, pages, created) VALUES (?,?,?)",
                      (filename, n_pages, datetime.now().isoformat(timespec="seconds")))
    return cur.lastrowid


def add_page(con, doc_id, page_num, text, method, confidence, gate):
    con.execute("INSERT INTO pages(doc_id, page_num, text, method, confidence, gate) VALUES (?,?,?,?,?,?)",
                (doc_id, page_num, text, method, confidence, gate))


def search(con, query, limit=10, min_conf=0):
    """FTS5 MATCH + snippet. min_conf: faqat ishonchli sahifalar (8-bob: filtering)."""
    sql = """SELECT p.id, d.filename, p.page_num, p.method, p.confidence, p.gate,
                    snippet(pages_fts, 0, '[', ']', '…', 12) AS snip, bm25(pages_fts) AS rank
             FROM pages_fts JOIN pages p ON p.id = pages_fts.rowid JOIN documents d ON d.id = p.doc_id
             WHERE pages_fts MATCH ? AND p.confidence >= ? ORDER BY rank LIMIT ?"""
    return con.execute(sql, (query, min_conf, limit)).fetchall()


def flagged(con):
    return con.execute("""SELECT p.id, d.filename, p.page_num, p.method, p.confidence, p.gate, p.review_status, substr(p.text, 1, 300) AS preview
                          FROM pages p JOIN documents d ON d.id = p.doc_id
                          WHERE p.gate != 'accept' ORDER BY p.confidence""").fetchall()


def set_review(con, page_id, status):
    con.execute("UPDATE pages SET review_status=? WHERE id=?", (status, page_id))
    con.commit()


def stats(con):
    return con.execute("""SELECT COUNT(*) AS pages, SUM(method='text_layer') AS text_layer,
                                 SUM(method LIKE 'ocr%') AS ocr, SUM(gate!='accept') AS flagged FROM pages""").fetchone()