Ma'lumot Yig'ish (Data Ingestion)

nihol.uz → Scrape (httpx) → Extract (BeautifulSoup/Trafilatura)
         → Normalize → Chunk → Embed → PostgreSQL

from bs4 import BeautifulSoup

html = "<div class='content'><p>Nihol kompaniyasi</p></div>"
soup = BeautifulSoup(html, "lxml")
text = soup.select_one(".content p").get_text(strip=True)
# Natija: "Nihol kompaniyasi"

# navigator.py dan olingan haqiqiy kod
def _extract_from_soup(soup, css_selector, base_url, page_url):
    base_domain = urlparse(base_url).netloc
    url_selector, name_selector = _parse_selector(css_selector)
    elements = soup.select(url_selector) if url_selector else soup.find_all("a", href=True)
    names = soup.select(name_selector) if name_selector else []

    results = []
    for i, el in enumerate(elements):
        href = el.get("href", "").strip()
        if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
            continue

        full_url = urljoin(page_url, href)
        parsed = urlparse(full_url)
        if parsed.netloc and parsed.netloc != base_domain:
            continue

        relative = parsed.path
        if not relative:
            continue

        if names and i < len(names):
            title = (names[i].get_text(strip=True) or relative)[:200]
        else:
            title = (el.get_text(strip=True) or relative)[:200]
        results.append((relative, title))

    return results

# extractor.py dan olingan haqiqiy kod
def _extract_trafilatura(html, url, soup):
    text = trafilatura.extract(html, url=url, include_comments=False) or ""
    metadata = trafilatura.extract(html, url=url, output_format="xmltei")
    title_tag = soup.find("title")
    title = title_tag.get_text(strip=True) if title_tag else ""
    return {"title": title, "content_text": text, "content_html": metadata or ""}

# extractor.py dan olingan haqiqiy kod
def extract_page(session, page):
    # ...
    html = resp.text
    soup = BeautifulSoup(html, "lxml")

    if page.css_selector_ctn:
        result = _extract_bs4(soup, page.css_selector_ctn)
    else:
        result = _extract_trafilatura(html, page_url, soup)

    result["content_text"] = normalize_text(result.get("content_text", ""))
    # ...

# Scrapy spider misoli (biz ishlatmaymiz, lekin katta loyihalarda kerak)
import scrapy

class NiholSpider(scrapy.Spider):
    name = "nihol"
    start_urls = ["https://nihol.uz/"]

    def parse(self, response):
        for link in response.css("a.project-link::attr(href)").getall():
            yield response.follow(link, callback=self.parse_project)

    def parse_project(self, response):
        yield {
            "title": response.css("h1::text").get(),
            "content": response.css(".content").get(),
            "url": response.url,
        }

import httpx

resp = httpx.get(page_url, timeout=30, follow_redirects=True)
resp.raise_for_status()
html = resp.text

import fitz  # PyMuPDF

doc = fitz.open("document.pdf")
for page in doc:
    text = page.get_text()
    print(text)

import pdfplumber

with pdfplumber.open("report.pdf") as pdf:
    for page in pdf.pages:
        tables = page.extract_tables()
        for table in tables:
            for row in table:
                print(row)

from unstructured.partition.auto import partition

elements = partition(filename="document.pdf")
for el in elements:
    print(el.category, el.text[:100])

# Tesseract OCR
import pytesseract
from PIL import Image

image = Image.open("scanned_page.png")
text = pytesseract.image_to_string(image, lang="rus+uzb")

master (generates_children_yn = "Y")
├── child_master (ham master, ham bola)
│   ├── detail (has_content_yn = "Y")
│   └── detail
└── detail

# navigator.py dan olingan haqiqiy kod
def _parse_master_child(css_selector):
    if css_selector and ">>" in css_selector:
        parts = [p.strip() for p in css_selector.split(">>", 1)]
        return parts[0], parts[1]  # master_selector, detail_selector
    return None, None

# navigator.py dan olingan haqiqiy kod
def _parse_selector(css_selector):
    if css_selector and "|" in css_selector:
        parts = [p.strip() for p in css_selector.split("|", 1)]
        return parts[0], parts[1]  # url_selector, name_selector
    return css_selector, None

# pagination.py dan olingan haqiqiy kod
def detect_pagination(html, current_url):
    soup = BeautifulSoup(html, "lxml")
    parsed_current = urlparse(current_url)

    urls = _try_rel_next_chain(soup, current_url)
    if urls:
        return urls

    urls = _try_numbered_sequences(soup, current_url, parsed_current)
    if urls:
        return urls

    urls = _try_url_pattern_analysis(soup, current_url, parsed_current)
    if urls:
        return urls

    return []

# extractor.py dan olingan haqiqiy kod
def extract_page(session, page):
    existing = crud.get_extracted_content(session, page.id)
    if existing:
        return {"id": existing[0].id, "url": existing[0].url, "skipped": True}

    base_url = page.site.base_url.rstrip("/")
    page_url = base_url + (page.relative_url or "")

    url_exists = crud.get_extracted_content_by_url(session, page_url)
    if url_exists:
        return {"id": url_exists.id, "url": page_url, "skipped": True}

    resp = httpx.get(page_url, timeout=30, follow_redirects=True)
    resp.raise_for_status()
    html = resp.text
    # ... extraction logic

# extractor.py dan olingan haqiqiy kod
def _clean_soup(element):
    for tag in element.find_all("img"):
        tag.decompose()                          # rasmlarni olib tashla
    for tag in element.find_all(["p", "h6", "span", "div"]):
        text = tag.get_text(strip=True).lower()
        if text in ("made with ai", "teglar:", "теги:", "tags:"):
            tag.decompose()                      # teglarni olib tashla
    for a in element.find_all("a"):
        text = a.get_text(strip=True).lower()
        if "ознакомиться здесь" in text:
            parent = a.find_parent("p")
            if parent:
                parent.decompose()               # "batafsil bu erda" havolalarini olib tashla
    _remove_contact_spans(element)               # kontakt ma'lumotlarini olib tashla

# normalizer.py dan olingan haqiqiy kod
def normalize_text(text):
    if not text:
        return ""

    text = text.replace("\r", "")           # Windows satr uzilishlari
    text = text.replace("\xad", "")          # soft hyphen (ko'rinmas tire)
    text = text.replace("\xa0", " ")         # non-breaking space -> oddiy bo'sh joy
    text = text.replace("\u2011", "-")       # maxsus tire -> oddiy tire
    text = text.replace("\u00b7", " ")       # middle dot -> bo'sh joy
    text = text.replace("\u2022", "- ")      # bullet -> tire
    text = re.sub(r"[ \t]+", " ", text)      # ko'p bo'sh joylarni birlashtir

# normalizer.py dan olingan haqiqiy kod
def _remove_contact_blocks(text):
    text = re.sub(r"Тел:?\s*[\d\s\-\(\)\+]{5,}", "", text)
    text = re.sub(r"^\+?\d[\d\s\-\(\)]{6,}$", "", text, flags=re.MULTILINE)
    text = re.sub(
        r"^(Начальник|Ведущий|Глава|ИТ.?Менеджер|Руководитель)\b.*$",
        "", text, flags=re.MULTILINE
    )
    # ... kontakt shaxslarini va mustaqil ismlarni olib tashlash

# normalizer.py dan olingan haqiqiy kod
def _remove_junk_lines(text):
    junk = [
        r"^Все проекты по годам\s*$",
        r"^Все проекты\s*$",
        r"^.*(?:смотрите|можно)\s+здесь\.?\s*$",
        r"^made with ai\s*$",
        r"^qo'shimcha ravishda bu erda\s*$",
        r"^Рисунок\s*\d+\.?\s*$",
        r"^Дата:\s*$",
    ]
    for pat in junk:
        text = re.sub(pat, "", text, flags=re.MULTILINE | re.IGNORECASE)
    return text

# pipeline.py dan olingan haqiqiy kod
def run_pipeline_for_site(session, site_id):
    total_discovered = 0
    total_extracted = 0
    errors = []

    # 1-bosqich: barcha master sahifalardan havolalarni kashf qilish (iterativ)
    processed_ids = set()
    while True:
        pages = crud.get_target_pages(session, site_id)
        masters = [
            p for p in pages
            if p.page_type and p.page_type.generates_children_yn == "Y"
            and p.active_yn == "Y" and p.id not in processed_ids
        ]
        if not masters:
            break
        for master in masters:
            processed_ids.add(master.id)
            try:
                result = discover_links(session, master)
                total_discovered += result["links"]
            except Exception as e:
                errors.append({"page_id": master.id, "phase": "navigate", "error": str(e)})

    # 2-bosqich: barcha detail sahifalardan kontentni olish
    details = crud.get_target_pages(session, site_id)
    details = [p for p in details if p.page_type and p.page_type.has_content_yn == "Y" and p.active_yn == "Y"]

    for detail in details:
        try:
            extract_page(session, detail)
            total_extracted += 1
        except Exception as e:
            errors.append({"page_id": detail.id, "phase": "extract", "error": str(e)})

    return {"pages_discovered": total_discovered, "pages_extracted": total_extracted, "errors": errors}

[nihol.uz]
    │
    ▼
┌──────────────┐
│  Navigator   │  CSS selectors orqali havolalarni topadi
│  (httpx +    │  Pagination ni aniqlaydi
│  BeautifulSoup)│  Master → Child Master → Detail ierarxiyasi
└──────┬───────┘
       │
       ▼
┌──────────────┐
│  Extractor   │  Har bir detail sahifani yuklaydi
│  (BS4 yoki   │  CSS selector yoki Trafilatura orqali matnni oladi
│  Trafilatura)│  Dublikatlarni tekshiradi
└──────┬───────┘
       │
       ▼
┌──────────────┐
│  Normalizer  │  Maxsus belgilarni tozalaydi
│  (regex)     │  Kontaktlarni, junk satrlarni olib tashlaydi
│              │  Bo'sh satrlarni tartibga soladi
└──────┬───────┘
       │
       ▼
┌──────────────┐
│  Chunker     │  Paragraph-based bo'lish (\n\n)
│              │  Prefix qo'shish (rubrika + nom)
│              │  MAX_TOKENS chegarasi
└──────┬───────┘
       │
       ▼
┌──────────────┐
│  Embedder    │  qwen3-embedding (local, 4096 o'lcham)
│  (Ollama)    │  yoki OpenAI (cloud, 1536 o'lcham)
└──────┬───────┘
       │
       ▼
┌──────────────┐
│  PostgreSQL  │  pgvector orqali vektor saqlash
│  + pgvector  │  embedding_local + embedding_cloud ustunlari
└──────────────┘

CSV qatori:
  loyiha_nomi,mijoz,yil,soha,byudjet
  Billing modernizatsiya,Ucell,2023,telekom,450000

Noto'g'ri (xom matn):
  "Billing modernizatsiya,Ucell,2023,telekom,450000"

To'g'ri (tabiiy jumla):
  "Billing modernizatsiya loyihasi Ucell kompaniyasi uchun 2023-yilda
   telekom sohasida bajarilgan. Loyiha byudjeti 450000 AQSH dollari."