Collecting your own data: scraping and APIs

import pandas as pd
show(pd.DataFrame([
    {"Route": "1. Official API", "Quality": "Best",
     "Note": "Clean structure, clear terms, doesn't break"},
    {"Route": "2. Open data / export", "Quality": "Good",
     "Note": "stat.uz, data.gov.uz, the site's own CSV export"},
    {"Route": "3. Ask for a partnership", "Quality": "Good",
     "Note": "Often works, just ask. Free and legal"},
    {"Route": "4. Scraping", "Quality": "Last resort",
     "Note": "Breaks when the site changes; the legal liability is yours"},
]))

from pathlib import Path
FIX = DATA / "fixture"
for p in sorted(FIX.iterdir()):
    print(f"  {p.name:<18} {p.stat().st_size:>6} bytes")
print()
raw_html = (FIX / "elonlar_1.html").read_text(encoding="utf-8")
print("What the first listing block looks like:")
start = raw_html.index('<div class="elon"')
print(raw_html[start:start + 560])

from bs4 import BeautifulSoup

soup = BeautifulSoup(raw_html, "html.parser")
listings = soup.select("div.elon")
print(f"Listings found on page 1: {len(listings)}")
print()
first = listings[0]
print(f"  data-id  : {first['data-id']}")
print(f"  title    : {first.select_one('h3.sarlavha').get_text(strip=True)}")
print(f"  price    : {first.select_one('div.narx').get_text(strip=True)}")
print(f"  district : {first.select_one('span.tuman').get_text(strip=True)}")
rooms = first.select_one("span.xona")
print(f"  rooms    : {rooms.get_text(strip=True) if rooms else 'MISSING'}")

"""Rules checked BEFORE collecting."""
import urllib.parse
import urllib.request
import urllib.robotparser

AGENT = "MLBookStudent/1.0 (+study project; contact: student@example.uz)"


def is_allowed(url, agent=AGENT):
    """Whether robots.txt allows fetching this URL.

    Note: robots.txt allowing it does NOT mean the ToS allows it.
    A person has to read the terms, not code.
    """
    parts = urllib.parse.urlparse(url)
    rp = urllib.robotparser.RobotFileParser()
    rp.set_url(f"{parts.scheme}://{parts.netloc}/robots.txt")
    try:
        rp.read()
    except Exception:
        return False            # if robots.txt can't be read, we treat it as NOT ALLOWED
    return rp.can_fetch(agent, url)


PERSONAL = {"tel", "telefon", "phone", "ism", "name", "email", "manzil", "address"}


def drop_personal(record):
    """Removes personal fields from a record.

    Called at the START of collection, not the END: data that was
    never written to disk can't be stolen.
    """
    return {k: v for k, v in record.items() if k.lower() not in PERSONAL}

"""A slow, forgiving, caching downloader."""
import hashlib
import time
import urllib.request
from pathlib import Path

CACHE = Path("cache")
DELAY = 1.0              # one request per second
ATTEMPTS = 3


def _cache_path(url):
    return CACHE / (hashlib.sha256(url.encode()).hexdigest()[:16] + ".html")


def fetch(url, agent="MLBookStudent/1.0", force=False):
    """Returns the text at a URL. Downloaded once, then served from the cache.

    The cache is the core of politeness: even if you run your code
    50 times, the site gets one request.
    """
    CACHE.mkdir(exist_ok=True)
    path = _cache_path(url)
    if path.exists() and not force:
        return path.read_text(encoding="utf-8"), "cache"

    last_error = None
    for attempt in range(1, ATTEMPTS + 1):
        try:
            request = urllib.request.Request(url, headers={"User-Agent": agent})
            with urllib.request.urlopen(request, timeout=20) as response:
                text = response.read().decode("utf-8", errors="replace")
            path.write_text(text, encoding="utf-8")
            time.sleep(DELAY)
            return text, "network"
        except Exception as x:
            last_error = x
            time.sleep(DELAY * 2 ** attempt)     # exponential backoff
    raise RuntimeError(f"not fetched after {ATTEMPTS} attempts: {url}") from last_error

"""HTML -> records. NO cleaning at all: stored raw."""
from bs4 import BeautifulSoup

from .rules import drop_personal


def text(block, selector):
    """Returns None if missing, not an AttributeError."""
    found = block.select_one(selector)
    return found.get_text(strip=True) if found else None


def parse_page(html_text):
    soup = BeautifulSoup(html_text, "html.parser")
    result = []
    for block in soup.select("div.elon"):
        record = {
            "source_id": block.get("data-id"),
            "title": text(block, "h3.sarlavha"),
            "price_raw": text(block, "div.narx"),
            "rooms_raw": text(block, "span.xona"),
            "area_raw": text(block, "span.maydon"),
            "district": text(block, "span.tuman"),
            "floor_raw": text(block, "span.qavat"),
            "phone": text(block, "div.aloqa"),          # collected, but NOT stored
        }
        result.append(drop_personal(record))
    return result

import sys
sys.path.insert(0, str(ISH))
from collect.parse import parse_page

pages = sorted(FIX.glob("elonlar_*.html"))
records = []
for page_path in pages:
    this = parse_page(page_path.read_text(encoding="utf-8"))
    records += this
    if page_path is pages[0] or page_path is pages[-1]:
        print(f"  {page_path.name} -> {len(this)} records")
print(f"  ... {len(pages)} pages read in total")
raw_df = pd.DataFrame(records)
print(f"\nTotal: {len(raw_df)} records, {len(raw_df.columns)} columns")
print(f"Columns: {list(raw_df.columns)}")
print()
print(f"Is there a phone column? {'phone' in raw_df.columns}  <- drop_personal() worked")
show(raw_df.head(6))

broken = (FIX / "elonlar_3.html").read_text(encoding="utf-8")
print(f"Page 3 has an unclosed <div>: {broken.count('<div')} opened, {broken.count('</div>')} closed")
print(f"Listings found anyway: {len(parse_page(broken))}")
print()
print("`html.parser` forgives broken HTML, like a browser does.")
print("That's why collection doesn't use an XML parser (`xml`, `lxml-xml`):")
print("it's strict and stops at the first error.")

table = []
for column in ["title", "price_raw", "rooms_raw", "area_raw", "district", "floor_raw"]:
    missing = int(raw_df[column].isna().sum())
    table.append({"column": column, "missing": missing,
                  "missing %": f"{missing / len(raw_df):.1%}",
                  "unique values": int(raw_df[column].nunique())})
show(pd.DataFrame(table))

print("The price column: how many different formats?")
for n in raw_df["price_raw"].dropna().unique()[:8]:
    print(f"  {n!r}")
print()
usd = raw_df["price_raw"].str.contains("у.е.", na=False).sum()
som = raw_df["price_raw"].str.contains("so'm", na=False).sum()
neg = raw_df["price_raw"].str.contains("Kelishilgan", na=False).sum()
print(f"  as у.е.        : {usd}")
print(f"  as so'm        : {som}")
print(f"  'Kelishilgan'  : {neg}   <- NO number, but the row is there")
print()
print("Three formats need three kinds of processing. That's Chapter 38's job.")

variants = [
    ("source_id (id the site gave)", ["source_id"]),
    ("title + district + area + price", ["title", "district", "area_raw", "price_raw"]),
    ("district + area + rooms", ["district", "area_raw", "rooms_raw"]),
]
results = []
for name, key in variants:
    kept = len(raw_df.drop_duplicates(subset=key))
    results.append({"duplicate key": name, "unique kept": kept,
                    "removed": len(raw_df) - kept})
show(pd.DataFrame(results))
print(f"Total records: {len(raw_df)}")

repeated = raw_df["area_raw"].value_counts()
extra = int((repeated[repeated > 1] - 1).sum())
print(f"Area values seen more than once: {int((repeated > 1).sum())}")
print(f"Actual number of extra records : {extra}")
print()
print("Area is a PHYSICAL property of the flat. If two listings have exactly")
print("the same area, it almost always means one flat was listed twice.")

import json
api = json.loads((FIX / "api.json").read_text(encoding="utf-8"))
print(f"API response keys: {list(api.keys())}")
print(f"Total records: {api['jami']}, page size: {api['sahifa_hajmi']}")
print()
print("First record:")
for k, v in api["natijalar"][0].items():
    print(f"  {k:<12} {v!r}  ({type(v).__name__})")
print()
api_df = pd.DataFrame(api["natijalar"])
print(f"DataFrame: {api_df.shape}")
print(f"Missing values: {int(api_df.isna().sum().sum())}")
print(f"Types are right: {dict(api_df.dtypes.astype(str))}")

show(pd.DataFrame([
    {"Aspect": "Price", "HTML scraping": "3 text formats", "API": "number, one unit"},
    {"Aspect": "Missing fields", "HTML scraping": "yes", "API": "no"},
    {"Aspect": "Duplicates", "HTML scraping": "yes", "API": "clean by id"},
    {"Aspect": "Site redesign", "HTML scraping": "code breaks", "API": "keeps working (versioned)"},
    {"Aspect": "Terms", "HTML scraping": "unclear", "API": "written down"},
    {"Aspect": "Speed", "HTML scraping": "slow (HTML is big)", "API": "fast (data only)"},
]))
print()
print("So the rule: if there's an API, you don't spend time writing a scraper.")

"""Fetching every record from a paginated API, respecting the limits."""
import json
import time
from pathlib import Path


def all_pages(fetch_page, page_size=10, max_pages=100, delay=1.0):
    """`fetch_page(offset, limit)` -> dict. Collects every record.

    max_pages is a safeguard: if the API keeps saying 'there's a next page'
    forever, the code must not loop forever.
    """
    everything, offset = [], 0
    for _ in range(max_pages):
        response = fetch_page(offset, page_size)
        chunk = response.get("natijalar", [])
        if not chunk:
            break
        everything += chunk
        offset += len(chunk)
        if offset >= response.get("jami", 0):
            break
        time.sleep(delay)
    return everything

from collect.api_client import all_pages

requests = {"count": 0}
def local_api(offset, limit):
    """Imitates a real API: no network, but with pagination."""
    requests["count"] += 1
    return {"jami": api["jami"], "natijalar": api["natijalar"][offset:offset + limit]}

fetched = all_pages(local_api, page_size=10, delay=0)
print(f"Requests       : {requests['count']}")
print(f"Records fetched: {len(fetched)}")
print(f"Total in API   : {api['jami']}")
print(f"Got everything : {len(fetched) == api['jami']}")

RAW = ISH / "raw"
RAW.mkdir(exist_ok=True)
path = RAW / "listings_raw.csv"
raw_df.to_csv(path, index=False)
print(f"Saved: ish_en/raw/{path.name}")
print(f"  {len(raw_df)} rows, {len(raw_df.columns)} columns")
print(f"  size: {path.stat().st_size} bytes")
print()
print("Note: this file is DELIBERATELY dirty. Cleaning is Chapter 38's job.")
print("Collecting and cleaning are separate steps: collecting takes the raw data, cleaning makes the decisions.")

show(pd.DataFrame([
    {"Step": "0. Check the rules", "Result": "robots.txt, ToS, decision on personal data"},
    {"Step": "1. Pick the source", "Result": "API > export > partnership > scraping"},
    {"Step": "2. Collect and keep the raw data", "Result": "cache/ folder, downloaded once"},
    {"Step": "3. Parse", "Result": "raw/listings_raw.csv, not cleaned"},
    {"Step": "4. Build the dataset", "Result": "Chapter 38: cleaning, dedup, split"},
]))

show(pd.DataFrame([
    {"Concept": "Rules before code", "In one sentence": "robots.txt, ToS, ask slowly, don't collect personal data"},
    {"Concept": "Source order", "In one sentence": "API > export > ask for a partnership > scraping"},
    {"Concept": "select_one returns None", "In one sentence": "Check every result, or collection stops"},
    {"Concept": "html.parser", "In one sentence": "Forgives broken HTML; no XML parser"},
    {"Concept": "Cache", "In one sentence": "Downloaded once: politeness and speed"},
    {"Concept": "Unique ids deceive", "In one sentence": "Measured: different id, same content"},
    {"Concept": "Personal data", "In one sentence": "Data never written can't be stolen"},
    {"Concept": "Keep the raw data", "In one sentence": "Re-collecting is expensive, disk is cheap"},
]))