Building a dataset: cleaning, labels and the split

import re
import sys
import numpy as np
import pandas as pd

# We reuse the collector we wrote in Chapter 37: the raw data comes from there.
# Every chapter has to work on its own, so we go back to the SOURCE, not to the file.
sys.path.insert(0, str(ISH))
from collect.parse import parse_page

FIX = DATA / "fixture"
records = []
for page in sorted(FIX.glob("elonlar_*.html")):
    records += parse_page(page.read_text(encoding="utf-8"))
raw = pd.DataFrame(records)

print(f"Raw data: {raw.shape[0]} rows, {raw.shape[1]} columns")
print()
print(raw.dtypes.to_string())
print()
print("Every column is `object`, meaning TEXT. Not a single number.")

class StepLog:
    """Records every cleaning step."""
    def __init__(self, initial):
        self.steps = [{"step": "start", "rows": initial, "lost": 0}]
    def record(self, name, rows):
        previous = self.steps[-1]["rows"]
        self.steps.append({"step": name, "rows": rows, "lost": previous - rows})
        return rows
    def table(self):
        t = pd.DataFrame(self.steps)
        t["kept %"] = (t["rows"] / t["rows"].iloc[0] * 100).round(1)
        return t

log = StepLog(len(raw))
print(f"Log started: {len(raw)} rows")

print("Raw price values (first 10 unique):")
for v in raw["price_raw"].dropna().unique()[:10]:
    print(f"  {v!r}")

"""Turning values written as text into numbers."""
import re

import numpy as np

SUM_RATE = 12500          # 1 USD = 12500 so'm (the rate when the dataset was made)


def parse_price(text, rate=SUM_RATE):
    """Converts a price string to USD.

    Three formats are supported:
      "500 у.е."        -> 500.0
      "6 250 000 so'm"  -> 500.0
      "Kelishilgan"     -> nan   (NO number, nothing is made up)
    """
    if not isinstance(text, str):
        return np.nan
    if "Kelishilgan" in text:
        return np.nan                      # no price, we don't guess
    digits = re.sub(r"[^\d]", "", text.replace("\xa0", " "))
    if not digits:
        return np.nan
    value = int(digits)
    return value / rate if "so'm" in text else float(value)


def first_number(text, pattern=r"([\d.]+)"):
    """Extracts the first number from a string: "45.5 m²" -> 45.5, "11-qavat" -> 11.0"""
    if not isinstance(text, str):
        return np.nan
    found = re.search(pattern, text)
    return float(found.group(1)) if found else np.nan

from cleaning.text_to_number import parse_price, first_number

tests = ["500\xa0у.е.", "6 250 000 so'm", "Kelishilgan", None, "12 500 000 so'm"]
print(f"{'input':<22} {'result (USD)':>14}")
print("-" * 38)
for s in tests:
    print(f"{repr(s):<22} {parse_price(s)!s:>14}")
print()
INTEGER = r"(\d+)"
area_test = first_number("45.5 m²")
floor_test = first_number("11-qavat", INTEGER)
print(f"first_number('45.5 m²')  = {area_test}")
print(f"first_number('11-qavat') = {floor_test}")

clean = pd.DataFrame({
    "source_id": raw["source_id"],
    "district": raw["district"],
    "rooms": raw["rooms_raw"].apply(lambda s: first_number(s, r"(\d+)")),
    "area_m2": raw["area_raw"].apply(first_number),
    "floor": raw["floor_raw"].apply(lambda s: first_number(s, r"(\d+)")),
    "price_usd": raw["price_raw"].apply(parse_price),
})
print("Missing values after conversion:")
for c in ["district", "rooms", "area_m2", "floor", "price_usd"]:
    n = int(clean[c].isna().sum())
    print(f"  {c:<12} {n:>4} missing  ({n / len(clean):.1%})")

no_price = int(clean["price_usd"].isna().sum())
clean_t = clean[clean["price_usd"].notna()].copy()
log.record("rows without a price removed", len(clean_t))
print(f"No price ('Kelishilgan' and empty): {no_price} rows")
print(f"Kept: {len(clean_t)} rows")
print()
print("These rows are NOT useless: they're used for prediction.")
print("They're just no good for TRAINING. So they can be stored separately.")

before = len(clean_t)
clean_t = clean_t.dropna(subset=["rooms", "area_m2", "floor"])
log.record("rows with incomplete features", len(clean_t))
print(f"Rows missing a feature: {before - len(clean_t)}")
print(f"Kept: {len(clean_t)} rows")

"""Dataset contract: type and range for every column."""

CONTRACT = {
    "district":  {"type": "text", "allowed": {"Yunusobod", "Chilonzor", "Mirzo Ulug'bek",
                                              "Shayxontohur", "Yakkasaroy", "Olmazor",
                                              "Sergeli", "Bektemir"}},
    "rooms":     {"type": "number", "min": 1, "max": 10},
    "area_m2":   {"type": "number", "min": 15, "max": 400},
    "floor":     {"type": "number", "min": 1, "max": 50},
    "price_usd": {"type": "number", "min": 50, "max": 20000},
}


def check(df, contract=CONTRACT):
    """Returns a list of contract violations. An empty list means all is well."""
    problems = []
    for column, rule in contract.items():
        if column not in df.columns:
            problems.append(f"{column}: column MISSING")
            continue
        value = df[column]
        if rule["type"] == "number":
            out_of_range = ((value < rule["min"]) | (value > rule["max"])).sum()
            if out_of_range:
                problems.append(
                    f"{column}: {out_of_range} values outside [{rule['min']}, {rule['max']}]")
        else:
            wrong = set(value.dropna().unique()) - rule["allowed"]
            if wrong:
                problems.append(f"{column}: value not allowed {sorted(wrong)[:5]}")
        if value.isna().any():
            problems.append(f"{column}: {int(value.isna().sum())} missing values")
    return problems

from cleaning.contract import check

problems = check(clean_t)
print(f"Contract check: {len(problems)} problems")
for m in problems:
    print(f"  - {m}")
if not problems:
    print("  (everything matches the contract)")
print()
print("The contract is an EXPECTATION written in code. When it breaks you know at once,")
print("instead of thinking 'my model is weird' 3 days later.")

clean_t["group"] = clean_t.groupby(["district", "area_m2", "rooms"]).ngroup()
unique_flats = clean_t["group"].nunique()
print(f"Rows                : {len(clean_t)}")
print(f"Unique flats (group): {unique_flats}")
print(f"Extra records       : {len(clean_t) - unique_flats}  ({(len(clean_t) - unique_flats) / len(clean_t):.1%})")
print()
largest = clean_t["group"].value_counts()
print(f"The most repeated flat was listed {int(largest.iloc[0])} times:")
show(clean_t[clean_t["group"] == largest.index[0]][["source_id", "district", "area_m2", "rooms", "price_usd"]])

from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.model_selection import train_test_split, GroupShuffleSplit
from sklearn.metrics import r2_score

X = pd.get_dummies(clean_t[["district", "rooms", "area_m2", "floor"]], columns=["district"])
y = clean_t["price_usd"].values
groups = clean_t["group"].values

result = []
for name, make in [("KNN (k=1)", lambda: KNeighborsRegressor(1)),
                   ("KNN (k=3)", lambda: KNeighborsRegressor(3)),
                   ("random forest", lambda: RandomForestRegressor(200, random_state=0))]:
    random_r2 = [r2_score(yb, make().fit(Xa, ya).predict(Xb))
                 for Xa, Xb, ya, yb in (train_test_split(X, y, test_size=.25, random_state=s)
                                        for s in range(5))]
    grouped = []
    for s in range(5):
        tr, te = next(GroupShuffleSplit(n_splits=1, test_size=.25,
                                        random_state=s).split(X, y, groups=groups))
        grouped.append(r2_score(y[te], make().fit(X.iloc[tr], y[tr]).predict(X.iloc[te])))
    result.append({"model": name,
                   "random split R2": round(float(np.mean(random_r2)), 3),
                   "group split R2": round(float(np.mean(grouped)), 3),
                   "fake gap": round(float(np.mean(random_r2) - np.mean(grouped)), 3)})
show(pd.DataFrame(result))

# Stage 1: split off the test set BY GROUP
tr_idx, te_idx = next(GroupShuffleSplit(n_splits=1, test_size=0.2,
                                        random_state=42).split(X, y, groups=groups))
rest = clean_t.iloc[tr_idx]
test = clean_t.iloc[te_idx]

# Stage 2: from the rest, split off validation, also BY GROUP
X2 = X.iloc[tr_idx]
tr2, va2 = next(GroupShuffleSplit(n_splits=1, test_size=0.2,
                                  random_state=42).split(X2, y[tr_idx],
                                                         groups=rest["group"].values))
train = rest.iloc[tr2]
val = rest.iloc[va2]

print(f"{'part':<8} {'rows':>7} {'unique flats':>13} {'share':>8}")
print("-" * 39)
for name, b in [("train", train), ("val", val), ("test", test)]:
    print(f"{name:<8} {len(b):>7} {b['group'].nunique():>13} {len(b) / len(clean_t):>7.1%}")
print()
overlap = (set(train["group"]) & set(test["group"])) | (set(train["group"]) & set(val["group"]))
print(f"Flats shared between parts: {len(overlap)}   <- MUST be 0")
log.record("split into three parts", len(train) + len(val) + len(test))

show(log.table())
print(f"Started: {log.steps[0]['rows']} rows")
print(f"Kept   : {log.steps[-1]['rows']} rows")
print(f"Lost   : {log.steps[0]['rows'] - log.steps[-1]['rows']} rows, each with its reason above")

datasheet = f"""# listings dataset: datasheet

## What
Tashkent rental listings. One row = one listing.

## Source
`datasets/fixture/elonlar_*.html`: SYNTHETIC saved pages
(`datasets/fixture_qur.py`, fixed seed). NOT a live site.
That's why the numbers come out the same every time.

## Size
- raw records: {len(raw)}
- after cleaning: {len(clean_t)}
- unique flats: {clean_t['group'].nunique()}
- train / val / test: {len(train)} / {len(val)} / {len(test)}

## Target
`price_usd`: monthly rent in US dollars.
Rate: 1 USD = 12500 so'm (listings priced in so'm were converted at this rate).

## IMPORTANT warning
In the generator the price was chosen INDEPENDENTLY of the features. So there is
NO real pattern to learn in this dataset. It is only for teaching:
cleaning, duplicates, splitting by group and measuring leakage.
Trying to get a good R2 on this dataset is pointless.

## Removed
- rows without a price ("Kelishilgan"): no good for training, the target is not made up
- rows missing a feature

## Personal data
Phone numbers were NOT COLLECTED (`collect/rules.py` -> `drop_personal`).
The dataset has no personal fields of any kind.

## Split rule
GroupShuffleSplit, group = (district, area_m2, rooms).
Each flat lands in exactly one part. A random split is NOT USED.
"""
path = ISH / "dataset"
path.mkdir(exist_ok=True)
(path / "DATASHEET.md").write_text(datasheet, encoding="utf-8")
print(datasheet)

for name, b in [("train", train), ("val", val), ("test", test)]:
    b.to_csv(path / f"{name}.csv", index=False)
no_price_df = clean[clean["price_usd"].isna()]
no_price_df.to_csv(path / "no_price_for_prediction.csv", index=False)

print("Saved to the ish_en/dataset/ folder:")
for p in sorted(path.iterdir()):
    print(f"  {p.name:<32} {p.stat().st_size:>7} bytes")
print()
print("Now this folder can be handed to someone else: data + documentation + split rule.")

show(pd.DataFrame([
    {"Step": "1. Work out column types", "In this chapter": "text -> number, three price formats"},
    {"Step": "2. Count missing values", "In this chapter": "with a log, at every step"},
    {"Step": "3. Define the target", "In this chapter": "price_usd; the target is not made up"},
    {"Step": "4. Split", "In this chapter": "GroupShuffleSplit: leakage measured"},
    {"Step": "5. Build a baseline", "Next": "Chapter 39: fine-tuning; Chapter 7: the baseline rule"},
]))

show(pd.DataFrame([
    {"Concept": "Step log", "In one sentence": "Rows are counted after every operation: nothing vanishes silently"},
    {"Concept": "The target is not made up", "In one sentence": "Features are filled in, the target isn't"},
    {"Concept": "Data contract", "In one sentence": "Type and range as an expectation written in code"},
    {"Concept": "Group identifier", "In one sentence": "One flat = one group, built from stable fields"},
    {"Concept": "GroupShuffleSplit", "In one sentence": "Measured: a random split inflates R2"},
    {"Concept": "Negative R2", "In one sentence": "Worse than the average: below Chapter 7's baseline"},
    {"Concept": "Datasheet", "In one sentence": "The dataset's passport: 10 minutes now, a year saved later"},
    {"Concept": "Rows without a price", "In one sentence": "No good for training, good for prediction"},
]))