Visualisation: looking at the data

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_theme(style="whitegrid", font_scale=1.05)
df = pd.read_csv(DATA / "uy_ijara.csv")

# The MATPLOTLIB way: group by hand, place by hand
figure, axes = plt.subplots(1, 2, figsize=(14, 4.8))

g = df.groupby("tuman").narx_usd.mean().sort_values()
axes[0].barh(range(len(g)), g.values, color="#5f6368")
axes[0].set_yticks(range(len(g)))
axes[0].set_yticklabels(g.index)
axes[0].set_xlabel("Average price (USD)")
axes[0].set_title("matplotlib: 6 lines of code", fontweight="bold")

# The SEABORN way: one line
sns.barplot(data=df, y="tuman", x="narx_usd", estimator="mean",
            errorbar=None, ax=axes[1], order=g.index, color="#1a73e8")
axes[1].set_xlabel("Average price (USD)")
axes[1].set_title("seaborn: 1 line of code", fontweight="bold")

fig("matplotlib_vs_seaborn.png", caption="The same result. On the left we group by hand, on the right seaborn groups for us.")

figure, axes = plt.subplots(1, 3, figsize=(15, 4.2))

sns.histplot(data=df, x="narx_usd", bins=35, kde=True, ax=axes[0], color="#1a73e8")
axes[0].set_title("histplot: shape")

sns.boxplot(data=df, x="narx_usd", ax=axes[1], color="#43a047")
axes[1].set_title("boxplot: median, quartiles, outliers")

sns.violinplot(data=df, x="narx_usd", ax=axes[2], color="#8e24aa")
axes[2].set_title("violinplot: shape + quartiles")

fig("taqsimot.png", caption="One column, three views. histplot shows the shape, boxplot shows the extreme values.")

print(df.narx_usd.describe().round(1).to_string())
print()
q1, q3 = df.narx_usd.quantile([0.25, 0.75])
iqr = q3 - q1
outliers = df[(df.narx_usd < q1 - 1.5 * iqr) | (df.narx_usd > q3 + 1.5 * iqr)]
print(f"Interquartile range (IQR): {iqr:.0f}")
print(f"Rows the boxplot marks as outliers: {len(outliers)} ({len(outliers)/len(df):.1%})")

figure, axes = plt.subplots(1, 2, figsize=(14, 4.8))

sns.scatterplot(data=df, x="maydon_m2", y="narx_usd", hue="mebel",
                alpha=.55, ax=axes[0], palette=["#c5221f", "#137333"])
axes[0].set_title("Area and price (colour = furnished)")

sns.scatterplot(data=df, x="metro_km", y="narx_usd", alpha=.45, ax=axes[1], color="#1a73e8")
sns.regplot(data=df, x="metro_km", y="narx_usd", scatter=False, ax=axes[1], color="#c5221f")
axes[1].set_title("Distance to metro and price (red line = trend)")

fig("boglanish.png", caption="Left: price rises with area. Right: price falls further from the metro, but the spread is wide.")

num_cols = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "narx_usd"]
corr = df[num_cols].corr()
print("Correlation with narx_usd:")
print(corr["narx_usd"].drop("narx_usd").sort_values(ascending=False).round(3).to_string())

figure, ax = plt.subplots(figsize=(8.5, 6.5))
sns.heatmap(corr, annot=True, fmt=".2f", cmap="RdBu_r", center=0,
            square=True, linewidths=.5, ax=ax, cbar_kws={"shrink": .8})
ax.set_title("Correlation matrix", fontweight="bold")
fig("heatmap.png", caption="Red = positive relationship, blue = negative. Note the 0.94 between xonalar and maydon_m2.")

figure, axes = plt.subplots(1, 3, figsize=(16, 4.5))

sns.countplot(data=df, y="tuman", ax=axes[0], color="#5f6368",
              order=df.tuman.value_counts().index)
axes[0].set_title("countplot: number of listings")

sns.boxplot(data=df, x="xonalar", y="narx_usd", ax=axes[1], hue="xonalar",
            legend=False, palette="Blues")
axes[1].set_title("boxplot: rooms and price")

sns.barplot(data=df, x="xonalar", y="ijaraga_berildi_30k", ax=axes[2],
            errorbar=("ci", 95), color="#137333")
axes[2].set_ylabel("Share rented fast")
axes[2].set_title("barplot: with a confidence interval")

fig("kategoriya.png", caption="countplot shows counts, boxplot the distribution, barplot the mean (with a confidence interval).")

print("Share rented fast, by number of rooms:")
g = df.groupby("xonalar").agg(n=("ijaraga_berildi_30k", "size"),
                              fast_share=("ijaraga_berildi_30k", "mean"))
g["fast_share"] = g.fast_share.round(3)
print(g.to_string())
print()
print("NOTE: there are few 5-room flats, so their share is not trustworthy.")

subset = df[df.xonalar.isin([1, 2, 3])]
g = sns.relplot(data=subset, x="maydon_m2", y="narx_usd",
                hue="mebel", col="xonalar", kind="scatter",
                height=3.8, aspect=1.05, alpha=.6,
                palette=["#c5221f", "#137333"])
g.figure.suptitle("Area and price, split by number of rooms", y=1.04, fontweight="bold")
fig("relplot.png", caption="col='xonalar' created three panels. hue='mebel' added a fourth dimension as colour.")

show(pd.DataFrame([
    {"Question": "How is one numeric column distributed?", "Chart": "histplot", "Extra": "kde=True smooths the shape"},
    {"Question": "Are there outliers?", "Chart": "boxplot", "Extra": "Dots = outside the IQR"},
    {"Question": "Is there a link between two numbers?", "Chart": "scatterplot", "Extra": "regplot adds a trend line"},
    {"Question": "All numeric columns against each other?", "Chart": "heatmap(corr)", "Extra": "or pairplot (slow)"},
    {"Question": "How many rows per category?", "Chart": "countplot", "Extra": "sort with order="},
    {"Question": "The mean per category?", "Chart": "barplot", "Extra": "DON'T turn off the errorbar"},
    {"Question": "The distribution per category?", "Chart": "boxplot / violinplot", "Extra": "tells you more than a barplot"},
    {"Question": "Change over time?", "Chart": "lineplot", "Extra": "the x axis should be a date"},
    {"Question": "Three or four dimensions together?", "Chart": "relplot / catplot", "Extra": "hue, col, row"},
]))

figure, axes = plt.subplots(1, 4, figsize=(17, 4))
for ax, c in zip(axes, ["maydon_m2", "metro_km", "narx_usd", "qavat"]):
    sns.boxplot(data=df, x="ijaraga_berildi_30k", y=c, ax=ax,
                hue="ijaraga_berildi_30k", legend=False, palette=["#c5221f", "#137333"])
    ax.set_xlabel("0 = went slow   1 = went fast")
    ax.set_title(c)
fig("target_ajratish.png", caption="The further apart the boxes are, the more useful that feature is.")

print("How strongly each feature separates the target:")
rows = []
for c in ["maydon_m2", "metro_km", "narx_usd", "qavat", "xonalar", "mebel"]:
    a = df[df.ijaraga_berildi_30k == 0][c].mean()
    b = df[df.ijaraga_berildi_30k == 1][c].mean()
    pooled_std = df[c].std()
    rows.append({"Column": c, "Slow mean": round(a, 2), "Fast mean": round(b, 2),
                 "Separation": round(abs(b - a) / pooled_std, 3)})
show(pd.DataFrame(rows).sort_values("Separation", ascending=False))

sns.set_theme(style="whitegrid", font_scale=1.1)
figure, ax = plt.subplots(figsize=(10, 5))
g = df.groupby("tuman").agg(avg=("narx_usd", "mean"),
                            n=("narx_usd", "size")).sort_values("avg")
sns.barplot(x=g.avg, y=g.index, ax=ax, color="#1a73e8")
for i, (v, n) in enumerate(zip(g.avg, g.n)):
    ax.text(v + 8, i, f"{v:.0f} USD  (n={n})", va="center", fontsize=9.5, color="#333")
ax.set_xlabel("Average rent, USD/month")
ax.set_ylabel("")
ax.set_title("Average rent by Tashkent district", fontweight="bold", pad=14)
ax.set_xlim(0, g.avg.max() * 1.28)
sns.despine(left=True, bottom=True)
fig("hisobot.png", caption="A report-ready chart: the values are written, the group size (n) is shown, the clutter is removed.")

show(pd.DataFrame([
    {"Rule": "Look first, build the model later", "Why": "A chart tells you more in a second than a table"},
    {"Rule": "seaborn speaks DataFrame", "Why": "You give a column name; it groups and draws"},
    {"Rule": "Check the chart WITH A NUMBER", "Why": "Eyes deceive; describe() and corr() confirm"},
    {"Rule": "Show the group size", "Why": "The mean of a 3-row group means nothing"},
    {"Rule": "Don't turn off the errorbar", "Why": "Overlapping intervals = the difference isn't reliable"},
    {"Rule": "Read an outlier before dropping it", "Why": "Not all of them are mistakes; some are real"},
    {"Rule": "Split by the target", "Why": "It shows in advance which features are useful"},
]))