A/B test: sababni tajriba bilan topish

ab = pd.read_csv(CSV.parent / "ab_test.csv")
print(ab.shape); show(ab.head(3))
r = ab.groupby("variant").converted.agg(n="count", konversiya="mean", sotib_oldi="sum")
r["konversiya_%"] = (100 * r.konversiya).round(2)
show(r[["n", "sotib_oldi", "konversiya_%"]])
lift = r.konversiya["B"] / r.konversiya["A"] - 1
print(f"farq: {100*(r.konversiya['B'] - r.konversiya['A']):+.2f} punkt | nisbiy o'sish (lift): {100*lift:+.1f}%")

from scipy import stats
def nisbat_testi(k1, n1, k2, n2):
    p1, p2, p = k1 / n1, k2 / n2, (k1 + k2) / (n1 + n2)
    z = (p2 - p1) / np.sqrt(p * (1 - p) * (1 / n1 + 1 / n2))
    se = np.sqrt(p1 * (1 - p1) / n1 + p2 * (1 - p2) / n2)
    return z, 2 * (1 - stats.norm.cdf(abs(z))), (p2 - p1 - 1.96 * se, p2 - p1 + 1.96 * se)

z, p, ci = nisbat_testi(r.sotib_oldi["A"], r.n["A"], r.sotib_oldi["B"], r.n["B"])
print(f"z = {z:.2f} | p = {p:.3f} | farq 95% IO: [{100*ci[0]:+.2f}, {100*ci[1]:+.2f}] punkt")

rng = np.random.default_rng(1)
a = ab[ab.variant == "A"].converted.values; b = ab[ab.variant == "B"].converted.values
farqlar = np.array([rng.choice(b, len(b)).mean() - rng.choice(a, len(a)).mean() for _ in range(3000)])
print(f"bootstrap 95% IO: [{100*np.percentile(farqlar, 2.5):+.2f}, {100*np.percentile(farqlar, 97.5):+.2f}] punkt | farq > 0 bo'lgan ulush: {100*(farqlar > 0).mean():.0f}%")

def namuna_hajmi(p0, mde, alpha=0.05, power=0.8):
    p1 = p0 + mde; z_a, z_b = stats.norm.ppf(1 - alpha / 2), stats.norm.ppf(power)
    pbar = (p0 + p1) / 2
    n = (z_a * np.sqrt(2 * pbar * (1 - pbar)) + z_b * np.sqrt(p0 * (1 - p0) + p1 * (1 - p1))) ** 2 / mde ** 2
    return int(np.ceil(n))
jadval = pd.DataFrame({"MDE_punkt": [0.25, 0.5, 1.0, 2.0]})
jadval["nisbiy_%"] = (100 * jadval.MDE_punkt / 4).round(0)
jadval["n_har_variant"] = [namuna_hajmi(0.04, m / 100) for m in jadval.MDE_punkt]
jadval["kunlar (1000 user/kun)"] = (2 * jadval.n_har_variant / 1000).round(0)
show(jadval)

rng = np.random.default_rng(3)
def sim(n, p0, p1, takror=1000):
    ka = rng.binomial(n, p0, takror); kb = rng.binomial(n, p1, takror)
    return np.mean([nisbat_testi(x, n, y, n)[1] < 0.05 for x, y in zip(ka, kb)])
for n in [9000, 24000, 50000]:
    print(f"n={n:6} har variantda: haqiqiy +0.5 punkt farqni sezish ehtimoli {100*sim(n, 0.04, 0.045):.0f}%")
print(f"n=9000, haqiqiy farq YO'Q: soxta 'sezilarli' chiqish ehtimoli {100*sim(9000, 0.04, 0.04):.1f}%  (bu alpha)")

seg = ab.groupby(["device", "variant"]).converted.agg(["count", "sum", "mean"]).unstack()
out = pd.DataFrame({"n_A": seg[("count", "A")], "n_B": seg[("count", "B")], "A_%": (100 * seg[("mean", "A")]).round(2), "B_%": (100 * seg[("mean", "B")]).round(2)})
out["p"] = [round(nisbat_testi(seg.loc[i, ("sum", "A")], seg.loc[i, ("count", "A")], seg.loc[i, ("sum", "B")], seg.loc[i, ("count", "B")])[1], 3) for i in out.index]
show(out)

rng = np.random.default_rng(11)
pv = []
for _ in range(2000):
    x = rng.permutation(ab.converted.values)                # variantlarni tasodifan aralashtiramiz: A/A
    ka, kb = x[:9000].sum(), x[9000:].sum()
    pv.append(nisbat_testi(ka, 9000, kb, 9000)[1])
pv = np.array(pv)
print(f"A/A: p < 0.05 chiqqan ulush {100*(pv < 0.05).mean():.1f}% | p < 0.01: {100*(pv < 0.01).mean():.1f}% | eng kichik p: {pv.min():.4f}")
f, ax = plt.subplots(figsize=(7, 3))
ax.hist(pv, bins=20, color="#4C72B0"); ax.set_title("A/A test: p-qiymat tekis taqsimlanadi, 5% i 0.05 dan kichik"); ax.set_xlabel("p-qiymat")
fig("aa.png", "Hech qanday farq yo'q, lekin 2000 ta A/A testdan 5% i 'sezilarli'. Har 20 ta testdan biri soxta. Shuning uchun bitta test bitta gipoteza, va natija takrorlanishi kerak.")

print("SRM tekshiruvi: chisquare([9000, 9000]) p =", round(stats.chisquare(r.n.values).pvalue, 3), "| takror user_id:", ab.user_id.duplicated().sum())

A/B: "Sotib olish" tugmasi (B: yashil, katta)      Muddat: 14 kun      n: 9 000 / 9 000
Asosiy metrika: sotib olish konversiyasi. Qarshi metrika: bekor foizi (o'zgarmadi).
Natija: A 4.11%, B 4.62%, farq +0.51 p.p. (95% IO: -0.09 .. +1.11), p = 0.09.
Qaror: chiqarmaymiz, testni davom ettiramiz: +0.5 p.p. ni sezish uchun 25 500 / variant kerak (yana ~33 kun).
Segment: web da +1.1 p.p., p = 0.16 (n kichik, gipoteza sifatida yoziladi, qaror emas).
Xavf: SRM yo'q, takror user yo'q, hafta sikli to'liq.