Neural network basics

import numpy as np
import pandas as pd

def sigmoid(z):
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

x = np.array([2.0, 48.0, 1.5, 1.0])      # one flat
w = np.array([0.3, 0.02, -0.4, 0.6])     # weights
b = -1.2                                  # bias

z = x @ w + b                              # the operation from Chapter 12
output = sigmoid(z)                        # the function from Chapter 16

print(f"z = x @ w + b = {z:.4f}")
print(f"sigmoid(z)    = {output:.4f}")
print()
print("This is ONE NEURON. And it is logistic regression.")

X_xor = np.array([[0., 0.], [0., 1.], [1., 0.], [1., 1.]])
y_xor = np.array([0, 1, 1, 0])            # 0 if they're the same, 1 if they differ

print("The XOR table:")
for xi, yi in zip(X_xor, y_xor):
    print(f"  {xi[0]:.0f} XOR {xi[1]:.0f} = {yi}")
print()
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression().fit(X_xor, y_xor)
print(f"Logistic regression accuracy: {lr.score(X_xor, y_xor):.4f}")
print("  predictions:", lr.predict(X_xor))
print()
print("It can't get even two of the four points right.")
print("Because XOR can't be separated with ONE straight line.")

# a two-layer network built BY HAND: the weights are chosen, not trained
W1 = np.array([[1.0, 1.0],      # neuron 1: like OR
               [1.0, 1.0]])     # neuron 2: like AND
b1 = np.array([-0.5, -1.5])
W2 = np.array([1.0, -2.0])      # OR minus AND -> XOR
b2 = -0.2

hidden = sigmoid(X_xor @ W1 + b1)
output = sigmoid(hidden @ W2 + b2)

print("Hidden layer output (2 neurons):")
print(np.round(hidden, 3))
print()
print("Final output vs the truth:")
for i in range(4):
    print(f"  {X_xor[i]} -> {output[i]:.4f}  (truth {y_xor[i]})")
print()
print(f"Accuracy: {((output > 0.5).astype(int) == y_xor).mean():.4f}")

# with NO activation: just matrix multiplication
y1 = X_xor @ W1 + b1
y2 = y1 @ W2 + b2
print("With NO activation:")
print("  output:", np.round(y2, 4))
print()
print("Mathematical proof: (X @ W1 + b1) @ W2 + b2 = X @ (W1 @ W2) + (b1 @ W2 + b2)")
W_combined = W1 @ W2
print(f"  W1 @ W2 = {np.round(W_combined, 4)}  -> ONE linear layer")
print()
print("So without activation, 100 layers equal ONE layer.")

import matplotlib.pyplot as plt

z = np.linspace(-5, 5, 300)
functions = {
    "sigmoid": (sigmoid(z), "0..1, old, the gradient vanishes"),
    "tanh": (np.tanh(z), "-1..1, centred on zero"),
    "ReLU": (np.maximum(0, z), "max(0,z), THE MOST USED"),
    "LeakyReLU": (np.where(z > 0, z, 0.1 * z), "ReLU, but with a gradient on the negative side too"),
}
figure, axes = plt.subplots(1, 4, figsize=(16, 3.4))
for ax, (name, (value, note)) in zip(axes, functions.items()):
    ax.plot(z, value, lw=2.3, color="#1a73e8")
    ax.axhline(0, color="k", lw=.7); ax.axvline(0, color="k", lw=.7)
    ax.set_title(name, fontweight="bold")
    ax.grid(alpha=.3)
fig("activation.png", caption="Four activation functions. ReLU is the simplest and the most widely used.")

show(pd.DataFrame([{"Function": k, "Note": v[1]} for k, v in functions.items()]))

print("Comparing the derivatives:")
print(f"{'z':>6} {'sigmoid derivative':>18} {'ReLU derivative':>15}")
print("-" * 42)
for zi in [-8, -3, 0, 3, 8]:
    s = sigmoid(zi)
    print(f"{zi:>6} {s*(1-s):>18.6f} {float(zi > 0):>15.1f}")
print()
print("At z=8 the sigmoid derivative is 0.0003: almost NO gradient.")
print("With ReLU it's 1.0: the gradient passes through in full.")

def train_network(X, y, hidden=4, alpha=0.5, steps=4000, seed=42):
    r = np.random.default_rng(seed)
    n_in = X.shape[1]
    W1 = r.normal(0, 1, (n_in, hidden))
    b1 = np.zeros(hidden)
    W2 = r.normal(0, 1, hidden)
    b2 = 0.0
    history = []
    for step in range(steps):
        # FORWARD
        z1 = X @ W1 + b1
        a1 = np.maximum(0, z1)              # ReLU
        z2 = a1 @ W2 + b2
        a2 = sigmoid(z2)
        # the loss (log loss, Chapter 16)
        loss = -np.mean(y * np.log(a2 + 1e-12) + (1 - y) * np.log(1 - a2 + 1e-12))
        history.append(loss)
        # BACKWARD: the chain rule
        dz2 = (a2 - y) / len(y)             # this is sigmoid + log loss combined
        dW2 = a1.T @ dz2
        db2 = dz2.sum()
        da1 = np.outer(dz2, W2)
        dz1 = da1 * (z1 > 0)                # the ReLU derivative
        dW1 = X.T @ dz1
        db1 = dz1.sum(axis=0)
        # update (Chapter 13)
        W2 -= alpha * dW2; b2 -= alpha * db2
        W1 -= alpha * dW1; b1 -= alpha * db1
    return (W1, b1, W2, b2), history

params, history = train_network(X_xor, y_xor.astype(float))
W1o, b1o, W2o, b2o = params
pred = sigmoid(np.maximum(0, X_xor @ W1o + b1o) @ W2o + b2o)

print(f"Starting loss : {history[0]:.4f}")
print(f"Final loss    : {history[-1]:.6f}")
print()
for i in range(4):
    print(f"  {X_xor[i]} -> {pred[i]:.4f}  (truth {y_xor[i]})")
print()
print(f"Accuracy: {((pred > 0.5).astype(int) == y_xor).mean():.4f}")

show(pd.DataFrame([
    {"Step": "z = X @ W + b", "From which chapter": "Chapter 12 (the dot product)"},
    {"Step": "sigmoid(z)", "From which chapter": "Chapter 16 (logistic regression)"},
    {"Step": "log loss", "From which chapter": "Chapter 16"},
    {"Step": "computing the gradient", "From which chapter": "Chapter 13"},
    {"Step": "W -= alpha * dW", "From which chapter": "Chapter 13 (gradient descent)"},
    {"Step": "the chain rule (backward)", "From which chapter": "NEW: Chapter 31"},
]))

from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier(hidden_layer_sizes=(16,), activation="relu",
                    max_iter=5000, random_state=42).fit(X_xor, y_xor)
print(f"MLPClassifier (16 neurons): {mlp.score(X_xor, y_xor):.4f}")
print(f"Our hand-written version  : {((pred > 0.5).astype(int) == y_xor).mean():.4f}")

import warnings
print("The same problem, the same model: only the seed differs:")
print(f"{'setting':<40} {'mean':>9} {'results'}")
print("-" * 72)
for name, kw in [
    ("(4,) neurons, learning_rate=0.1", dict(hidden_layer_sizes=(4,), learning_rate_init=0.1)),
    ("(16,) neurons, default settings", dict(hidden_layer_sizes=(16,))),
]:
    scores = []
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        for s in range(8):
            m = MLPClassifier(max_iter=5000, random_state=s, **kw).fit(X_xor, y_xor)
            scores.append(m.score(X_xor, y_xor))
    print(f"{name:<40} {np.mean(scores):>9.3f}   {sorted(set(scores))}")

from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
import time

df = pd.read_csv(DATA / "uy_ijara.csv")
num = ["xonalar", "maydon_m2", "qavat", "qavatlar_soni", "metro_km", "mebel", "qurilgan_yil"]
cat = ["tuman"]
X, y = df[num + cat], df["ijaraga_berildi_30k"]

def pipe(model):
    return Pipeline([
        ("t", ColumnTransformer([
            ("num", Pipeline([("i", SimpleImputer(strategy="median")), ("sc", StandardScaler())]), num),
            ("cat", OneHotEncoder(handle_unknown="ignore"), cat),
        ])),
        ("m", model),
    ])

print(f"{'model':<40} {'CV F1':>9} {'±':>8} {'time':>8}")
print("-" * 68)
for name, m in [
    ("LogisticRegression (Chapter 16)", LogisticRegression(max_iter=3000)),
    ("MLP (1 layer, 16 neurons)", MLPClassifier((16,), max_iter=1500, random_state=42)),
    ("MLP (2 layers, 32-16)", MLPClassifier((32, 16), max_iter=1500, random_state=42)),
    ("RandomForest (Chapter 19)", RandomForestClassifier(n_estimators=200, random_state=42)),
]:
    t0 = time.perf_counter()
    b = cross_val_score(pipe(m), X, y, cv=5, scoring="f1")
    dt = time.perf_counter() - t0
    print(f"{name:<40} {b.mean():>9.4f} {b.std():>8.4f} {dt:>7.1f}s")

show(pd.DataFrame([
    {"Decision": "Number of layers", "Practical answer": "1-3 (tabular), 10+ for images",
     "Note": "Depth grows with the amount of data"},
    {"Decision": "Number of neurons", "Practical answer": "Between the input and the output, a power of 2",
     "Note": "32, 64, 128: no exact rule, you try"},
    {"Decision": "Activation (hidden)", "Practical answer": "ReLU",
     "Note": "The default choice, for the reason above"},
    {"Decision": "Activation (output)", "Practical answer": "binary: sigmoid, multiclass: softmax, regression: none",
     "Note": "Fixed by the task"},
    {"Decision": "Loss function", "Practical answer": "binary: log loss, multiclass: cross-entropy, regression: MSE",
     "Note": "Paired with the activation"},
]))

print("How the number of neurons affects the result:")
print(f"{'architecture':>18} {'CV F1':>9} {'±':>8} {'parameters':>13}")
print("-" * 52)
for hid in [(4,), (16,), (64,), (16, 8), (64, 32)]:
    m = MLPClassifier(hid, max_iter=1200, random_state=42)
    b = cross_val_score(pipe(m), X, y, cv=5, scoring="f1")
    mm = pipe(m).fit(X, y)
    n_param = sum(c.size for c in mm.named_steps["m"].coefs_) + \
              sum(ii.size for ii in mm.named_steps["m"].intercepts_)
    print(f"{str(hid):>18} {b.mean():>9.4f} {b.std():>8.4f} {n_param:>13}")

show(pd.DataFrame([
    {"Concept": "A neuron", "In one sentence": "sigmoid(x @ w + b): that's logistic regression"},
    {"Concept": "Why layers are needed", "In one sentence": "One neuron only draws a straight line (XOR)"},
    {"Concept": "Activation", "In one sentence": "The ONLY reason depth works: without it the layers collapse"},
    {"Concept": "ReLU", "In one sentence": "The default choice for hidden layers"},
    {"Concept": "The sigmoid problem", "In one sentence": "Where it's flat, the gradient is near zero"},
    {"Concept": "Inside the network", "In one sentence": "The operations from Chapters 12, 13, 16 + the chain rule"},
    {"Concept": "Tabular data", "In one sentence": "A neural network does NOT win: start with boosting"},
]))