Spark on Kubernetes

diagramma(
    {
        "spark-submit": (0, 1.8, 2.2, 1.0, "manba"),
        "Kubernetes API\n(scheduler)": (3.0, 1.8, 2.4, 1.0, "qayta"),
        "Driver POD\n(bosh oshpaz)": (6.2, 3.2, 2.3, 1.0, "qayta"),
        "Executor POD 1": (6.2, 1.7, 2.3, 0.9, "xizmat"),
        "Executor POD 2": (6.2, 0.4, 2.3, 0.9, "xizmat"),
        "Node 1": (9.6, 2.6, 1.8, 0.9, "saqlash"),
        "Node 2": (9.6, 1.0, 1.8, 0.9, "saqlash"),
    },
    [
        ("spark-submit", "Kubernetes API\n(scheduler)"),
        ("Kubernetes API\n(scheduler)", "Driver POD\n(bosh oshpaz)"),
        ("Driver POD\n(bosh oshpaz)", "Executor POD 1", "men 2 ta soray"),
        ("Driver POD\n(bosh oshpaz)", "Executor POD 2"),
        ("Executor POD 1", "Node 1"),
        ("Executor POD 2", "Node 2"),
    ],
    nom="spark_k8s.png", kenglik=14,
    izoh="Diqqat: executorlarni siz emas, DRIVER soraydi. Bu 12-bobdagi bosh oshpaz.",
)

export PATH=$HOME/.local/bin:$PATH
if k3d cluster list 2>/dev/null | grep -q kitob; then
  echo "klaster allaqachon ishlayapti"
else
  k3d cluster create kitob --agents 1 --api-port 16550 \
    --image rancher/k3s:v1.28.8-k3s1 --k3s-arg "--disable=traefik@server:0" > /dev/null 2>&1
  echo "klaster yaratildi"
fi
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
kubectl get nodes 2>/dev/null

export PATH=$HOME/.local/bin:$PATH
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
kubectl create namespace spark > /dev/null 2>&1
kubectl create serviceaccount spark -n spark > /dev/null 2>&1
kubectl create clusterrolebinding spark-rb \
  --clusterrole=edit --serviceaccount=spark:spark > /dev/null 2>&1
echo "namespace va service account tayyor"
kubectl get serviceaccount spark -n spark --no-headers 2>/dev/null

for NODE in k3d-kitob-server-0 k3d-kitob-agent-0; do
  if docker exec $NODE crictl images 2>/dev/null | grep -q "apache/spark"; then
    echo "$NODE: rasm bor"
  else
    docker exec $NODE crictl pull apache/spark:3.5.4 > /dev/null 2>&1 && echo "$NODE: tortildi"
  fi
done
docker exec k3d-kitob-server-0 crictl images 2>/dev/null | grep spark

export PATH=$HOME/.local/bin:$PATH
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
kubectl delete pod -n spark --all > /dev/null 2>&1
TOKEN=$(kubectl create token spark -n spark --duration=3600s 2>/dev/null)

docker run --rm --network k3d-kitob apache/spark:3.5.4 /opt/spark/bin/spark-submit \
  --master k8s://https://k3d-kitob-server-0:6443 \
  --deploy-mode cluster \
  --name kitob-pi \
  --class org.apache.spark.examples.SparkPi \
  --conf spark.kubernetes.namespace=spark \
  --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
  --conf spark.kubernetes.authenticate.submission.oauthToken=$TOKEN \
  --conf spark.kubernetes.trust.certificates=true \
  --conf spark.kubernetes.container.image=apache/spark:3.5.4 \
  --conf spark.kubernetes.container.image.pullPolicy=IfNotPresent \
  --conf spark.executor.instances=2 \
  --conf spark.driver.memory=512m \
  --conf spark.executor.memory=512m \
  local:///opt/spark/examples/jars/spark-examples_2.12-3.5.4.jar 10 2>&1 \
  | grep -iE "pod name|namespace|phase:|Exception" | head -8

export PATH=$HOME/.local/bin:$PATH
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
echo "=== PODLAR ==="
kubectl get pods -n spark --no-headers 2>/dev/null | awk '{printf "%-38s %s\n", $1, $3}'
echo ""
echo "=== NATIJA ==="
DRIVER=$(kubectl get pods -n spark --no-headers 2>/dev/null | grep driver | awk '{print $1}' | head -1)
kubectl logs -n spark $DRIVER 2>/dev/null | grep -iE "Pi is roughly" | head -1
echo ""
echo "=== DRIVER NIMA QILDI ==="
kubectl logs -n spark $DRIVER 2>/dev/null | grep -iE "Going to request|Limiting resource" | head -2 | cut -c1-105

export PATH=$HOME/.local/bin:$PATH
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
echo "node larda mavjud resurs:"
kubectl describe nodes 2>/dev/null | grep -A3 "Allocatable:" | grep -E "cpu|memory" | head -4

import pandas as pd
show(pd.DataFrame([
    {"Rejim": "cluster", "Driver qayerda": "Klaster ichida, pod sifatida",
     "Qachon": "Ishlab chiqarish, jadval bo'yicha ishlar",
     "Afzalligi": "Noutbukingiz o'chsa ham ish davom etadi"},
    {"Rejim": "client", "Driver qayerda": "Siz turgan mashinada",
     "Qachon": "Tajriba, Jupyter, nosozlik qidirish",
     "Afzalligi": "Natijani darhol ko'rasiz"},
]))

export PATH=$HOME/.local/bin:$PATH
export KUBECONFIG=$(k3d kubeconfig write kitob 2>/dev/null)
kubectl delete pod -n spark --all > /dev/null 2>&1
echo "podlar tozalandi, klaster ishlashda qoldi"
kubectl get nodes --no-headers 2>/dev/null | awk '{print "  ", $1, $2}'