Monitoring: jim buzilishni eshitiladigan qilish

import pandas as pd
show(pd.DataFrame([
    {"Qatlam": "Infratuzilma", "Misol": "CPU, xotira, disk",
     "Model eskirsa ko'rinadimi": "Yo'q"},
    {"Qatlam": "Servis", "Misol": "So'rov soni, xato ulushi, p99",
     "Model eskirsa ko'rinadimi": "Yo'q"},
    {"Qatlam": "**Model**", "Misol": "Bashorat taqsimoti, ishonch",
     "Model eskirsa ko'rinadimi": "**Ha**"},
    {"Qatlam": "**Biznes**", "Misol": "Shikoyatlar, qo'lda tuzatishlar",
     "Model eskirsa ko'rinadimi": "**Ha, lekin kech**"},
]))

"""Prometheus formatidagi o'lchovlar. Tashqi kutubxonasiz.

Format: har qator `nom{yorliq="qiymat"} son`.
Prometheus shu matnni o'qiydi va vaqt qatoriga aylantiradi.
"""
import threading
import time
from collections import Counter

_qulf = threading.Lock()

# hisoblagichlar: faqat oshadi
_sorovlar = Counter()          # holat kodi bo'yicha
_bashoratlar = Counter()       # broker / ega

# kechikish: oddiy gistogramma
_chegaralar = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0)
_paqir = Counter()
_jami_vaqt = 0.0
_jami_soni = 0

# ehtimol taqsimoti: o'nta oraliq
_ehtimol = Counter()

_boshlandi = time.time()


def sorov_yoz(kod: int, sekund: float) -> None:
    global _jami_vaqt, _jami_soni
    with _qulf:
        _sorovlar[kod] += 1
        _jami_vaqt += sekund
        _jami_soni += 1
        for c in _chegaralar:
            if sekund <= c:
                _paqir[c] += 1


def bashorat_yoz(ehtimol: float, broker: bool) -> None:
    with _qulf:
        _bashoratlar["broker" if broker else "ega"] += 1
        _ehtimol[min(int(ehtimol * 10), 9)] += 1


def matn() -> str:
    """Prometheus o'qiydigan matn."""
    q = []
    q.append("# HELP servis_sorovlar_jami Jami so'rovlar")
    q.append("# TYPE servis_sorovlar_jami counter")
    for kod, n in sorted(_sorovlar.items()):
        q.append(f'servis_sorovlar_jami{{kod="{kod}"}} {n}')

    q.append("# HELP servis_kechikish_sekund So'rov kechikishi")
    q.append("# TYPE servis_kechikish_sekund histogram")
    yigindi = 0
    for c in _chegaralar:
        yigindi = _paqir[c]
        q.append(f'servis_kechikish_sekund_bucket{{le="{c}"}} {yigindi}')
    q.append(f'servis_kechikish_sekund_bucket{{le="+Inf"}} {_jami_soni}')
    q.append(f"servis_kechikish_sekund_sum {_jami_vaqt:.6f}")
    q.append(f"servis_kechikish_sekund_count {_jami_soni}")

    # MODEL qatlami - eng muhimi
    q.append("# HELP model_bashoratlar_jami Qaror bo'yicha bashoratlar")
    q.append("# TYPE model_bashoratlar_jami counter")
    for nom, n in sorted(_bashoratlar.items()):
        q.append(f'model_bashoratlar_jami{{qaror="{nom}"}} {n}')

    q.append("# HELP model_ehtimol_taqsimoti Ehtimol oraliqlari")
    q.append("# TYPE model_ehtimol_taqsimoti counter")
    for i in range(10):
        q.append(f'model_ehtimol_taqsimoti{{oraliq="0.{i}"}} {_ehtimol[i]}')

    q.append("# HELP servis_ishlash_sekund Ishga tushgandan beri")
    q.append("# TYPE servis_ishlash_sekund gauge")
    q.append(f"servis_ishlash_sekund {time.time() - _boshlandi:.1f}")
    return "\n".join(q) + "\n"

"""27-bobdagi servis + o'lchovlar."""
import os
import time
from pathlib import Path
from typing import Annotated, Any

from fastapi import FastAPI, HTTPException, Request, Response
from pydantic import BaseModel, Field

from quvur.format import onnx_ehtimol, onnx_sessiya
from quvur.xususiyat import bitta_qator
from xizmat.olchov import bashorat_yoz, matn, sorov_yoz

MODEL_YOLI = os.environ.get("MODEL_YOLI", "model.onnx")
CHEGARA = float(os.environ.get("BROKER_CHEGARASI", "0.5"))

app = FastAPI(title="Broker aniqlash (o'lchovli)")
HOLAT: dict[str, Any] = {"sessiya": None}


def yukla() -> None:
    p = Path(MODEL_YOLI)
    if p.exists():
        HOLAT["sessiya"] = onnx_sessiya(p)


yukla()


@app.middleware("http")
async def olchov_oraliq(request: Request, call_next):
    """Har so'rovni o'lchaydi — bitta joyda, har yo'l uchun alohida emas."""
    t0 = time.perf_counter()
    javob = await call_next(request)
    if request.url.path != "/metrics":
        sorov_yoz(javob.status_code, time.perf_counter() - t0)
    return javob


class Elon(BaseModel):
    xonalar: Annotated[int, Field(ge=1, le=10)]
    maydon: Annotated[float, Field(ge=10, le=500)]
    qavat: Annotated[int, Field(ge=1, le=60)]
    jami_qavat: Annotated[int, Field(ge=1, le=60)]
    narx_mln: Annotated[float, Field(ge=0.5, le=200)]
    tavsif_uzunligi: Annotated[int, Field(ge=0, le=5000)]
    rasmlar_soni: Annotated[int, Field(ge=0, le=100)]
    telefon_takrorlanishi: Annotated[int, Field(ge=1, le=500)]
    kunlik_elonlar: Annotated[int, Field(ge=1, le=200)]
    egasi_tasdiqlangan: Annotated[int, Field(ge=0, le=1)]


@app.get("/healthz")
def tirikmi() -> dict[str, str]:
    return {"holat": "tirik"}


@app.get("/ready")
def tayyormi() -> dict[str, Any]:
    if HOLAT["sessiya"] is None:
        raise HTTPException(status_code=503, detail="model yuklanmagan")
    return {"holat": "tayyor"}


@app.get("/metrics")
def olchovlar() -> Response:
    return Response(content=matn(), media_type="text/plain; version=0.0.4")


@app.post("/bashorat")
def bashorat(elon: Elon) -> dict[str, Any]:
    if HOLAT["sessiya"] is None:
        raise HTTPException(status_code=503, detail="model yuklanmagan")
    x = bitta_qator(elon.model_dump())
    e = float(onnx_ehtimol(HOLAT["sessiya"], x.values)[0])
    qaror = e >= CHEGARA
    bashorat_yoz(e, qaror)
    return {"broker_ehtimoli": round(e, 4), "broker": qaror}

python3 -c "
import sys, pathlib; sys.path.insert(0, 'src')
if not pathlib.Path('model.onnx').exists():
    import pandas as pd
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.model_selection import train_test_split
    from quvur.xususiyat import XUSUSIYATLAR, tayyorla
    from quvur.format import onnx_ga
    d = pd.read_csv('../datasets/ijara.csv')
    x, y = tayyorla(d), d['broker']
    Xtr, _, ytr, _ = train_test_split(x, y, test_size=0.25, stratify=y, random_state=42)
    m = RandomForestClassifier(n_estimators=150, min_samples_leaf=3,
                               random_state=42, n_jobs=1).fit(Xtr, ytr)
    onnx_ga(m, len(XUSUSIYATLAR), 'model.onnx')
print('model tayyor')
"
export PYTHONPATH=src
python3 -m uvicorn xizmat.servis_olchovli:app --host 127.0.0.1 --port 18150 \
  --log-level warning > olchov.log 2>&1 < /dev/null &
echo $! > olchov.pid
for i in $(seq 1 40); do
  [ "$(curl -s -o /dev/null -w %{http_code} http://127.0.0.1:18150/ready 2>/dev/null)" = "200" ] && { echo "servis tayyor"; break; }
  sleep 0.5
done

python3 -c "
import json, urllib.request, urllib.error
import pandas as pd

d = pd.read_csv('../datasets/ijara.csv').head(400)
XUS = ['xonalar','maydon','qavat','jami_qavat','narx_mln','tavsif_uzunligi',
       'rasmlar_soni','telefon_takrorlanishi','kunlik_elonlar','egasi_tasdiqlangan']

ok = xato = 0
for e in d[XUS].to_dict('records'):
    r = urllib.request.Request('http://127.0.0.1:18150/bashorat',
                               data=json.dumps(e).encode(),
                               headers={'Content-Type': 'application/json'})
    try:
        with urllib.request.urlopen(r, timeout=10) as j: j.read(); ok += 1
    except Exception: xato += 1

# ataylab yomon sorovlar - xato hisoblagichini tekshirish uchun
for _ in range(25):
    r = urllib.request.Request('http://127.0.0.1:18150/bashorat',
                               data=json.dumps({'xonalar': 2, 'maydon': -5}).encode(),
                               headers={'Content-Type': 'application/json'})
    try:
        with urllib.request.urlopen(r, timeout=5) as j: j.read()
    except urllib.error.HTTPError: xato += 1

print(f'yuborildi: {ok} yaxshi, {xato} yomon')
"

curl -s http://127.0.0.1:18150/metrics | grep -vE "^#" | grep -E "sorovlar_jami|bashoratlar_jami|kechikish_sekund_(sum|count)"

curl -s http://127.0.0.1:18150/metrics | grep "model_ehtimol_taqsimoti" | grep -v "^#"

kill "$(cat olchov.pid)" 2>/dev/null
sleep 1
echo "o'lchovli servis to'xtatildi"
ss -ltn 2>/dev/null | grep -c 18150 || echo "18150 port bo'sh"