tez10 va tez11: qayta training qayerda to'xtadi

import json
from collections import Counter

def qatorlar(model):
    return [json.loads(q) for q in open(Path.home() / f"tez_runs/{model}/data/train.jsonl", encoding="utf-8")]

t8, t10 = qatorlar("tez8"), qatorlar("tez10")
print(f"tez8: {len(t8):,} qator, tez10: {len(t10):,} qator, farq: {len(t10) - len(t8):,}")

def kalit(qator):
    return json.dumps(qator["messages"][1:], ensure_ascii=False)

def tizim(qator):
    return qator["messages"][0]["content"]

savol8 = {kalit(q): tizim(q) for q in t8}
umumiy = [q for q in t10 if kalit(q) in savol8]
print(f"tez8 ning so'rov va javoblari tez10 da: {len(umumiy):,} ta")
print(f"ulardan tizim prompti ham aynan bir xil: {sum(tizim(q) == savol8[kalit(q)] for q in umumiy):,} ta")

birinchi = umumiy[0]
eski = savol8[kalit(birinchi)].splitlines()
yangi = tizim(birinchi).splitlines()
for a, b in zip(eski, yangi):
    if a != b:
        print("tez8 :", a)
        print("tez10:", b)

def jurnal(raqam):
    for qator in Path("/mnt/c/Users/Admin/Desktop/Offline/12-mistakes-and-lessons.md").read_text(encoding="utf-8").splitlines():
        if qator.startswith(f"| {raqam} |"):
            return [q.strip() for q in qator.strip("|").split("|")]

for ustun, matn in zip(("xato", "narxi", "qoida"), jurnal(21)[2:]):
    print(f"{ustun}: {matn}\n")

import glob
import random

TOZA = ["loans", "deposits", "cash_desk", "transactions", "realistic_loans", "realistic_deposits", "realistic_cash"]
GURUHLAR = {
    "toza": TOZA,
    "shovqinli": [f"noisy_{t}" for t in TOZA],
    "real": sorted(Path(f).parent.name for f in glob.glob(str(TEZ / "accuracy/suites/real_*/questions.json"))),
}
HAMMASI = sum(GURUHLAR.values(), [])

def hukmlar(prefiks, toplamlar):
    return {(t, j["id"]): j for t in toplamlar for j in map(json.loads, open(TEZ / f"results/{prefiks}_cpu_{t}.jsonl", encoding="utf-8"))}

def oraliq(prefiks_a, prefiks_b, toplamlar, marta=10000):
    a, b = hukmlar(prefiks_a, toplamlar), hukmlar(prefiks_b, toplamlar)
    farqlar = [int(b[k]["correct"]) - int(a[k]["correct"]) for k in sorted(a)]
    tasodif = random.Random(1)
    yigindilar = sorted(sum(tasodif.choice(farqlar) for _ in farqlar) for _ in range(marta))
    return sum(farqlar), yigindilar[int(0.025 * marta)], yigindilar[int(0.975 * marta)]

for guruh, toplamlar in GURUHLAR.items():
    a, b = hukmlar("tez8g", toplamlar), hukmlar("tez10g", toplamlar)
    print(f"{guruh:10s} {len(a):4d} savol:  tez8 {sum(j['correct'] for j in a.values())},  tez10 {sum(j['correct'] for j in b.values())}")
for nom, toplamlar in (("real", GURUHLAR["real"]), ("hammasi", HAMMASI)):
    farq, past, yuqori = oraliq("tez8g", "tez10g", toplamlar)
    print(f"{nom:8s} tez10 - tez8 = {farq:+d}, 95% oraliq [{past:+d}, {yuqori:+d}]")

def matritsa(prefiks, toplamlar):
    sanoq = Counter(j["reason"] if not j["reason"].startswith("plan failed") else "plan failed"
                    for j in hukmlar(prefiks, toplamlar).values())
    javoblar = sum(v for k, v in sanoq.items() if k not in ("asked", "asked on a clear request"))
    savollar = sanoq["asked"] + sanoq["guessed instead of asking"]
    return sanoq["same result"], javoblar, sanoq["asked"], savollar

for prefiks in ("tez8g", "tez10g"):
    togri, javoblar, topildi, savollar = matritsa(prefiks, GURUHLAR["real"])
    print(f"{prefiks[:-1]:6s} real: precision {togri}/{javoblar} = {togri / javoblar:.1%},  ask recall {topildi}/{savollar}")

yangi = [json.loads(q) for q in open(Path.home() / "tez_runs/tez11/data/new.jsonl", encoding="utf-8")]
qator_soz = Counter(len(q["messages"][-2]["content"].split()) for q in yangi)
test_soz = Counter(len(s["text"].split()) for t in GURUHLAR["real"]
                   for s in json.load(open(TEZ / f"accuracy/suites/{t}/questions.json", encoding="utf-8")) if s.get("trap") == "tool_gap")
print(f"o'qitish qatorlari: {len(yangi)} ta, eng qisqasi {min(qator_soz)} so'z, eng uzuni {max(qator_soz)} so'z")
print(f"  10-16 so'zli: {sum(v for k, v in qator_soz.items() if 10 <= k <= 16)} ta")
print(f"test tool_gap savollari: {sum(test_soz.values())} ta, {min(test_soz)} dan {max(test_soz)} so'zgacha")
print("namuna:", yangi[0]["messages"][-2]["content"])

for ustun, matn in zip(("xato", "narxi", "qoida"), jurnal(22)[2:]):
    print(f"{ustun}: {matn}\n")

for prefiks in ("tez9g", "tez10g"):
    a, b = hukmlar("tez8g", HAMMASI), hukmlar(prefiks, HAMMASI)
    ozgardi = sum(a[k]["output"] != b[k]["output"] for k in a)
    buzildi = sum(a[k]["correct"] and not b[k]["correct"] for k in a)
    tuzaldi = sum(b[k]["correct"] and not a[k]["correct"] for k in a)
    print(f"{prefiks[:-1]:6s}: {ozgardi} ta javob o'zgardi ({ozgardi / len(a):.1%}), {tuzaldi} tasi tuzaldi, {buzildi} tasi buzildi")

for ustun, matn in zip(("xato", "narxi", "qoida"), jurnal(23)[2:]):
    print(f"{ustun}: {matn}\n")

import ast
manba = (TEZ / "training/update.py").read_text(encoding="utf-8")
for tugun in ast.parse(manba).body:
    if isinstance(tugun, ast.FunctionDef) and tugun.name in ("mix", "load_model"):
        print(ast.get_source_segment(manba, tugun), end="\n\n")

yozuv = json.load(open(Path.home() / "tez_runs/tez11/tez11a/output/update_run.json", encoding="utf-8"))
for k in ("rows", "replay_rows", "lr", "epochs"):
    print(f"{k}: {yozuv[k]}")

a, b = hukmlar("tez8g", HAMMASI), hukmlar("tez11ag", HAMMASI)
for guruh, toplamlar in GURUHLAR.items():
    print(f"{guruh:10s} tez8 {sum(a[k]['correct'] for k in a if k[0] in toplamlar)},  tez11a {sum(b[k]['correct'] for k in b if k[0] in toplamlar)}")
ozgardi = sum(a[k]["output"] != b[k]["output"] for k in a)
print(f"o'zgargan javoblar: {ozgardi} ({ozgardi / len(a):.1%}), tuzaldi {sum(b[k]['correct'] and not a[k]['correct'] for k in a)}, buzildi {sum(a[k]['correct'] and not b[k]['correct'] for k in a)}")

tayyor = [t for t in HAMMASI if (TEZ / f"results/tez11bg_cpu_{t}.jsonl").exists()
          and sum(1 for _ in open(TEZ / f"results/tez11bg_cpu_{t}.jsonl")) == len([k for k in a if k[0] == t])]
a2, b2 = hukmlar("tez8g", tayyor), hukmlar("tez11bg", tayyor)
print(f"tez11b ({len(a2)} savolda to'xtatildi): tuzaldi {sum(b2[k]['correct'] and not a2[k]['correct'] for k in a2)}, buzildi {sum(a2[k]['correct'] and not b2[k]['correct'] for k in a2)}")

ishonch = {(t, j["id"]): j["min_prob"] for t in HAMMASI for j in map(json.loads, open(TEZ / f"results/tez8c_cpu_{t}.jsonl", encoding="utf-8"))}
ozgargan = sorted(ishonch[k] for k in a if a[k]["output"] != b[k]["output"])
qolgan = sorted(ishonch[k] for k in a if a[k]["output"] == b[k]["output"])
for nom, guruh in (("o'zgargan", ozgargan), ("o'zgarmagan", qolgan)):
    print(f"{nom:12s} {len(guruh):4d} ta: median {guruh[len(guruh) // 2]:.3f},  0.7 dan past: {sum(p < 0.7 for p in guruh) / len(guruh):.0%}")

for ustun, matn in zip(("xato", "narxi", "qoida"), jurnal(24)[2:]):
    print(f"{ustun}: {matn}\n")