Ma'lumot versiyalash: qaysi ma'lumotda o'qitilgan?

import shutil
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score

XUS = ["xonalar", "maydon", "qavat", "jami_qavat", "narx_mln", "tavsif_uzunligi",
       "rasmlar_soni", "telefon_takrorlanishi", "kunlik_elonlar", "egasi_tasdiqlangan"]

ISH.mkdir(parents=True, exist_ok=True)
ish_fayl = ISH / "ijara_ishchi.csv"
shutil.copy(DATA / "ijara.csv", ish_fayl)

def oqit(yol):
    d = pd.read_csv(yol)
    Xtr, Xte, ytr, yte = train_test_split(d[XUS], d["broker"], test_size=0.25,
                                          stratify=d["broker"], random_state=42)
    m = RandomForestClassifier(n_estimators=150, min_samples_leaf=3,
                               random_state=42, n_jobs=1).fit(Xtr, ytr)
    return round(f1_score(yte, m.predict(Xte)), 4), len(d)

f1_avval, n_avval = oqit(ish_fayl)
print(f"BIRINCHI o'qitish : F1 = {f1_avval}  ({n_avval:,} qator)")

d = pd.read_csv(ish_fayl)
CHEGARA = 10.2                                # 90-foizli: eng qimmat 10%
tozalangan = d[d["narx_mln"] < CHEGARA]       # "juda qimmat" e'lonlarni olib tashladi
tozalangan.to_csv(ish_fayl, index=False)      # O'SHA nomga yozdi

f1_keyin, n_keyin = oqit(ish_fayl)
print(f"IKKINCHI o'qitish : F1 = {f1_keyin}  ({n_keyin:,} qator)")
print(f"\nfayl nomi o'zgardimi : yo'q, {ish_fayl.name}")
print(f"qatorlar o'zgardimi  : {n_avval:,} -> {n_keyin:,}  ({n_avval - n_keyin} qator o'chdi)")
print(f"F1 o'zgardimi        : {f1_avval} -> {f1_keyin}")

"""Ma'lumot manifesti: qaysi fayl, qanday mazmun, qancha qator.

Git kodni saqlaydi; manifest esa ma'lumotning BARMOQ IZINI saqlaydi.
Faylning o'zi git ga tushmaydi (katta), lekin uning izi tushadi.
"""
import hashlib
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import pandas as pd


def fayl_hash(yol: str | Path, blok: int = 1 << 20) -> str:
    h = hashlib.sha256()
    with open(yol, "rb") as f:
        while chunk := f.read(blok):
            h.update(chunk)
    return h.hexdigest()


def tavsifla(yol: str | Path) -> dict[str, Any]:
    """Bitta ma'lumot faylining to'liq tavsifi."""
    p = Path(yol)
    d = pd.read_parquet(p) if p.suffix == ".parquet" else pd.read_csv(p)
    return {
        "yol": str(p.name),
        "sha256": fayl_hash(p),
        "hajm_bayt": p.stat().st_size,
        "qatorlar": int(len(d)),
        "ustunlar": list(d.columns),
        "turlar": {k: str(v) for k, v in d.dtypes.items()},
        "yozilgan": datetime.now(timezone.utc).isoformat(timespec="seconds"),
    }


def manifest(yollar: list[str | Path]) -> dict[str, Any]:
    return {"versiya": 1, "fayllar": [tavsifla(y) for y in yollar]}


def saqla(m: dict[str, Any], yol: str | Path) -> None:
    Path(yol).write_text(json.dumps(m, ensure_ascii=False, indent=2), encoding="utf-8")


def yukla(yol: str | Path) -> dict[str, Any]:
    return json.loads(Path(yol).read_text(encoding="utf-8"))


def solishtir(eski: dict[str, Any], yangi: dict[str, Any]) -> list[str]:
    """Ikki manifest orasidagi farqlarni odam o'qiydigan ro'yxat qilib qaytaradi."""
    farqlar: list[str] = []
    e = {f["yol"]: f for f in eski["fayllar"]}
    y = {f["yol"]: f for f in yangi["fayllar"]}

    for nom in sorted(set(e) | set(y)):
        if nom not in y:
            farqlar.append(f"{nom}: O'CHIRILGAN")
            continue
        if nom not in e:
            farqlar.append(f"{nom}: YANGI ({y[nom]['qatorlar']:,} qator)")
            continue
        a, b = e[nom], y[nom]
        if a["sha256"] == b["sha256"]:
            continue
        farqlar.append(f"{nom}: MAZMUN O'ZGARGAN")
        if a["qatorlar"] != b["qatorlar"]:
            farq = b["qatorlar"] - a["qatorlar"]
            farqlar.append(f"    qatorlar: {a['qatorlar']:,} -> {b['qatorlar']:,} ({farq:+,})")
        qoshilgan = set(b["ustunlar"]) - set(a["ustunlar"])
        yoqolgan = set(a["ustunlar"]) - set(b["ustunlar"])
        if qoshilgan:
            farqlar.append(f"    yangi ustunlar: {sorted(qoshilgan)}")
        if yoqolgan:
            farqlar.append(f"    yo'qolgan ustunlar: {sorted(yoqolgan)}")
        for u in sorted(set(a["turlar"]) & set(b["turlar"])):
            if a["turlar"][u] != b["turlar"][u]:
                farqlar.append(f"    '{u}' turi: {a['turlar'][u]} -> {b['turlar'][u]}")
    return farqlar

python3 -c "
import sys, shutil; sys.path.insert(0, 'src')
from pathlib import Path
import pandas as pd
from quvur.manifest import manifest, saqla, yukla, solishtir

shutil.copy('../datasets/ijara.csv', 'ijara_ishchi.csv')
saqla(manifest(['ijara_ishchi.csv']), 'manifest_v1.json')
print('v1 manifest yozildi')

d = pd.read_csv('ijara_ishchi.csv')
d[d['narx_mln'] < 10.2].to_csv('ijara_ishchi.csv', index=False)
saqla(manifest(['ijara_ishchi.csv']), 'manifest_v2.json')
print('v2 manifest yozildi')
print()
print('FARQLAR:')
for q in solishtir(yukla('manifest_v1.json'), yukla('manifest_v2.json')):
    print(' ', q)
"

python3 -c "
import sys; sys.path.insert(0, 'src')
import pandas as pd
from quvur.manifest import manifest, saqla, yukla, solishtir

d = pd.read_csv('ijara_ishchi.csv')
d['yangi_belgi'] = d['narx_mln'] / d['maydon']     # yangi ustun
d['xonalar'] = d['xonalar'].astype(float)          # tur o'zgardi
d.to_csv('ijara_ishchi.csv', index=False)
saqla(manifest(['ijara_ishchi.csv']), 'manifest_v3.json')

for q in solishtir(yukla('manifest_v2.json'), yukla('manifest_v3.json')):
    print(' ', q)
"

diagramma(
    {
        "ijara.csv\n(ish nomi)": (0, 1.2, 2.4, 1.0, "manba"),
        "sha256 hisoblanadi": (3.2, 1.2, 2.6, 1.0, "qayta"),
        "ombor/ab/cd1234...\n(mazmun bo'yicha)": (6.6, 0, 2.8, 1.0, "saqlash"),
        "manifest.json\n(nom -> hash)": (6.6, 2.4, 2.8, 1.0, "saqlash"),
    },
    [
        ("ijara.csv\n(ish nomi)", "sha256 hisoblanadi"),
        ("sha256 hisoblanadi", "ombor/ab/cd1234...\n(mazmun bo'yicha)", "nusxa"),
        ("sha256 hisoblanadi", "manifest.json\n(nom -> hash)", "yozuv"),
    ],
    nom="mazmun_ombori.png", kenglik=12,
    izoh="Fayl o'z hash i bilan saqlanadi. Bir xil mazmun — bir xil joy, ikki marta saqlanmaydi.",
)

"""Mazmun bo'yicha saqlash (content-addressed storage).

Fayl o'z sha256 hash i bilan saqlanadi. Ikki xil nom bir xil mazmunga ega bo'lsa,
disk da bitta nusxa yotadi — git ham shunday ishlaydi.
"""
import shutil
from pathlib import Path

from .manifest import fayl_hash


class Ombor:
    def __init__(self, ildiz: str | Path = ".ombor") -> None:
        self.ildiz = Path(ildiz)
        self.ildiz.mkdir(parents=True, exist_ok=True)

    def _yol(self, h: str) -> Path:
        # birinchi 2 belgi — papka (bitta papkada million fayl bo'lmasin)
        return self.ildiz / h[:2] / h[2:]

    def qoy(self, fayl: str | Path) -> tuple[str, bool]:
        """Faylni omborga qo'yadi. (hash, yangi_edimi) qaytaradi."""
        h = fayl_hash(fayl)
        nishon = self._yol(h)
        if nishon.exists():
            return h, False                      # allaqachon bor — nusxalamaymiz
        nishon.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(fayl, nishon)
        return h, True

    def ol(self, h: str, nishon: str | Path) -> Path:
        """Hash bo'yicha faylni tiklaydi."""
        manba = self._yol(h)
        if not manba.exists():
            raise FileNotFoundError(f"omborda yo'q: {h[:12]}...")
        Path(nishon).parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(manba, nishon)
        return Path(nishon)

    def hajmi(self) -> tuple[int, int]:
        """(fayllar soni, jami bayt)."""
        fayllar = [p for p in self.ildiz.rglob("*") if p.is_file()]
        return len(fayllar), sum(p.stat().st_size for p in fayllar)

python3 -c "
import sys, shutil; sys.path.insert(0, 'src')
from pathlib import Path
import pandas as pd
from quvur.ombor import Ombor

o = Ombor('.ombor')

# v1 ni qo'yamiz
shutil.copy('../datasets/ijara.csv', 'ijara_ishchi.csv')
h1, yangi1 = o.qoy('ijara_ishchi.csv')
print(f'v1 saqlandi: {h1[:12]}...  yangi={yangi1}')

# o'sha faylni YANA qo'yamiz — nusxalanmasligi kerak
h1b, yangi1b = o.qoy('ijara_ishchi.csv')
print(f'v1 qayta   : {h1b[:12]}...  yangi={yangi1b}   <- nusxalanmadi')

# v2: o'zgartiramiz
d = pd.read_csv('ijara_ishchi.csv')
d[d['narx_mln'] < 10.2].to_csv('ijara_ishchi.csv', index=False)
h2, yangi2 = o.qoy('ijara_ishchi.csv')
print(f'v2 saqlandi: {h2[:12]}...  yangi={yangi2}')

soni, bayt = o.hajmi()
print(f'\nomborda: {soni} fayl, {bayt/1024:.0f} KB')

# v1 ni TIKLAYMIZ
o.ol(h1, 'tiklangan.csv')
t = pd.read_csv('tiklangan.csv')
print(f'v1 tiklandi: {len(t):,} qator (hozirgi ish fayli: {len(pd.read_csv(\"ijara_ishchi.csv\")):,})')
"

python3 -m dvc version 2>/dev/null | head -6 || echo "DVC o'rnatilmagan (pip install dvc)"

import pandas as pd
show(pd.DataFrame([
    {"buyruq": "dvc init", "nima qiladi": "Git omborida DVC ni yoqadi"},
    {"buyruq": "dvc add malumot/ijara.csv", "nima qiladi": "Faylni kuzatuvga oladi, .dvc izi yasaydi"},
    {"buyruq": "git add malumot/ijara.csv.dvc .gitignore", "nima qiladi": "IZ git ga tushadi, fayl emas"},
    {"buyruq": "dvc remote add -d ombor s3://...", "nima qiladi": "Uzoq omborni ulaydi (MinIO, DO Spaces)"},
    {"buyruq": "dvc push", "nima qiladi": "Ma'lumotni omborga yuboradi"},
    {"buyruq": "git checkout <eski-commit>", "nima qiladi": "Kod eski holatga qaytadi"},
    {"buyruq": "dvc checkout", "nima qiladi": "MA'LUMOT ham o'sha holatga qaytadi"},
]))

cat > namuna.csv <<'CSV'
ism,baho
Aziz,85
Bobur,92
CSV
python3 -c "
import sys; sys.path.insert(0, 'src')
from quvur.manifest import fayl_hash
print('# namuna.csv.dvc fayli taxminan shunday ko\'rinadi:')
print('outs:')
print(f'- md5: {fayl_hash(\"namuna.csv\")[:32]}')
print(f'  size: {__import__(\"pathlib\").Path(\"namuna.csv\").stat().st_size}')
print('  path: namuna.csv')
"
rm -f namuna.csv

import pandas as pd
show(pd.DataFrame([
    {"holat": "Ma'lumot kichik (< 10 MB), kam o'zgaradi", "yechim": "To'g'ridan-to'g'ri git ga",
     "murakkablik": "yo'q"},
    {"holat": "O'rta hajm, bitta jamoa", "yechim": "Manifest + mazmun ombori (yuqoridagi kod)",
     "murakkablik": "past"},
    {"holat": "Katta, ko'p jamoa, uzoq ombor", "yechim": "DVC yoki LakeFS",
     "murakkablik": "o'rtacha"},
    {"holat": "Jadval, ko'p yozuvchi, time travel", "yechim": "Iceberg (19-bob)",
     "murakkablik": "yuqori"},
]))

python3 -c "
import sys, json, shutil; sys.path.insert(0, 'src')
from pathlib import Path
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from quvur.manifest import tavsifla
from quvur.ombor import Ombor

XUS = ['xonalar','maydon','qavat','jami_qavat','narx_mln','tavsif_uzunligi',
       'rasmlar_soni','telefon_takrorlanishi','kunlik_elonlar','egasi_tasdiqlangan']

shutil.copy('../datasets/ijara.csv', 'ijara_ishchi.csv')
o = Ombor('.ombor')
h, _ = o.qoy('ijara_ishchi.csv')

d = pd.read_csv('ijara_ishchi.csv')
Xtr, Xte, ytr, yte = train_test_split(d[XUS], d['broker'], test_size=0.25,
                                      stratify=d['broker'], random_state=42)
m = RandomForestClassifier(n_estimators=150, min_samples_leaf=3, random_state=42, n_jobs=1).fit(Xtr, ytr)

yozuv = {
    'malumot_hash': h,
    'malumot': tavsifla('ijara_ishchi.csv'),
    'metrika': {'f1': round(f1_score(yte, m.predict(Xte)), 4)},
}
Path('oqitish_yozuvi.json').write_text(json.dumps(yozuv, ensure_ascii=False, indent=2), encoding='utf-8')
print('ma\'lumot hash :', h[:16], '...')
print('qatorlar      :', yozuv['malumot']['qatorlar'])
print('F1            :', yozuv['metrika']['f1'])
print()
print('=> Bu F1 AYNAN shu hash li ma\'lumotga tegishli.')
print('   Ombor bor ekan, istalgan paytda o\'sha ma\'lumotni tiklab, natijani tekshirish mumkin.')
"