"""
Faz 1 / Adım 4 — Shard'ları HF dataset repo'ya push (Faz 1 tam koşusu bitince).

Her kaynağın shard'larını + manifest'ini `<user>/smartcore-v1-data` (PRIVATE) altına
`<source>/` yoluna yükler. Eğitimde Colab bu repo'dan streaming ile çeker.

HF_TOKEN gerekli. Kullanım:
  HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py
  HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py --only en_fineweb_edu
"""
import os, sys, json, glob, argparse

SHARDS = "kod/data/shards"
SOURCES = ["en_fineweb_edu", "tr_fineweb2_hq", "code_codeparrot", "math_openwebmath"]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--only", default=None)
    ap.add_argument("--repo", default=None, help="varsayılan: <user>/smartcore-v1-data")
    ap.add_argument("--private", action="store_true", default=True)
    args = ap.parse_args()

    tok = os.environ.get("HF_TOKEN")
    if not tok:
        print("HATA: HF_TOKEN env yok. Örnek: HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py")
        sys.exit(1)
    from huggingface_hub import HfApi
    api = HfApi(token=tok)
    user = api.whoami()["name"]
    repo = args.repo or f"{user}/smartcore-v1-data"
    api.create_repo(repo, repo_type="dataset", private=True, exist_ok=True)
    print(f"dataset repo: {repo} (private={args.private})")

    # dataset kartı (README) → repo kökü
    readme = os.path.join(SHARDS, "README.md")
    if os.path.exists(readme):
        api.upload_file(path_or_fileobj=readme, path_in_repo="README.md",
                        repo_id=repo, repo_type="dataset", commit_message="dataset kartı")
        print("  README.md yüklendi (repo kökü)")

    sources = [args.only] if args.only else SOURCES
    grand = 0
    for src in sources:
        d = os.path.join(SHARDS, src)
        mf = os.path.join(d, "manifest.json")
        if not os.path.exists(mf):
            print(f"  ATLA {src}: manifest yok"); continue
        m = json.load(open(mf, encoding="utf-8"))
        n_par = len(glob.glob(os.path.join(d, "*.parquet")))
        print(f"  push {src}: {n_par} shard, {m.get('n_tokens',0)/1e9:.2f}B token ...", flush=True)
        api.upload_folder(folder_path=d, repo_id=repo, repo_type="dataset",
                          path_in_repo=src, commit_message=f"Faz 1 shards: {src}")
        grand += m.get("n_tokens", 0)
        print(f"    ✓ {src} yüklendi")
    print(f"\nTOPLAM yüklenen: {grand/1e9:.2f}B token -> https://huggingface.co/datasets/{repo}")


if __name__ == "__main__":
    main()