| """
|
| Faz 1 / Adım 4 — Shard'ları HF dataset repo'ya push (Faz 1 tam koşusu bitince).
|
|
|
| Her kaynağın shard'larını + manifest'ini `<user>/smartcore-v1-data` (PRIVATE) altına
|
| `<source>/` yoluna yükler. Eğitimde Colab bu repo'dan streaming ile çeker.
|
|
|
| HF_TOKEN gerekli. Kullanım:
|
| HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py
|
| HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py --only en_fineweb_edu
|
| """
|
| import os, sys, json, glob, argparse
|
|
|
| SHARDS = "kod/data/shards"
|
| SOURCES = ["en_fineweb_edu", "tr_fineweb2_hq", "code_codeparrot", "math_openwebmath"]
|
|
|
|
|
| def main():
|
| ap = argparse.ArgumentParser()
|
| ap.add_argument("--only", default=None)
|
| ap.add_argument("--repo", default=None, help="varsayılan: <user>/smartcore-v1-data")
|
| ap.add_argument("--private", action="store_true", default=True)
|
| args = ap.parse_args()
|
|
|
| tok = os.environ.get("HF_TOKEN")
|
| if not tok:
|
| print("HATA: HF_TOKEN env yok. Örnek: HF_TOKEN=hf_xxx python kod/faz1_04_push_hf.py")
|
| sys.exit(1)
|
| from huggingface_hub import HfApi
|
| api = HfApi(token=tok)
|
| user = api.whoami()["name"]
|
| repo = args.repo or f"{user}/smartcore-v1-data"
|
| api.create_repo(repo, repo_type="dataset", private=True, exist_ok=True)
|
| print(f"dataset repo: {repo} (private={args.private})")
|
|
|
|
|
| readme = os.path.join(SHARDS, "README.md")
|
| if os.path.exists(readme):
|
| api.upload_file(path_or_fileobj=readme, path_in_repo="README.md",
|
| repo_id=repo, repo_type="dataset", commit_message="dataset kartı")
|
| print(" README.md yüklendi (repo kökü)")
|
|
|
| sources = [args.only] if args.only else SOURCES
|
| grand = 0
|
| for src in sources:
|
| d = os.path.join(SHARDS, src)
|
| mf = os.path.join(d, "manifest.json")
|
| if not os.path.exists(mf):
|
| print(f" ATLA {src}: manifest yok"); continue
|
| m = json.load(open(mf, encoding="utf-8"))
|
| n_par = len(glob.glob(os.path.join(d, "*.parquet")))
|
| print(f" push {src}: {n_par} shard, {m.get('n_tokens',0)/1e9:.2f}B token ...", flush=True)
|
| api.upload_folder(folder_path=d, repo_id=repo, repo_type="dataset",
|
| path_in_repo=src, commit_message=f"Faz 1 shards: {src}")
|
| grand += m.get("n_tokens", 0)
|
| print(f" ✓ {src} yüklendi")
|
| print(f"\nTOPLAM yüklenen: {grand/1e9:.2f}B token -> https://huggingface.co/datasets/{repo}")
|
|
|
|
|
| if __name__ == "__main__":
|
| main()
|
|
|
