Upload pipeline.py with huggingface_hub

pipeline.py

@@ -0,0 +1,181 @@
+import argparse
+import datetime
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT))
+
+import train
+
+
+def git_short():
+    try:
+        return subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"], cwd=str(ROOT), text=True
+        ).strip()
+    except Exception:
+        return "nogit"
+
+
+def pull_hf(repo_id, dest, patterns):
+    from huggingface_hub import snapshot_download
+
+    path = snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        allow_patterns=patterns,
+        local_dir=str(dest),
+    )
+    n = sum(
+        1 for f in Path(path).rglob("*")
+        if f.is_file() and ".cache" not in f.parts and f.suffix.lstrip(".") in
+        {p.lstrip("*.") for p in patterns}
+    )
+    print(f"[pipeline] dataset '{repo_id}' ditarik -> {dest} ({n} file)")
+    return str(dest)
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Pipeline latih Indigo: kumpul data -> tokenizer -> training -> manifest -> finalize/push"
+    )
+    ap.add_argument("--tag", required=True, help="nama run; artefak di <runs>/<tag>/")
+    ap.add_argument("--data", nargs="*", default=[], help="file/folder teks lokal tambahan")
+    ap.add_argument("--hf-dataset", default=None, help="repo dataset HF untuk menarik file teks")
+    ap.add_argument("--hf-patterns", nargs="*", default=["*.txt", "*.md"])
+    ap.add_argument("--runs", default="runs")
+    ap.add_argument("--device", default=None, help="diteruskan ke train.py (default auto)")
+    ap.add_argument("--finalize", action="store_true",
+                    help="promosikan checkpoint terbaik run ini ke folder out/ kanonik")
+    ap.add_argument("--push", action="store_true", help="upload checkpoint terbaik ke repo HF")
+    ap.add_argument("--repo", default="adyoi/indigo")
+    t = ap.add_argument_group("hyperparameter training (diteruskan ke train.py)")
+    t.add_argument("--steps", type=int, default=2000)
+    t.add_argument("--batch-size", type=int, default=32)
+    t.add_argument("--block-size", type=int, default=128)
+    t.add_argument("--n-layer", type=int, default=4)
+    t.add_argument("--n-head", type=int, default=4)
+    t.add_argument("--n-embd", type=int, default=128)
+    t.add_argument("--dropout", type=float, default=0.1)
+    t.add_argument("--lr", type=float, default=3e-4)
+    t.add_argument("--warmup", type=int, default=100)
+    t.add_argument("--weight-decay", type=float, default=0.1)
+    t.add_argument("--eval-interval", type=int, default=200)
+    t.add_argument("--eval-iters", type=int, default=20)
+    t.add_argument("--seed", type=int, default=1337)
+    t.add_argument("--init-from", default=None)
+    t.add_argument("--tokenizer", default="char", choices=["char", "bpe"])
+    t.add_argument("--vocab-size", type=int, default=512)
+    t.add_argument("--val-fraction", type=float, default=0.1)
+    args = ap.parse_args()
+
+    run_dir = ROOT / args.runs / args.tag
+    data_dir = run_dir / "data"
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    sources = []
+    for p in args.data:
+        src = Path(p)
+        if not src.exists():
+            raise SystemExit(f"sumber data tidak ada: {src}")
+        if src.is_dir():
+            shutil.copytree(src, data_dir / src.name, dirs_exist_ok=True)
+        else:
+            shutil.copy2(src, data_dir / src.name)
+        sources.append(str(src))
+
+    if args.hf_dataset:
+        pull_hf(args.hf_dataset, data_dir / "_hf", args.hf_patterns)
+        sources.append(f"hf:{args.hf_dataset}")
+
+    out_dir = run_dir / "ckpt"
+    argv = [
+        "--data", str(data_dir),
+        "--out", str(out_dir),
+        "--steps", str(args.steps),
+        "--batch-size", str(args.batch_size),
+        "--block-size", str(args.block_size),
+        "--n-layer", str(args.n_layer),
+        "--n-head", str(args.n_head),
+        "--n-embd", str(args.n_embd),
+        "--dropout", str(args.dropout),
+        "--lr", str(args.lr),
+        "--warmup", str(args.warmup),
+        "--weight-decay", str(args.weight_decay),
+        "--eval-interval", str(args.eval_interval),
+        "--eval-iters", str(args.eval_iters),
+        "--seed", str(args.seed),
+        "--tokenizer", args.tokenizer,
+        "--vocab-size", str(args.vocab_size),
+        "--val-fraction", str(args.val_fraction),
+    ]
+    if args.init_from:
+        argv += ["--init-from", args.init_from]
+    if args.device:
+        argv += ["--device", args.device]
+
+    print(f"[pipeline] run='{args.tag}' | sumber={sources or ['<kosong>' ]}")
+    stats = train.main(argv) or {}
+    stats["pipeline"] = {
+        "tag": args.tag,
+        "timestamp": datetime.datetime.now().isoformat(timespec="seconds"),
+        "git": git_short(),
+        "sources": sources,
+        "hf_dataset": args.hf_dataset,
+        "run_dir": str(run_dir),
+    }
+
+    manifest = run_dir / "manifest.json"
+    with open(manifest, "w", encoding="utf-8") as f:
+        json.dump(stats, f, ensure_ascii=False, indent=2)
+    print(f"[pipeline] manifest -> {manifest}")
+
+    best = out_dir / "indigo_best.safetensors"
+    if args.finalize:
+        has_best = (out_dir / "indigo_best.safetensors").exists()
+        has_final = (out_dir / "indigo.safetensors").exists()
+        if not has_best and not has_final:
+            raise SystemExit("finalize dibatalkan: tidak ada checkpoint")
+        if not has_best:
+            print("[pipeline] peringatan: tanpa data validasi; memakai checkpoint akhir sebagai best")
+        dst = ROOT / "out"
+        dst.mkdir(exist_ok=True)
+        copied = []
+        for name in ("indigo_best.safetensors", "indigo_best_meta.json",
+                     "indigo.safetensors", "indigo_meta.json", "indigo_optimizer.pt"):
+            srcp = out_dir / name
+            if srcp.exists():
+                shutil.copy2(srcp, dst / name)
+                copied.append(name)
+        print(f"[pipeline] finalisasi ke out/: {', '.join(copied)}")
+
+    if args.push:
+        from huggingface_hub import HfApi
+
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=str(best),
+            path_in_repo="out/indigo_best.safetensors",
+            repo_id=args.repo,
+            repo_type="model",
+        )
+        meta_f = out_dir / "indigo_best_meta.json"
+        if meta_f.exists():
+            api.upload_file(
+                path_or_fileobj=str(meta_f),
+                path_in_repo="out/indigo_best_meta.json",
+                repo_id=args.repo,
+                repo_type="model",
+            )
+        print(f"[pipeline] checkpoint best ter-upload ke {args.repo}")
+
+    print("[pipeline] selesai.")
+
+
+if __name__ == "__main__":
+    main()