Upload pipeline.py with huggingface_hub

pipeline.py

@@ -54,6 +54,9 @@ def main():
                     help="promosikan checkpoint terbaik run ini ke folder out/ kanonik")
     ap.add_argument("--push", action="store_true", help="upload checkpoint terbaik ke repo HF")
     ap.add_argument("--repo", default="adyoi/indigo")
+    ap.add_argument("--guard", default=None,
+                    help="file kamus kata (satu kata per baris); tambahkan metrik rasio ejaan ke manifest")
+    ap.add_argument("--guard-max-new", type=int, default=120)
     t = ap.add_argument_group("hyperparameter training (diteruskan ke train.py)")
     t.add_argument("--steps", type=int, default=2000)
     t.add_argument("--batch-size", type=int, default=32)
@@ -121,6 +124,30 @@ def main():
 
     print(f"[pipeline] run='{args.tag}' | sumber={sources or ['<kosong>' ]}")
     stats = train.main(argv) or {}
+
+    if args.guard:
+        import torch
+
+        from indigo.common import load_wordlist, load_meta, build_tokenizer, word_known_ratio
+        from indigo.model import GPT, GPTConfig
+        from safetensors.torch import load_file
+
+        probe_path = out_dir / "indigo_best.safetensors"
+        if not probe_path.exists():
+            probe_path = out_dir / "indigo.safetensors"
+        wordset = load_wordlist(args.guard)
+        meta = load_meta(str(probe_path))
+        tok = build_tokenizer(meta.get("tokenizer") or {"type": "char"}, meta.get("vocab"))
+        mdl = GPT(GPTConfig(**meta["config"]))
+        mdl.load_state_dict(load_file(str(probe_path)), strict=False)
+        mdl.eval()
+        ids = tok.encode("\n") or [0]
+        idx = torch.tensor([ids], dtype=torch.long)
+        with torch.no_grad():
+            out = mdl.generate(idx, args.guard_max_new, temperature=0.8, top_k=40)
+        text = tok.decode(out[0].tolist())
+        stats["kamus_ratio"] = round(word_known_ratio(text, wordset), 4)
+        print(f"[pipeline] kamus_ratio={stats['kamus_ratio']:.2%}")
     stats["pipeline"] = {
         "tag": args.tag,
         "timestamp": datetime.datetime.now().isoformat(timespec="seconds"),