Upload pipeline.py with huggingface_hub

pipeline.py

@@ -57,6 +57,8 @@ def main():
     ap.add_argument("--guard", default=None,
                     help="file kamus kata (satu kata per baris); tambahkan metrik rasio ejaan ke manifest")
     ap.add_argument("--guard-max-new", type=int, default=120)
+    ap.add_argument("--guard-prefiks", default=None, help="default data/prefiks.txt bila ada")
+    ap.add_argument("--guard-sufiks", default=None, help="default data/sufiks.txt bila ada")
     t = ap.add_argument_group("hyperparameter training (diteruskan ke train.py)")
     t.add_argument("--steps", type=int, default=2000)
     t.add_argument("--batch-size", type=int, default=32)
@@ -136,6 +138,15 @@ def main():
         if not probe_path.exists():
             probe_path = out_dir / "indigo.safetensors"
         wordset = load_wordlist(args.guard)
+        p_def, s_def = ROOT / "data" / "prefiks.txt", ROOT / "data" / "sufiks.txt"
+
+        def muat_afiks(flag, default_path):
+            if flag:
+                return load_wordlist(flag) if Path(flag).exists() else None
+            return load_wordlist(str(default_path)) if default_path.exists() else None
+
+        prefiks = muat_afiks(args.guard_prefiks, p_def)
+        sufiks = muat_afiks(args.guard_sufiks, s_def)
         meta = load_meta(str(probe_path))
         tok = build_tokenizer(meta.get("tokenizer") or {"type": "char"}, meta.get("vocab"))
         mdl = GPT(GPTConfig(**meta["config"]))
@@ -146,7 +157,7 @@ def main():
         with torch.no_grad():
             out = mdl.generate(idx, args.guard_max_new, temperature=0.8, top_k=40)
         text = tok.decode(out[0].tolist())
-        stats["kamus_ratio"] = round(word_known_ratio(text, wordset), 4)
+        stats["kamus_ratio"] = round(word_known_ratio(text, wordset, prefiks, sufiks), 4)
         print(f"[pipeline] kamus_ratio={stats['kamus_ratio']:.2%}")
     stats["pipeline"] = {
         "tag": args.tag,