Upload audit_kamus.py with huggingface_hub

audit_kamus.py

@@ -0,0 +1,117 @@
+import argparse
+import json
+import os
+import random
+import time
+from datetime import datetime
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+UA = {"User-Agent": "Mozilla/5.0 (audit-kamus-indigo)"}
+
+
+def cek_pypi(word):
+    try:
+        from kbbi import KBBI, TidakDitemukan
+    except ImportError:
+        return None
+    try:
+        KBBI(word)
+        return True
+    except TidakDitemukan:
+        return False
+    except Exception:
+        return None
+
+
+def cek_web(word):
+    import requests
+
+    try:
+        from urllib.parse import quote
+
+        r = requests.get(
+            f"https://kbbi.web.id/{quote(word)}", timeout=15, headers=UA
+        )
+        if r.status_code != 200:
+            return None
+        marker = "Maaf, tidak ditemukan kata yang dicari"
+        jumlah = r.text.count(marker)
+        if jumlah == 0:
+            return None
+        return jumlah < 3
+    except Exception:
+        return None
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Audit wordlist kamus terhadap KBBI daring (sampling acak)"
+    )
+    ap.add_argument("--kamus", default=str(ROOT / "data" / "kamus_id.txt"))
+    ap.add_argument("--n", type=int, default=200, help="jumlah kata yang diambil sampel")
+    ap.add_argument("--seed", type=int, default=1337)
+    ap.add_argument("--delay", type=float, default=0.8, help="jeda antar-permintaan (detik)")
+    ap.add_argument("--backend", choices=["auto", "pypi", "web"], default="auto",
+                    help="web: kbbi.web.id (heuristik jumlah marker; rapuh)")
+    ap.add_argument("--apply", action="store_true",
+                    help="hapus kata TIDAK-DIKENAL dari kamus (backup otomatis)")
+    ap.add_argument("--out", default=None, help="path laporan json")
+    args = ap.parse_args()
+
+    kamus_path = Path(args.kamus)
+    kata_semua = [w.strip() for w in kamus_path.read_text(encoding="utf-8").splitlines() if w.strip()]
+    rng = random.Random(args.seed)
+    sampel = sorted(rng.sample(kata_semua, min(args.n, len(kata_semua))))
+    print(f"kamus={kamus_path.name} ({len(kata_semua):,} kata) | sampel={len(sampel)} | seed={args.seed}")
+
+    hasil = {}
+    n_ada = n_tidak = n_gagal = 0
+    t0 = time.time()
+    for i, w in enumerate(sampel, 1):
+        status = None
+        if args.backend in ("auto", "pypi"):
+            status = cek_pypi(w)
+        if status is None and args.backend in ("auto", "web"):
+            status = cek_web(w)
+        hasil[w] = status
+        if status is True:
+            n_ada += 1
+        elif status is False:
+            n_tidak += 1
+        else:
+            n_gagal += 1
+        if i % 10 == 0 or i == len(sampel):
+            print(f"  {i}/{len(sampel)} | ada={n_ada} tidak={n_tidak} gagal={n_gagal} | {time.time() - t0:.0f}s")
+        time.sleep(args.delay)
+
+    ditolak = sorted(w for w, s in hasil.items() if s is False)
+    laporan = {
+        "timestamp": datetime.now().isoformat(timespec="seconds"),
+        "kamus": str(kamus_path),
+        "ukuran_kamus": len(kata_semua),
+        "sampel": len(sampel),
+        "seed": args.seed,
+        "ada": n_ada,
+        "tidak_dikenal": n_tidak,
+        "gagal_ceks": n_gagal,
+        "estimasi_entri_tidak_valid": round(n_tidak / max(1, len(sampel) - n_gagal) * len(kata_semua)),
+        "kata_ditolak": ditolak,
+    }
+    out = Path(args.out) if args.out else ROOT / "runs" / f"kamus_audit_{datetime.now():%Y%m%d_%H%M%S}.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(laporan, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"laporan -> {out}")
+    print(f"ringkasan: ada={n_ada} | tidak-dikenal={n_tidak} | gagal={n_gagal} "
+          f"| estimasi entri tak-valid di kamus: ~{laporan['estimasi_entri_tidak_valid']:,}")
+
+    if args.apply and ditolak:
+        backup = kamus_path.with_suffix(".bak.txt")
+        backup.write_text("\n".join(sorted(kata_semua)) + "\n", encoding="utf-8")
+        bersih = [w for w in kata_semua if w not in set(ditolak)]
+        kamus_path.write_text("\n".join(bersih) + "\n", encoding="utf-8")
+        print(f"--apply: {len(ditolak)} kata dihapus, kamus {len(kata_semua):,} -> {len(bersih):,}, backup={backup.name}")
+
+
+if __name__ == "__main__":
+    main()