Download tests/eval_01_intent.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 4.77 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/eval_01_intent.py
- Command line
-
hf download hf://recommender-system/llmproccrs/tests/eval_01_intent.py
-
curl -L -o eval_01_intent.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/eval_01_intent.py
4.77 kB
| import sys | |
| import os | |
| import csv | |
| import time | |
| sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) | |
| from modules.dialog_manager.intent import IntentDetector | |
| def run_eval_intent(): | |
| print("\n" + "="*60) | |
| print("๐ EVALUASI SKENARIO 1: INTENT DETECTION (With CSV Report)") | |
| print("="*60) | |
| # Init Output Folder | |
| report_dir = os.path.join(os.path.dirname(__file__), "reports") | |
| os.makedirs(report_dir, exist_ok=True) | |
| csv_file = os.path.join(report_dir, "report_01_intent_mistral.csv") | |
| # csv_file = os.path.join(report_dir, "report_01_intent_llama3.csv") | |
| # csv_file = os.path.join(report_dir, "report_01_intent_qwen3.csv") | |
| # csv_file = os.path.join(report_dir, "report_01_intent_gemma3.csv") | |
| try: | |
| detector = IntentDetector() | |
| print("โ IntentDetector Ready.\n") | |
| except Exception as e: | |
| print(f"โ Init Failed: {e}") | |
| return | |
| test_cases = [ | |
| # Basa-basi | |
| ("Pagi min, on gak?", "basa_basi", {}), | |
| ("Siang min, on gak?", "basa_basi", {}), | |
| ("Malam min, on gak?", "basa_basi", {}), | |
| ("Selamat pagi Bapak/Ibu admin.", "basa_basi", {}), | |
| ("Selamat siang Bapak/Ibu admin.", "basa_basi", {}), | |
| ("Selamat malam Bapak/Ibu admin.", "basa_basi", {}), | |
| ("Haloo Bot, mau tanya seputar produk barang nih?", "basa_basi", {}), | |
| ("Haloo Bot, mau tanya seputar produk jasa nih?", "basa_basi", {}), | |
| ("Haloo Bot, mau tanya seputar produk digital nih?", "basa_basi", {}), | |
| ("Makasih infonnya ya kak.", "basa_basi", {}), | |
| ("Makasih infonnya ya Gan.", "basa_basi", {}), | |
| ("Oke deh ntar saya kabari lagi.", "basa_basi", {}), | |
| ("Oke deh ntar saya coba ulangi lagi.", "basa_basi", {}), | |
| ("Saya bingung cara pakainya gimana.", "basa_basi", {}), | |
| ("Halo, saya PPK pengadaan yang mau cari penyedia" , "basa_basi", {}), | |
| ("Halo, saya KPA pengadaan yang mau cari penyedia" , "basa_basi", {}), | |
| ("Halo, saya SatGas pengadaan yang mau cari penyedia" , "basa_basi", {}), | |
| ("Websitenya e-Katalog lagi agak lemot ya.", "basa_basi", {}), | |
| ("Assalamualaikum.", "basa_basi", {}), | |
| ("Siap 86, laksanakan.", "basa_basi", {}), | |
| # Cari Produk | |
| ("Cari laptop lenovo ada?", "cari_produk", {}), | |
| ("Cari laptop Asus terbaru?", "cari_produk", {}), | |
| ("Cari laptop Acer murah?", "cari_produk", {}), | |
| ("Cari laptop dengan nilai TKDN diatas 80?", "cari_produk", {}), | |
| ("Cari laptop yang disediakan penyedia UMKM?", "cari_produk", {}), | |
| ("Butuh alat buat ngeprint undangan nih.", "cari_produk", {}), | |
| ("Ada jasa service AC gak disini?", "cari_produk", {}), | |
| ("Ada jasa service AC di Bandung kah?", "cari_produk", {}), | |
| ("Ada penyedia konsumsi di Bandung kah?", "cari_produk", {}), | |
| ("Yang jual semen di daerah Banten siapa ya?", "cari_produk", {}), | |
| ("Laptop yang 3 jutaan aja buat admin.", "cari_produk", {}), | |
| ("Cari switch hub cisco 24 port gigabit.", "cari_produk", {}), | |
| ("Cari jasa penyusun arsitektur SPBE.", "cari_produk", {}), | |
| ("Stok kertas A4 masih ready?", "cari_produk", {}), | |
| ("Kalau Printer yang merk Epson ada gak?", "cari_produk", {"product": "printer"}), | |
| ("Spill harga beton cor dong.", "cari_produk", {}), | |
| ("Spill harga kaos buat pelatihan dong.", "cari_produk", {}), | |
| ("Ada jasa pemasangan kabel FO ga ya", "cari_produk", {}), | |
| ("Ada jasa pemasangan tower ga ya", "cari_produk", {}), | |
| ("Di Sidoarjo ada laptop Apple ga ya", "cari_produk", {"product": "laptop"}), | |
| ] | |
| correct_count = 0 | |
| total_count = len(test_cases) | |
| # Tulis CSV | |
| with open(csv_file, mode='w', newline='', encoding='utf-8') as f: | |
| writer = csv.writer(f) | |
| # Header | |
| writer.writerow(["No", "Input User", "Context", "Expected Intent", "Predicted Intent", "Status"]) | |
| print(f"{'INPUT USER':<45} | {'PREDICTED':<12} | {'STATUS'}") | |
| print("-" * 75) | |
| for i, (text, expected, context) in enumerate(test_cases): | |
| mock_state = {"user_text": text, "collected_constraints": context} | |
| predicted = detector.detect(mock_state) | |
| is_correct = (predicted == expected) | |
| status = "PASS" if is_correct else "FAIL" | |
| if is_correct: correct_count += 1 | |
| # Print Console | |
| print(f"{text:<45} | {predicted:<12} | {status}") | |
| # Write CSV | |
| writer.writerow([i+1, text, str(context), expected, predicted, status]) | |
| accuracy = (correct_count / total_count) * 100 | |
| print("-" * 75) | |
| print(f"โ Report saved to: {csv_file}") | |
| print(f"๐ Accuracy: {accuracy:.1f}%") | |
| print("="*60) | |
| if __name__ == "__main__": | |
| run_eval_intent() |