import sys import os import csv import time sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from modules.dialog_manager.intent import IntentDetector def run_eval_intent(): print("\n" + "="*60) print("📊 EVALUASI SKENARIO 1: INTENT DETECTION (With CSV Report)") print("="*60) # Init Output Folder report_dir = os.path.join(os.path.dirname(__file__), "reports") os.makedirs(report_dir, exist_ok=True) csv_file = os.path.join(report_dir, "report_01_intent_mistral.csv") # csv_file = os.path.join(report_dir, "report_01_intent_llama3.csv") # csv_file = os.path.join(report_dir, "report_01_intent_qwen3.csv") # csv_file = os.path.join(report_dir, "report_01_intent_gemma3.csv") try: detector = IntentDetector() print("✅ IntentDetector Ready.\n") except Exception as e: print(f"❌ Init Failed: {e}") return test_cases = [ # Basa-basi ("Pagi min, on gak?", "basa_basi", {}), ("Siang min, on gak?", "basa_basi", {}), ("Malam min, on gak?", "basa_basi", {}), ("Selamat pagi Bapak/Ibu admin.", "basa_basi", {}), ("Selamat siang Bapak/Ibu admin.", "basa_basi", {}), ("Selamat malam Bapak/Ibu admin.", "basa_basi", {}), ("Haloo Bot, mau tanya seputar produk barang nih?", "basa_basi", {}), ("Haloo Bot, mau tanya seputar produk jasa nih?", "basa_basi", {}), ("Haloo Bot, mau tanya seputar produk digital nih?", "basa_basi", {}), ("Makasih infonnya ya kak.", "basa_basi", {}), ("Makasih infonnya ya Gan.", "basa_basi", {}), ("Oke deh ntar saya kabari lagi.", "basa_basi", {}), ("Oke deh ntar saya coba ulangi lagi.", "basa_basi", {}), ("Saya bingung cara pakainya gimana.", "basa_basi", {}), ("Halo, saya PPK pengadaan yang mau cari penyedia" , "basa_basi", {}), ("Halo, saya KPA pengadaan yang mau cari penyedia" , "basa_basi", {}), ("Halo, saya SatGas pengadaan yang mau cari penyedia" , "basa_basi", {}), ("Websitenya e-Katalog lagi agak lemot ya.", "basa_basi", {}), ("Assalamualaikum.", "basa_basi", {}), ("Siap 86, laksanakan.", "basa_basi", {}), # Cari Produk ("Cari laptop lenovo ada?", "cari_produk", {}), ("Cari laptop Asus terbaru?", "cari_produk", {}), ("Cari laptop Acer murah?", "cari_produk", {}), ("Cari laptop dengan nilai TKDN diatas 80?", "cari_produk", {}), ("Cari laptop yang disediakan penyedia UMKM?", "cari_produk", {}), ("Butuh alat buat ngeprint undangan nih.", "cari_produk", {}), ("Ada jasa service AC gak disini?", "cari_produk", {}), ("Ada jasa service AC di Bandung kah?", "cari_produk", {}), ("Ada penyedia konsumsi di Bandung kah?", "cari_produk", {}), ("Yang jual semen di daerah Banten siapa ya?", "cari_produk", {}), ("Laptop yang 3 jutaan aja buat admin.", "cari_produk", {}), ("Cari switch hub cisco 24 port gigabit.", "cari_produk", {}), ("Cari jasa penyusun arsitektur SPBE.", "cari_produk", {}), ("Stok kertas A4 masih ready?", "cari_produk", {}), ("Kalau Printer yang merk Epson ada gak?", "cari_produk", {"product": "printer"}), ("Spill harga beton cor dong.", "cari_produk", {}), ("Spill harga kaos buat pelatihan dong.", "cari_produk", {}), ("Ada jasa pemasangan kabel FO ga ya", "cari_produk", {}), ("Ada jasa pemasangan tower ga ya", "cari_produk", {}), ("Di Sidoarjo ada laptop Apple ga ya", "cari_produk", {"product": "laptop"}), ] correct_count = 0 total_count = len(test_cases) # Tulis CSV with open(csv_file, mode='w', newline='', encoding='utf-8') as f: writer = csv.writer(f) # Header writer.writerow(["No", "Input User", "Context", "Expected Intent", "Predicted Intent", "Status"]) print(f"{'INPUT USER':<45} | {'PREDICTED':<12} | {'STATUS'}") print("-" * 75) for i, (text, expected, context) in enumerate(test_cases): mock_state = {"user_text": text, "collected_constraints": context} predicted = detector.detect(mock_state) is_correct = (predicted == expected) status = "PASS" if is_correct else "FAIL" if is_correct: correct_count += 1 # Print Console print(f"{text:<45} | {predicted:<12} | {status}") # Write CSV writer.writerow([i+1, text, str(context), expected, predicted, status]) accuracy = (correct_count / total_count) * 100 print("-" * 75) print(f"✅ Report saved to: {csv_file}") print(f"📊 Accuracy: {accuracy:.1f}%") print("="*60) if __name__ == "__main__": run_eval_intent()