File size: 4,770 Bytes
2055a21 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | import sys
import os
import csv
import time
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from modules.dialog_manager.intent import IntentDetector
def run_eval_intent():
print("\n" + "="*60)
print("๐ EVALUASI SKENARIO 1: INTENT DETECTION (With CSV Report)")
print("="*60)
# Init Output Folder
report_dir = os.path.join(os.path.dirname(__file__), "reports")
os.makedirs(report_dir, exist_ok=True)
csv_file = os.path.join(report_dir, "report_01_intent_mistral.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_llama3.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_qwen3.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_gemma3.csv")
try:
detector = IntentDetector()
print("โ
IntentDetector Ready.\n")
except Exception as e:
print(f"โ Init Failed: {e}")
return
test_cases = [
# Basa-basi
("Pagi min, on gak?", "basa_basi", {}),
("Siang min, on gak?", "basa_basi", {}),
("Malam min, on gak?", "basa_basi", {}),
("Selamat pagi Bapak/Ibu admin.", "basa_basi", {}),
("Selamat siang Bapak/Ibu admin.", "basa_basi", {}),
("Selamat malam Bapak/Ibu admin.", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk barang nih?", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk jasa nih?", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk digital nih?", "basa_basi", {}),
("Makasih infonnya ya kak.", "basa_basi", {}),
("Makasih infonnya ya Gan.", "basa_basi", {}),
("Oke deh ntar saya kabari lagi.", "basa_basi", {}),
("Oke deh ntar saya coba ulangi lagi.", "basa_basi", {}),
("Saya bingung cara pakainya gimana.", "basa_basi", {}),
("Halo, saya PPK pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Halo, saya KPA pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Halo, saya SatGas pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Websitenya e-Katalog lagi agak lemot ya.", "basa_basi", {}),
("Assalamualaikum.", "basa_basi", {}),
("Siap 86, laksanakan.", "basa_basi", {}),
# Cari Produk
("Cari laptop lenovo ada?", "cari_produk", {}),
("Cari laptop Asus terbaru?", "cari_produk", {}),
("Cari laptop Acer murah?", "cari_produk", {}),
("Cari laptop dengan nilai TKDN diatas 80?", "cari_produk", {}),
("Cari laptop yang disediakan penyedia UMKM?", "cari_produk", {}),
("Butuh alat buat ngeprint undangan nih.", "cari_produk", {}),
("Ada jasa service AC gak disini?", "cari_produk", {}),
("Ada jasa service AC di Bandung kah?", "cari_produk", {}),
("Ada penyedia konsumsi di Bandung kah?", "cari_produk", {}),
("Yang jual semen di daerah Banten siapa ya?", "cari_produk", {}),
("Laptop yang 3 jutaan aja buat admin.", "cari_produk", {}),
("Cari switch hub cisco 24 port gigabit.", "cari_produk", {}),
("Cari jasa penyusun arsitektur SPBE.", "cari_produk", {}),
("Stok kertas A4 masih ready?", "cari_produk", {}),
("Kalau Printer yang merk Epson ada gak?", "cari_produk", {"product": "printer"}),
("Spill harga beton cor dong.", "cari_produk", {}),
("Spill harga kaos buat pelatihan dong.", "cari_produk", {}),
("Ada jasa pemasangan kabel FO ga ya", "cari_produk", {}),
("Ada jasa pemasangan tower ga ya", "cari_produk", {}),
("Di Sidoarjo ada laptop Apple ga ya", "cari_produk", {"product": "laptop"}),
]
correct_count = 0
total_count = len(test_cases)
# Tulis CSV
with open(csv_file, mode='w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
# Header
writer.writerow(["No", "Input User", "Context", "Expected Intent", "Predicted Intent", "Status"])
print(f"{'INPUT USER':<45} | {'PREDICTED':<12} | {'STATUS'}")
print("-" * 75)
for i, (text, expected, context) in enumerate(test_cases):
mock_state = {"user_text": text, "collected_constraints": context}
predicted = detector.detect(mock_state)
is_correct = (predicted == expected)
status = "PASS" if is_correct else "FAIL"
if is_correct: correct_count += 1
# Print Console
print(f"{text:<45} | {predicted:<12} | {status}")
# Write CSV
writer.writerow([i+1, text, str(context), expected, predicted, status])
accuracy = (correct_count / total_count) * 100
print("-" * 75)
print(f"โ
Report saved to: {csv_file}")
print(f"๐ Accuracy: {accuracy:.1f}%")
print("="*60)
if __name__ == "__main__":
run_eval_intent() |