llmproccrs / tests /eval_01_intent.py
aitek230telu's picture
Upload 58 files
2055a21 verified
Raw History Blame Contribute Delete
4.77 kB
import sys
import os
import csv
import time
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from modules.dialog_manager.intent import IntentDetector
def run_eval_intent():
print("\n" + "="*60)
print("๐Ÿ“Š EVALUASI SKENARIO 1: INTENT DETECTION (With CSV Report)")
print("="*60)
# Init Output Folder
report_dir = os.path.join(os.path.dirname(__file__), "reports")
os.makedirs(report_dir, exist_ok=True)
csv_file = os.path.join(report_dir, "report_01_intent_mistral.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_llama3.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_qwen3.csv")
# csv_file = os.path.join(report_dir, "report_01_intent_gemma3.csv")
try:
detector = IntentDetector()
print("โœ… IntentDetector Ready.\n")
except Exception as e:
print(f"โŒ Init Failed: {e}")
return
test_cases = [
# Basa-basi
("Pagi min, on gak?", "basa_basi", {}),
("Siang min, on gak?", "basa_basi", {}),
("Malam min, on gak?", "basa_basi", {}),
("Selamat pagi Bapak/Ibu admin.", "basa_basi", {}),
("Selamat siang Bapak/Ibu admin.", "basa_basi", {}),
("Selamat malam Bapak/Ibu admin.", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk barang nih?", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk jasa nih?", "basa_basi", {}),
("Haloo Bot, mau tanya seputar produk digital nih?", "basa_basi", {}),
("Makasih infonnya ya kak.", "basa_basi", {}),
("Makasih infonnya ya Gan.", "basa_basi", {}),
("Oke deh ntar saya kabari lagi.", "basa_basi", {}),
("Oke deh ntar saya coba ulangi lagi.", "basa_basi", {}),
("Saya bingung cara pakainya gimana.", "basa_basi", {}),
("Halo, saya PPK pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Halo, saya KPA pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Halo, saya SatGas pengadaan yang mau cari penyedia" , "basa_basi", {}),
("Websitenya e-Katalog lagi agak lemot ya.", "basa_basi", {}),
("Assalamualaikum.", "basa_basi", {}),
("Siap 86, laksanakan.", "basa_basi", {}),
# Cari Produk
("Cari laptop lenovo ada?", "cari_produk", {}),
("Cari laptop Asus terbaru?", "cari_produk", {}),
("Cari laptop Acer murah?", "cari_produk", {}),
("Cari laptop dengan nilai TKDN diatas 80?", "cari_produk", {}),
("Cari laptop yang disediakan penyedia UMKM?", "cari_produk", {}),
("Butuh alat buat ngeprint undangan nih.", "cari_produk", {}),
("Ada jasa service AC gak disini?", "cari_produk", {}),
("Ada jasa service AC di Bandung kah?", "cari_produk", {}),
("Ada penyedia konsumsi di Bandung kah?", "cari_produk", {}),
("Yang jual semen di daerah Banten siapa ya?", "cari_produk", {}),
("Laptop yang 3 jutaan aja buat admin.", "cari_produk", {}),
("Cari switch hub cisco 24 port gigabit.", "cari_produk", {}),
("Cari jasa penyusun arsitektur SPBE.", "cari_produk", {}),
("Stok kertas A4 masih ready?", "cari_produk", {}),
("Kalau Printer yang merk Epson ada gak?", "cari_produk", {"product": "printer"}),
("Spill harga beton cor dong.", "cari_produk", {}),
("Spill harga kaos buat pelatihan dong.", "cari_produk", {}),
("Ada jasa pemasangan kabel FO ga ya", "cari_produk", {}),
("Ada jasa pemasangan tower ga ya", "cari_produk", {}),
("Di Sidoarjo ada laptop Apple ga ya", "cari_produk", {"product": "laptop"}),
]
correct_count = 0
total_count = len(test_cases)
# Tulis CSV
with open(csv_file, mode='w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
# Header
writer.writerow(["No", "Input User", "Context", "Expected Intent", "Predicted Intent", "Status"])
print(f"{'INPUT USER':<45} | {'PREDICTED':<12} | {'STATUS'}")
print("-" * 75)
for i, (text, expected, context) in enumerate(test_cases):
mock_state = {"user_text": text, "collected_constraints": context}
predicted = detector.detect(mock_state)
is_correct = (predicted == expected)
status = "PASS" if is_correct else "FAIL"
if is_correct: correct_count += 1
# Print Console
print(f"{text:<45} | {predicted:<12} | {status}")
# Write CSV
writer.writerow([i+1, text, str(context), expected, predicted, status])
accuracy = (correct_count / total_count) * 100
print("-" * 75)
print(f"โœ… Report saved to: {csv_file}")
print(f"๐Ÿ“Š Accuracy: {accuracy:.1f}%")
print("="*60)
if __name__ == "__main__":
run_eval_intent()