File size: 4,770 Bytes
2055a21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import sys
import os
import csv
import time

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from modules.dialog_manager.intent import IntentDetector

def run_eval_intent():
    print("\n" + "="*60)
    print("๐Ÿ“Š EVALUASI SKENARIO 1: INTENT DETECTION (With CSV Report)")
    print("="*60)
    
    # Init Output Folder
    report_dir = os.path.join(os.path.dirname(__file__), "reports")
    os.makedirs(report_dir, exist_ok=True)
    csv_file = os.path.join(report_dir, "report_01_intent_mistral.csv")
    # csv_file = os.path.join(report_dir, "report_01_intent_llama3.csv")
    # csv_file = os.path.join(report_dir, "report_01_intent_qwen3.csv")
    # csv_file = os.path.join(report_dir, "report_01_intent_gemma3.csv")

    try:
        detector = IntentDetector()
        print("โœ… IntentDetector Ready.\n")
    except Exception as e:
        print(f"โŒ Init Failed: {e}")
        return

    test_cases = [
        # Basa-basi
        ("Pagi min, on gak?", "basa_basi", {}),
 	("Siang min, on gak?", "basa_basi", {}),
	("Malam min, on gak?", "basa_basi", {}),
        ("Selamat pagi Bapak/Ibu admin.", "basa_basi", {}),
	("Selamat siang Bapak/Ibu admin.", "basa_basi", {}),
	("Selamat malam Bapak/Ibu admin.", "basa_basi", {}),
        ("Haloo Bot, mau tanya seputar produk barang nih?", "basa_basi", {}),
	("Haloo Bot, mau tanya seputar produk jasa nih?", "basa_basi", {}),
	("Haloo Bot, mau tanya seputar produk digital nih?", "basa_basi", {}),
        ("Makasih infonnya ya kak.", "basa_basi", {}),
	("Makasih infonnya ya Gan.", "basa_basi", {}),
        ("Oke deh ntar saya kabari lagi.", "basa_basi", {}),
        ("Oke deh ntar saya coba ulangi lagi.", "basa_basi", {}),
        ("Saya bingung cara pakainya gimana.", "basa_basi", {}),
        ("Halo, saya PPK pengadaan yang mau cari penyedia" , "basa_basi", {}),
        ("Halo, saya KPA pengadaan yang mau cari penyedia" , "basa_basi", {}),
        ("Halo, saya SatGas pengadaan yang mau cari penyedia" , "basa_basi", {}),
        ("Websitenya e-Katalog lagi agak lemot ya.", "basa_basi", {}),
        ("Assalamualaikum.", "basa_basi", {}),
        ("Siap 86, laksanakan.", "basa_basi", {}),

        # Cari Produk
        ("Cari laptop lenovo ada?", "cari_produk", {}),
        ("Cari laptop Asus terbaru?", "cari_produk", {}),
	("Cari laptop Acer murah?", "cari_produk", {}),
	("Cari laptop dengan nilai TKDN diatas 80?", "cari_produk", {}),
	("Cari laptop yang disediakan penyedia UMKM?", "cari_produk", {}),
        ("Butuh alat buat ngeprint undangan nih.", "cari_produk", {}),
        ("Ada jasa service AC gak disini?", "cari_produk", {}),
	("Ada jasa service AC di Bandung kah?", "cari_produk", {}),
	("Ada penyedia konsumsi di Bandung kah?", "cari_produk", {}),
        ("Yang jual semen di daerah Banten siapa ya?", "cari_produk", {}),
        ("Laptop yang 3 jutaan aja buat admin.", "cari_produk", {}),
        ("Cari switch hub cisco 24 port gigabit.", "cari_produk", {}),
	("Cari jasa penyusun arsitektur SPBE.", "cari_produk", {}),
        ("Stok kertas A4 masih ready?", "cari_produk", {}),
        ("Kalau Printer yang merk Epson ada gak?", "cari_produk", {"product": "printer"}),
        ("Spill harga beton cor dong.", "cari_produk", {}),
        ("Spill harga kaos buat pelatihan dong.", "cari_produk", {}),
        ("Ada jasa pemasangan kabel FO ga ya", "cari_produk", {}),
        ("Ada jasa pemasangan tower ga ya", "cari_produk", {}),
        ("Di Sidoarjo ada laptop Apple ga ya", "cari_produk", {"product": "laptop"}), 
    ]

    correct_count = 0
    total_count = len(test_cases)
    
    # Tulis CSV
    with open(csv_file, mode='w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        # Header
        writer.writerow(["No", "Input User", "Context", "Expected Intent", "Predicted Intent", "Status"])
        
        print(f"{'INPUT USER':<45} | {'PREDICTED':<12} | {'STATUS'}")
        print("-" * 75)

        for i, (text, expected, context) in enumerate(test_cases):
            mock_state = {"user_text": text, "collected_constraints": context}
            predicted = detector.detect(mock_state)
            
            is_correct = (predicted == expected)
            status = "PASS" if is_correct else "FAIL"
            if is_correct: correct_count += 1
            
            # Print Console
            print(f"{text:<45} | {predicted:<12} | {status}")
            
            # Write CSV
            writer.writerow([i+1, text, str(context), expected, predicted, status])

    accuracy = (correct_count / total_count) * 100
    print("-" * 75)
    print(f"โœ… Report saved to: {csv_file}")
    print(f"๐Ÿ“Š Accuracy: {accuracy:.1f}%")
    print("="*60)

if __name__ == "__main__":
    run_eval_intent()