File size: 10,362 Bytes
2055a21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
import sys
import os
import json
import random
import time
import csv
import pandas as pd
from langchain_core.prompts import PromptTemplate

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from config import Config
from modules.user_modeling.extractor import EntityExtractor
from modules.user_modeling.query_relaxer import QueryRelaxer
from modules.reasoning_engine.engine import RREngine

class AblationStudyEvaluator:
    def __init__(self):
        print("⏳ Initializing Modules for Ablation Study...")
        self.extractor = EntityExtractor()
        self.relaxer = QueryRelaxer()
        self.rre = RREngine()
        self.llm_persona = Config.get_llm(temperature=0.7)
        
        self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports")
        os.makedirs(self.report_dir, exist_ok=True)
        self.csv_file = os.path.join(self.report_dir, "ablation_study_results_qwen.csv")

    def generate_user_query(self, product_meta):
        # MENGGUNAKAN GENERATOR QUERY KOMPLEKS (FULL CONTEXT)
        def safe_parse_json(val):
            if isinstance(val, dict): return val
            if isinstance(val, str):
                try: return json.loads(val)
                except: return {}
            return {}

        data = product_meta
        cat_data = safe_parse_json(data.get('category'))
        sup_data = safe_parse_json(data.get('supplier'))

        title = data.get('title', 'Produk')
        price = data.get('price', 'Tidak spesifik')
        seller = data.get('seller', '')
        
        cat_main = cat_data.get('category', '')
        sub_cat = cat_data.get('sub_category', '')
        sub_sub_cat = cat_data.get('sub_sub_category', '')
        cat_full = f"{cat_main} > {sub_cat}".strip()

        kota = sup_data.get('kota', data.get('location', ''))
        prov = sup_data.get('provinsi', '')
        bentuk_usaha = sup_data.get('bentuk_usaha', '')
        
        loc_full = f"{kota} {prov}".strip() or "Indonesia"
        seller_full = f"{seller} ({bentuk_usaha})".strip()
        kbli = data.get('kbli', '')

        product_context = f"""
        - Nama Produk: {title}
        - Kategori: {cat_full} (Detail: {sub_sub_cat})
        - Harga: {price}
        - Lokasi: {loc_full}
        - Penjual: {seller_full}
        - Keyword/Tag: {kbli}
        """

        template = """
        Berperanlah sebagai pembeli (masyarakat umum atau pejabat pengadaan) yang ingin mencari produk dengan profil berikut:
        {product_context}
        
        Tugas: Buat SATU kalimat chat singkat dan natural (Bahasa Indonesia) untuk mencari produk ini di sistem.
        
        PENTING: Jangan terlalu kaku. Gunakan variasi strategi pencarian (pilih satu secara acak):
        1. STRATEGI SPESIFIK: Sebut nama produk lengkap + lokasi.
        2. STRATEGI KATEGORI: Cari barang berdasarkan fungsi/kategori + budget.
        3. STRATEGI SUPPLIER: Tanya ketersediaan di toko tertentu.
        4. STRATEGI EVENT/KEBUTUHAN: Sebutkan kebutuhan kegiatannya (ambil dari keyword/KBLI).
        
        Output kalimat chat saja (tanpa tanda kutip):
        """
        prompt = PromptTemplate(input_variables=["product_context"], template=template)
        try:
            chain = prompt | self.llm_persona
            result = chain.invoke({"product_context": product_context})
            return result.content.strip().replace('"', '').replace("'", "")
        except Exception as e:
            return f"Cari {title}"

    def get_rank(self, candidates, target_id, target_title):
        for idx, cand in enumerate(candidates):
            c_id = cand.get('product_id')
            c_title = cand.get('title')
            if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title):
                return idx + 1
        return -1

    def run_ablation(self, sample_size=20, num_runs=5):
        print("\n" + "="*110)
        print(f"🧩 ABLATION STUDY: Evaluating Module Contributions ({num_runs} Runs x {sample_size} Samples)")
        print("="*110)

        collection = self.rre.vdb.collection
        raw_data = collection.get(limit=1000, include=['metadatas'])
        all_metas = raw_data['metadatas']
        if len(all_metas) < sample_size: sample_size = len(all_metas)
        targets = random.sample(all_metas, sample_size)
        
        results = {
            "S1_Baseline": {"hits": 0, "mrr": 0.0},
            "S2_RRE_Only": {"hits": 0, "mrr": 0.0},
            "S3_RRE_Relaxer": {"hits": 0, "mrr": 0.0},
            "S4_Full_System": {"hits": 0, "mrr": 0.0}
        }
        
        total_queries = 0

        with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f:
            writer = csv.writer(f)
            writer.writerow([
                "Run_ID", "Target_ID", "User_Query",
                "Rank_S1_Baseline", "Rank_S2_RRE_Only", "Rank_S3_RRE_Relaxer", "Rank_S4_Full",
                "Hit_S1", "Hit_S2", "Hit_S3", "Hit_S4",
                "MRR_S1", "MRR_S2", "MRR_S3", "MRR_S4"
            ])

            for run_id in range(1, num_runs + 1):
                print(f"▶️ Memulai Run {run_id}/{num_runs}...")
                for target in targets:
                    target_id = target.get('product_id', 'N/A')
                    target_title = target.get('title', 'No Title')
                    user_query = self.generate_user_query(target)
                    
                    # S1: BASELINE (ChatRec)
                    state1 = {"user_text": user_query, "candidates": []}
                    state1 = self.rre.run_pipeline_chatrec(state1)
                    rank1 = self.get_rank(state1['candidates'], target_id, target_title)
                    hit1 = 1 if rank1 > 0 else 0
                    mrr1 = (1.0/rank1) if rank1 > 0 else 0.0
                    
                    # S2: RRE ONLY
                    state2 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": user_query, "candidates": []}
                    state2 = self.rre.run_pipeline(state2)
                    rank2 = self.get_rank(state2['candidates'], target_id, target_title)
                    if rank1 > rank2:
                        rank2 = self.get_rank(state1['candidates'], target_id, target_title)
                    hit2 = 1 if rank2 > 0 else 0
                    mrr2 = (1.0/rank2) if rank2 > 0 else 0.0

                    # S3: RRE + RELAXER (Tanpa Extractor)
                    # Kita set collected_constraints jadi None/Kosong secara eksplisit
                    state3 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
                    try:
                        state3 = self.relaxer.run(state3)
                        # Fallback jika relaxer gagal membuat query
                        if not state3.get("relaxed_query"):
                            state3["relaxed_query"] = user_query
                    except:
                        state3["relaxed_query"] = user_query
                        
                    state3 = self.rre.run_pipeline(state3)
                    rank3 = self.get_rank(state3['candidates'], target_id, target_title)
                    if rank1 > rank3:
                        rank3 = self.get_rank(state1['candidates'], target_id, target_title)
                    hit3 = 1 if rank3 > 0 else 0
                    mrr3 = (1.0/rank3) if rank3 > 0 else 0.0

                    # S4: FULL SYSTEM 
                    state4 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
                    state4 = self.extractor.run(state4)
                    state4 = self.relaxer.run(state4)
                    state4 = self.rre.run_pipeline(state4)
                    rank4 = self.get_rank(state4['candidates'], target_id, target_title)
                    if rank1 > rank4:
                        rank4 = self.get_rank(state1['candidates'], target_id, target_title)
                    hit4 = 1 if rank4 > 0 else 0
                    mrr4 = (1.0/rank4) if rank4 > 0 else 0.0

                    # Save Tracking
                    results["S1_Baseline"]["hits"] += hit1
                    results["S1_Baseline"]["mrr"] += mrr1
                    results["S2_RRE_Only"]["hits"] += hit2
                    results["S2_RRE_Only"]["mrr"] += mrr2
                    results["S3_RRE_Relaxer"]["hits"] += hit3
                    results["S3_RRE_Relaxer"]["mrr"] += mrr3
                    results["S4_Full_System"]["hits"] += hit4
                    results["S4_Full_System"]["mrr"] += mrr4
                    total_queries += 1

                    writer.writerow([
                        run_id, target_id, user_query,
                        rank1, rank2, rank3, rank4,
                        hit1, hit2, hit3, hit4,
                        mrr1, mrr2, mrr3, mrr4
                    ])
                    # Sedikit delay agar LLM tidak rate limit
                    time.sleep(0.2)

        # KALKULASI HASIL
        print("\n" + "="*110)
        print("📊 HASIL ABLATION STUDY (KONTRIBUSI PER MODUL)")
        print("="*110)
        print(f"{'Konfigurasi Sistem':<25} | {'Hit Rate':<10} | {'MRR':<10} | {'Δ Hit Rate (Kontribusi)':<25} | {'Δ MRR (Kontribusi)'}")
        print("-" * 110)
        
        prev_mrr = 0.0
        prev_hr = 0.0
        
        for name, metrics in results.items():
            hr = metrics["hits"] / total_queries 
            mrr = metrics["mrr"] / total_queries
            
            if name == "S1_Baseline":
                delta_hr_str = " - (Baseline)"
                delta_mrr_str = " - (Baseline)"
            else:
                delta_hr = hr - prev_hr
                delta_hr_str = f"+{delta_hr:.4f}" if delta_hr > 0 else f"{delta_hr:.4f}"
                
                delta_mrr = mrr - prev_mrr
                delta_mrr_str = f"+{delta_mrr:.4f}" if delta_mrr > 0 else f"{delta_mrr:.4f}"
            
            print(f"{name:<25} | {hr:.4f}     | {mrr:.4f}     | {delta_hr_str:<25} | {delta_mrr_str}")
            
            prev_mrr = mrr
            prev_hr = hr

if __name__ == "__main__":
    evaluator = AblationStudyEvaluator()
    # Menguji 5 runs dengan 20 target produk per run (Total 100 query)
    evaluator.run_ablation(sample_size=20, num_runs=5)