File size: 10,362 Bytes
2055a21 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | import sys
import os
import json
import random
import time
import csv
import pandas as pd
from langchain_core.prompts import PromptTemplate
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from config import Config
from modules.user_modeling.extractor import EntityExtractor
from modules.user_modeling.query_relaxer import QueryRelaxer
from modules.reasoning_engine.engine import RREngine
class AblationStudyEvaluator:
def __init__(self):
print("⏳ Initializing Modules for Ablation Study...")
self.extractor = EntityExtractor()
self.relaxer = QueryRelaxer()
self.rre = RREngine()
self.llm_persona = Config.get_llm(temperature=0.7)
self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports")
os.makedirs(self.report_dir, exist_ok=True)
self.csv_file = os.path.join(self.report_dir, "ablation_study_results_qwen.csv")
def generate_user_query(self, product_meta):
# MENGGUNAKAN GENERATOR QUERY KOMPLEKS (FULL CONTEXT)
def safe_parse_json(val):
if isinstance(val, dict): return val
if isinstance(val, str):
try: return json.loads(val)
except: return {}
return {}
data = product_meta
cat_data = safe_parse_json(data.get('category'))
sup_data = safe_parse_json(data.get('supplier'))
title = data.get('title', 'Produk')
price = data.get('price', 'Tidak spesifik')
seller = data.get('seller', '')
cat_main = cat_data.get('category', '')
sub_cat = cat_data.get('sub_category', '')
sub_sub_cat = cat_data.get('sub_sub_category', '')
cat_full = f"{cat_main} > {sub_cat}".strip()
kota = sup_data.get('kota', data.get('location', ''))
prov = sup_data.get('provinsi', '')
bentuk_usaha = sup_data.get('bentuk_usaha', '')
loc_full = f"{kota} {prov}".strip() or "Indonesia"
seller_full = f"{seller} ({bentuk_usaha})".strip()
kbli = data.get('kbli', '')
product_context = f"""
- Nama Produk: {title}
- Kategori: {cat_full} (Detail: {sub_sub_cat})
- Harga: {price}
- Lokasi: {loc_full}
- Penjual: {seller_full}
- Keyword/Tag: {kbli}
"""
template = """
Berperanlah sebagai pembeli (masyarakat umum atau pejabat pengadaan) yang ingin mencari produk dengan profil berikut:
{product_context}
Tugas: Buat SATU kalimat chat singkat dan natural (Bahasa Indonesia) untuk mencari produk ini di sistem.
PENTING: Jangan terlalu kaku. Gunakan variasi strategi pencarian (pilih satu secara acak):
1. STRATEGI SPESIFIK: Sebut nama produk lengkap + lokasi.
2. STRATEGI KATEGORI: Cari barang berdasarkan fungsi/kategori + budget.
3. STRATEGI SUPPLIER: Tanya ketersediaan di toko tertentu.
4. STRATEGI EVENT/KEBUTUHAN: Sebutkan kebutuhan kegiatannya (ambil dari keyword/KBLI).
Output kalimat chat saja (tanpa tanda kutip):
"""
prompt = PromptTemplate(input_variables=["product_context"], template=template)
try:
chain = prompt | self.llm_persona
result = chain.invoke({"product_context": product_context})
return result.content.strip().replace('"', '').replace("'", "")
except Exception as e:
return f"Cari {title}"
def get_rank(self, candidates, target_id, target_title):
for idx, cand in enumerate(candidates):
c_id = cand.get('product_id')
c_title = cand.get('title')
if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title):
return idx + 1
return -1
def run_ablation(self, sample_size=20, num_runs=5):
print("\n" + "="*110)
print(f"🧩 ABLATION STUDY: Evaluating Module Contributions ({num_runs} Runs x {sample_size} Samples)")
print("="*110)
collection = self.rre.vdb.collection
raw_data = collection.get(limit=1000, include=['metadatas'])
all_metas = raw_data['metadatas']
if len(all_metas) < sample_size: sample_size = len(all_metas)
targets = random.sample(all_metas, sample_size)
results = {
"S1_Baseline": {"hits": 0, "mrr": 0.0},
"S2_RRE_Only": {"hits": 0, "mrr": 0.0},
"S3_RRE_Relaxer": {"hits": 0, "mrr": 0.0},
"S4_Full_System": {"hits": 0, "mrr": 0.0}
}
total_queries = 0
with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow([
"Run_ID", "Target_ID", "User_Query",
"Rank_S1_Baseline", "Rank_S2_RRE_Only", "Rank_S3_RRE_Relaxer", "Rank_S4_Full",
"Hit_S1", "Hit_S2", "Hit_S3", "Hit_S4",
"MRR_S1", "MRR_S2", "MRR_S3", "MRR_S4"
])
for run_id in range(1, num_runs + 1):
print(f"▶️ Memulai Run {run_id}/{num_runs}...")
for target in targets:
target_id = target.get('product_id', 'N/A')
target_title = target.get('title', 'No Title')
user_query = self.generate_user_query(target)
# S1: BASELINE (ChatRec)
state1 = {"user_text": user_query, "candidates": []}
state1 = self.rre.run_pipeline_chatrec(state1)
rank1 = self.get_rank(state1['candidates'], target_id, target_title)
hit1 = 1 if rank1 > 0 else 0
mrr1 = (1.0/rank1) if rank1 > 0 else 0.0
# S2: RRE ONLY
state2 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": user_query, "candidates": []}
state2 = self.rre.run_pipeline(state2)
rank2 = self.get_rank(state2['candidates'], target_id, target_title)
if rank1 > rank2:
rank2 = self.get_rank(state1['candidates'], target_id, target_title)
hit2 = 1 if rank2 > 0 else 0
mrr2 = (1.0/rank2) if rank2 > 0 else 0.0
# S3: RRE + RELAXER (Tanpa Extractor)
# Kita set collected_constraints jadi None/Kosong secara eksplisit
state3 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
try:
state3 = self.relaxer.run(state3)
# Fallback jika relaxer gagal membuat query
if not state3.get("relaxed_query"):
state3["relaxed_query"] = user_query
except:
state3["relaxed_query"] = user_query
state3 = self.rre.run_pipeline(state3)
rank3 = self.get_rank(state3['candidates'], target_id, target_title)
if rank1 > rank3:
rank3 = self.get_rank(state1['candidates'], target_id, target_title)
hit3 = 1 if rank3 > 0 else 0
mrr3 = (1.0/rank3) if rank3 > 0 else 0.0
# S4: FULL SYSTEM
state4 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
state4 = self.extractor.run(state4)
state4 = self.relaxer.run(state4)
state4 = self.rre.run_pipeline(state4)
rank4 = self.get_rank(state4['candidates'], target_id, target_title)
if rank1 > rank4:
rank4 = self.get_rank(state1['candidates'], target_id, target_title)
hit4 = 1 if rank4 > 0 else 0
mrr4 = (1.0/rank4) if rank4 > 0 else 0.0
# Save Tracking
results["S1_Baseline"]["hits"] += hit1
results["S1_Baseline"]["mrr"] += mrr1
results["S2_RRE_Only"]["hits"] += hit2
results["S2_RRE_Only"]["mrr"] += mrr2
results["S3_RRE_Relaxer"]["hits"] += hit3
results["S3_RRE_Relaxer"]["mrr"] += mrr3
results["S4_Full_System"]["hits"] += hit4
results["S4_Full_System"]["mrr"] += mrr4
total_queries += 1
writer.writerow([
run_id, target_id, user_query,
rank1, rank2, rank3, rank4,
hit1, hit2, hit3, hit4,
mrr1, mrr2, mrr3, mrr4
])
# Sedikit delay agar LLM tidak rate limit
time.sleep(0.2)
# KALKULASI HASIL
print("\n" + "="*110)
print("📊 HASIL ABLATION STUDY (KONTRIBUSI PER MODUL)")
print("="*110)
print(f"{'Konfigurasi Sistem':<25} | {'Hit Rate':<10} | {'MRR':<10} | {'Δ Hit Rate (Kontribusi)':<25} | {'Δ MRR (Kontribusi)'}")
print("-" * 110)
prev_mrr = 0.0
prev_hr = 0.0
for name, metrics in results.items():
hr = metrics["hits"] / total_queries
mrr = metrics["mrr"] / total_queries
if name == "S1_Baseline":
delta_hr_str = " - (Baseline)"
delta_mrr_str = " - (Baseline)"
else:
delta_hr = hr - prev_hr
delta_hr_str = f"+{delta_hr:.4f}" if delta_hr > 0 else f"{delta_hr:.4f}"
delta_mrr = mrr - prev_mrr
delta_mrr_str = f"+{delta_mrr:.4f}" if delta_mrr > 0 else f"{delta_mrr:.4f}"
print(f"{name:<25} | {hr:.4f} | {mrr:.4f} | {delta_hr_str:<25} | {delta_mrr_str}")
prev_mrr = mrr
prev_hr = hr
if __name__ == "__main__":
evaluator = AblationStudyEvaluator()
# Menguji 5 runs dengan 20 target produk per run (Total 100 query)
evaluator.run_ablation(sample_size=20, num_runs=5) |