llmproccrs / tests /ablation_study.py
aitek230telu's picture
Upload 58 files
2055a21 verified
Raw History Blame Contribute Delete
10.4 kB
import sys
import os
import json
import random
import time
import csv
import pandas as pd
from langchain_core.prompts import PromptTemplate
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from config import Config
from modules.user_modeling.extractor import EntityExtractor
from modules.user_modeling.query_relaxer import QueryRelaxer
from modules.reasoning_engine.engine import RREngine
class AblationStudyEvaluator:
def __init__(self):
print("⏳ Initializing Modules for Ablation Study...")
self.extractor = EntityExtractor()
self.relaxer = QueryRelaxer()
self.rre = RREngine()
self.llm_persona = Config.get_llm(temperature=0.7)
self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports")
os.makedirs(self.report_dir, exist_ok=True)
self.csv_file = os.path.join(self.report_dir, "ablation_study_results_qwen.csv")
def generate_user_query(self, product_meta):
# MENGGUNAKAN GENERATOR QUERY KOMPLEKS (FULL CONTEXT)
def safe_parse_json(val):
if isinstance(val, dict): return val
if isinstance(val, str):
try: return json.loads(val)
except: return {}
return {}
data = product_meta
cat_data = safe_parse_json(data.get('category'))
sup_data = safe_parse_json(data.get('supplier'))
title = data.get('title', 'Produk')
price = data.get('price', 'Tidak spesifik')
seller = data.get('seller', '')
cat_main = cat_data.get('category', '')
sub_cat = cat_data.get('sub_category', '')
sub_sub_cat = cat_data.get('sub_sub_category', '')
cat_full = f"{cat_main} > {sub_cat}".strip()
kota = sup_data.get('kota', data.get('location', ''))
prov = sup_data.get('provinsi', '')
bentuk_usaha = sup_data.get('bentuk_usaha', '')
loc_full = f"{kota} {prov}".strip() or "Indonesia"
seller_full = f"{seller} ({bentuk_usaha})".strip()
kbli = data.get('kbli', '')
product_context = f"""
- Nama Produk: {title}
- Kategori: {cat_full} (Detail: {sub_sub_cat})
- Harga: {price}
- Lokasi: {loc_full}
- Penjual: {seller_full}
- Keyword/Tag: {kbli}
"""
template = """
Berperanlah sebagai pembeli (masyarakat umum atau pejabat pengadaan) yang ingin mencari produk dengan profil berikut:
{product_context}
Tugas: Buat SATU kalimat chat singkat dan natural (Bahasa Indonesia) untuk mencari produk ini di sistem.
PENTING: Jangan terlalu kaku. Gunakan variasi strategi pencarian (pilih satu secara acak):
1. STRATEGI SPESIFIK: Sebut nama produk lengkap + lokasi.
2. STRATEGI KATEGORI: Cari barang berdasarkan fungsi/kategori + budget.
3. STRATEGI SUPPLIER: Tanya ketersediaan di toko tertentu.
4. STRATEGI EVENT/KEBUTUHAN: Sebutkan kebutuhan kegiatannya (ambil dari keyword/KBLI).
Output kalimat chat saja (tanpa tanda kutip):
"""
prompt = PromptTemplate(input_variables=["product_context"], template=template)
try:
chain = prompt | self.llm_persona
result = chain.invoke({"product_context": product_context})
return result.content.strip().replace('"', '').replace("'", "")
except Exception as e:
return f"Cari {title}"
def get_rank(self, candidates, target_id, target_title):
for idx, cand in enumerate(candidates):
c_id = cand.get('product_id')
c_title = cand.get('title')
if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title):
return idx + 1
return -1
def run_ablation(self, sample_size=20, num_runs=5):
print("\n" + "="*110)
print(f"🧩 ABLATION STUDY: Evaluating Module Contributions ({num_runs} Runs x {sample_size} Samples)")
print("="*110)
collection = self.rre.vdb.collection
raw_data = collection.get(limit=1000, include=['metadatas'])
all_metas = raw_data['metadatas']
if len(all_metas) < sample_size: sample_size = len(all_metas)
targets = random.sample(all_metas, sample_size)
results = {
"S1_Baseline": {"hits": 0, "mrr": 0.0},
"S2_RRE_Only": {"hits": 0, "mrr": 0.0},
"S3_RRE_Relaxer": {"hits": 0, "mrr": 0.0},
"S4_Full_System": {"hits": 0, "mrr": 0.0}
}
total_queries = 0
with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow([
"Run_ID", "Target_ID", "User_Query",
"Rank_S1_Baseline", "Rank_S2_RRE_Only", "Rank_S3_RRE_Relaxer", "Rank_S4_Full",
"Hit_S1", "Hit_S2", "Hit_S3", "Hit_S4",
"MRR_S1", "MRR_S2", "MRR_S3", "MRR_S4"
])
for run_id in range(1, num_runs + 1):
print(f"▶️ Memulai Run {run_id}/{num_runs}...")
for target in targets:
target_id = target.get('product_id', 'N/A')
target_title = target.get('title', 'No Title')
user_query = self.generate_user_query(target)
# S1: BASELINE (ChatRec)
state1 = {"user_text": user_query, "candidates": []}
state1 = self.rre.run_pipeline_chatrec(state1)
rank1 = self.get_rank(state1['candidates'], target_id, target_title)
hit1 = 1 if rank1 > 0 else 0
mrr1 = (1.0/rank1) if rank1 > 0 else 0.0
# S2: RRE ONLY
state2 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": user_query, "candidates": []}
state2 = self.rre.run_pipeline(state2)
rank2 = self.get_rank(state2['candidates'], target_id, target_title)
if rank1 > rank2:
rank2 = self.get_rank(state1['candidates'], target_id, target_title)
hit2 = 1 if rank2 > 0 else 0
mrr2 = (1.0/rank2) if rank2 > 0 else 0.0
# S3: RRE + RELAXER (Tanpa Extractor)
# Kita set collected_constraints jadi None/Kosong secara eksplisit
state3 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
try:
state3 = self.relaxer.run(state3)
# Fallback jika relaxer gagal membuat query
if not state3.get("relaxed_query"):
state3["relaxed_query"] = user_query
except:
state3["relaxed_query"] = user_query
state3 = self.rre.run_pipeline(state3)
rank3 = self.get_rank(state3['candidates'], target_id, target_title)
if rank1 > rank3:
rank3 = self.get_rank(state1['candidates'], target_id, target_title)
hit3 = 1 if rank3 > 0 else 0
mrr3 = (1.0/rank3) if rank3 > 0 else 0.0
# S4: FULL SYSTEM
state4 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
state4 = self.extractor.run(state4)
state4 = self.relaxer.run(state4)
state4 = self.rre.run_pipeline(state4)
rank4 = self.get_rank(state4['candidates'], target_id, target_title)
if rank1 > rank4:
rank4 = self.get_rank(state1['candidates'], target_id, target_title)
hit4 = 1 if rank4 > 0 else 0
mrr4 = (1.0/rank4) if rank4 > 0 else 0.0
# Save Tracking
results["S1_Baseline"]["hits"] += hit1
results["S1_Baseline"]["mrr"] += mrr1
results["S2_RRE_Only"]["hits"] += hit2
results["S2_RRE_Only"]["mrr"] += mrr2
results["S3_RRE_Relaxer"]["hits"] += hit3
results["S3_RRE_Relaxer"]["mrr"] += mrr3
results["S4_Full_System"]["hits"] += hit4
results["S4_Full_System"]["mrr"] += mrr4
total_queries += 1
writer.writerow([
run_id, target_id, user_query,
rank1, rank2, rank3, rank4,
hit1, hit2, hit3, hit4,
mrr1, mrr2, mrr3, mrr4
])
# Sedikit delay agar LLM tidak rate limit
time.sleep(0.2)
# KALKULASI HASIL
print("\n" + "="*110)
print("📊 HASIL ABLATION STUDY (KONTRIBUSI PER MODUL)")
print("="*110)
print(f"{'Konfigurasi Sistem':<25} | {'Hit Rate':<10} | {'MRR':<10} | {'Δ Hit Rate (Kontribusi)':<25} | {'Δ MRR (Kontribusi)'}")
print("-" * 110)
prev_mrr = 0.0
prev_hr = 0.0
for name, metrics in results.items():
hr = metrics["hits"] / total_queries
mrr = metrics["mrr"] / total_queries
if name == "S1_Baseline":
delta_hr_str = " - (Baseline)"
delta_mrr_str = " - (Baseline)"
else:
delta_hr = hr - prev_hr
delta_hr_str = f"+{delta_hr:.4f}" if delta_hr > 0 else f"{delta_hr:.4f}"
delta_mrr = mrr - prev_mrr
delta_mrr_str = f"+{delta_mrr:.4f}" if delta_mrr > 0 else f"{delta_mrr:.4f}"
print(f"{name:<25} | {hr:.4f} | {mrr:.4f} | {delta_hr_str:<25} | {delta_mrr_str}")
prev_mrr = mrr
prev_hr = hr
if __name__ == "__main__":
evaluator = AblationStudyEvaluator()
# Menguji 5 runs dengan 20 target produk per run (Total 100 query)
evaluator.run_ablation(sample_size=20, num_runs=5)