import sys import os import json import random import time import csv import pandas as pd from langchain_core.prompts import PromptTemplate sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from config import Config from modules.user_modeling.extractor import EntityExtractor from modules.user_modeling.query_relaxer import QueryRelaxer from modules.reasoning_engine.engine import RREngine class AblationStudyEvaluator: def __init__(self): print("⏳ Initializing Modules for Ablation Study...") self.extractor = EntityExtractor() self.relaxer = QueryRelaxer() self.rre = RREngine() self.llm_persona = Config.get_llm(temperature=0.7) self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports") os.makedirs(self.report_dir, exist_ok=True) self.csv_file = os.path.join(self.report_dir, "ablation_study_results_qwen.csv") def generate_user_query(self, product_meta): # MENGGUNAKAN GENERATOR QUERY KOMPLEKS (FULL CONTEXT) def safe_parse_json(val): if isinstance(val, dict): return val if isinstance(val, str): try: return json.loads(val) except: return {} return {} data = product_meta cat_data = safe_parse_json(data.get('category')) sup_data = safe_parse_json(data.get('supplier')) title = data.get('title', 'Produk') price = data.get('price', 'Tidak spesifik') seller = data.get('seller', '') cat_main = cat_data.get('category', '') sub_cat = cat_data.get('sub_category', '') sub_sub_cat = cat_data.get('sub_sub_category', '') cat_full = f"{cat_main} > {sub_cat}".strip() kota = sup_data.get('kota', data.get('location', '')) prov = sup_data.get('provinsi', '') bentuk_usaha = sup_data.get('bentuk_usaha', '') loc_full = f"{kota} {prov}".strip() or "Indonesia" seller_full = f"{seller} ({bentuk_usaha})".strip() kbli = data.get('kbli', '') product_context = f""" - Nama Produk: {title} - Kategori: {cat_full} (Detail: {sub_sub_cat}) - Harga: {price} - Lokasi: {loc_full} - Penjual: {seller_full} - Keyword/Tag: {kbli} """ template = """ Berperanlah sebagai pembeli (masyarakat umum atau pejabat pengadaan) yang ingin mencari produk dengan profil berikut: {product_context} Tugas: Buat SATU kalimat chat singkat dan natural (Bahasa Indonesia) untuk mencari produk ini di sistem. PENTING: Jangan terlalu kaku. Gunakan variasi strategi pencarian (pilih satu secara acak): 1. STRATEGI SPESIFIK: Sebut nama produk lengkap + lokasi. 2. STRATEGI KATEGORI: Cari barang berdasarkan fungsi/kategori + budget. 3. STRATEGI SUPPLIER: Tanya ketersediaan di toko tertentu. 4. STRATEGI EVENT/KEBUTUHAN: Sebutkan kebutuhan kegiatannya (ambil dari keyword/KBLI). Output kalimat chat saja (tanpa tanda kutip): """ prompt = PromptTemplate(input_variables=["product_context"], template=template) try: chain = prompt | self.llm_persona result = chain.invoke({"product_context": product_context}) return result.content.strip().replace('"', '').replace("'", "") except Exception as e: return f"Cari {title}" def get_rank(self, candidates, target_id, target_title): for idx, cand in enumerate(candidates): c_id = cand.get('product_id') c_title = cand.get('title') if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title): return idx + 1 return -1 def run_ablation(self, sample_size=20, num_runs=5): print("\n" + "="*110) print(f"🧩 ABLATION STUDY: Evaluating Module Contributions ({num_runs} Runs x {sample_size} Samples)") print("="*110) collection = self.rre.vdb.collection raw_data = collection.get(limit=1000, include=['metadatas']) all_metas = raw_data['metadatas'] if len(all_metas) < sample_size: sample_size = len(all_metas) targets = random.sample(all_metas, sample_size) results = { "S1_Baseline": {"hits": 0, "mrr": 0.0}, "S2_RRE_Only": {"hits": 0, "mrr": 0.0}, "S3_RRE_Relaxer": {"hits": 0, "mrr": 0.0}, "S4_Full_System": {"hits": 0, "mrr": 0.0} } total_queries = 0 with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f: writer = csv.writer(f) writer.writerow([ "Run_ID", "Target_ID", "User_Query", "Rank_S1_Baseline", "Rank_S2_RRE_Only", "Rank_S3_RRE_Relaxer", "Rank_S4_Full", "Hit_S1", "Hit_S2", "Hit_S3", "Hit_S4", "MRR_S1", "MRR_S2", "MRR_S3", "MRR_S4" ]) for run_id in range(1, num_runs + 1): print(f"▶️ Memulai Run {run_id}/{num_runs}...") for target in targets: target_id = target.get('product_id', 'N/A') target_title = target.get('title', 'No Title') user_query = self.generate_user_query(target) # S1: BASELINE (ChatRec) state1 = {"user_text": user_query, "candidates": []} state1 = self.rre.run_pipeline_chatrec(state1) rank1 = self.get_rank(state1['candidates'], target_id, target_title) hit1 = 1 if rank1 > 0 else 0 mrr1 = (1.0/rank1) if rank1 > 0 else 0.0 # S2: RRE ONLY state2 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": user_query, "candidates": []} state2 = self.rre.run_pipeline(state2) rank2 = self.get_rank(state2['candidates'], target_id, target_title) if rank1 > rank2: rank2 = self.get_rank(state1['candidates'], target_id, target_title) hit2 = 1 if rank2 > 0 else 0 mrr2 = (1.0/rank2) if rank2 > 0 else 0.0 # S3: RRE + RELAXER (Tanpa Extractor) # Kita set collected_constraints jadi None/Kosong secara eksplisit state3 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} try: state3 = self.relaxer.run(state3) # Fallback jika relaxer gagal membuat query if not state3.get("relaxed_query"): state3["relaxed_query"] = user_query except: state3["relaxed_query"] = user_query state3 = self.rre.run_pipeline(state3) rank3 = self.get_rank(state3['candidates'], target_id, target_title) if rank1 > rank3: rank3 = self.get_rank(state1['candidates'], target_id, target_title) hit3 = 1 if rank3 > 0 else 0 mrr3 = (1.0/rank3) if rank3 > 0 else 0.0 # S4: FULL SYSTEM state4 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} state4 = self.extractor.run(state4) state4 = self.relaxer.run(state4) state4 = self.rre.run_pipeline(state4) rank4 = self.get_rank(state4['candidates'], target_id, target_title) if rank1 > rank4: rank4 = self.get_rank(state1['candidates'], target_id, target_title) hit4 = 1 if rank4 > 0 else 0 mrr4 = (1.0/rank4) if rank4 > 0 else 0.0 # Save Tracking results["S1_Baseline"]["hits"] += hit1 results["S1_Baseline"]["mrr"] += mrr1 results["S2_RRE_Only"]["hits"] += hit2 results["S2_RRE_Only"]["mrr"] += mrr2 results["S3_RRE_Relaxer"]["hits"] += hit3 results["S3_RRE_Relaxer"]["mrr"] += mrr3 results["S4_Full_System"]["hits"] += hit4 results["S4_Full_System"]["mrr"] += mrr4 total_queries += 1 writer.writerow([ run_id, target_id, user_query, rank1, rank2, rank3, rank4, hit1, hit2, hit3, hit4, mrr1, mrr2, mrr3, mrr4 ]) # Sedikit delay agar LLM tidak rate limit time.sleep(0.2) # KALKULASI HASIL print("\n" + "="*110) print("📊 HASIL ABLATION STUDY (KONTRIBUSI PER MODUL)") print("="*110) print(f"{'Konfigurasi Sistem':<25} | {'Hit Rate':<10} | {'MRR':<10} | {'Δ Hit Rate (Kontribusi)':<25} | {'Δ MRR (Kontribusi)'}") print("-" * 110) prev_mrr = 0.0 prev_hr = 0.0 for name, metrics in results.items(): hr = metrics["hits"] / total_queries mrr = metrics["mrr"] / total_queries if name == "S1_Baseline": delta_hr_str = " - (Baseline)" delta_mrr_str = " - (Baseline)" else: delta_hr = hr - prev_hr delta_hr_str = f"+{delta_hr:.4f}" if delta_hr > 0 else f"{delta_hr:.4f}" delta_mrr = mrr - prev_mrr delta_mrr_str = f"+{delta_mrr:.4f}" if delta_mrr > 0 else f"{delta_mrr:.4f}" print(f"{name:<25} | {hr:.4f} | {mrr:.4f} | {delta_hr_str:<25} | {delta_mrr_str}") prev_mrr = mrr prev_hr = hr if __name__ == "__main__": evaluator = AblationStudyEvaluator() # Menguji 5 runs dengan 20 target produk per run (Total 100 query) evaluator.run_ablation(sample_size=20, num_runs=5)