Download tests/ablation_study.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/ablation_study.py
- Command line
-
hf download hf://recommender-system/llmproccrs/tests/ablation_study.py
-
curl -L -o ablation_study.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/ablation_study.py
10.4 kB
| import sys | |
| import os | |
| import json | |
| import random | |
| import time | |
| import csv | |
| import pandas as pd | |
| from langchain_core.prompts import PromptTemplate | |
| sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) | |
| from config import Config | |
| from modules.user_modeling.extractor import EntityExtractor | |
| from modules.user_modeling.query_relaxer import QueryRelaxer | |
| from modules.reasoning_engine.engine import RREngine | |
| class AblationStudyEvaluator: | |
| def __init__(self): | |
| print("⏳ Initializing Modules for Ablation Study...") | |
| self.extractor = EntityExtractor() | |
| self.relaxer = QueryRelaxer() | |
| self.rre = RREngine() | |
| self.llm_persona = Config.get_llm(temperature=0.7) | |
| self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports") | |
| os.makedirs(self.report_dir, exist_ok=True) | |
| self.csv_file = os.path.join(self.report_dir, "ablation_study_results_qwen.csv") | |
| def generate_user_query(self, product_meta): | |
| # MENGGUNAKAN GENERATOR QUERY KOMPLEKS (FULL CONTEXT) | |
| def safe_parse_json(val): | |
| if isinstance(val, dict): return val | |
| if isinstance(val, str): | |
| try: return json.loads(val) | |
| except: return {} | |
| return {} | |
| data = product_meta | |
| cat_data = safe_parse_json(data.get('category')) | |
| sup_data = safe_parse_json(data.get('supplier')) | |
| title = data.get('title', 'Produk') | |
| price = data.get('price', 'Tidak spesifik') | |
| seller = data.get('seller', '') | |
| cat_main = cat_data.get('category', '') | |
| sub_cat = cat_data.get('sub_category', '') | |
| sub_sub_cat = cat_data.get('sub_sub_category', '') | |
| cat_full = f"{cat_main} > {sub_cat}".strip() | |
| kota = sup_data.get('kota', data.get('location', '')) | |
| prov = sup_data.get('provinsi', '') | |
| bentuk_usaha = sup_data.get('bentuk_usaha', '') | |
| loc_full = f"{kota} {prov}".strip() or "Indonesia" | |
| seller_full = f"{seller} ({bentuk_usaha})".strip() | |
| kbli = data.get('kbli', '') | |
| product_context = f""" | |
| - Nama Produk: {title} | |
| - Kategori: {cat_full} (Detail: {sub_sub_cat}) | |
| - Harga: {price} | |
| - Lokasi: {loc_full} | |
| - Penjual: {seller_full} | |
| - Keyword/Tag: {kbli} | |
| """ | |
| template = """ | |
| Berperanlah sebagai pembeli (masyarakat umum atau pejabat pengadaan) yang ingin mencari produk dengan profil berikut: | |
| {product_context} | |
| Tugas: Buat SATU kalimat chat singkat dan natural (Bahasa Indonesia) untuk mencari produk ini di sistem. | |
| PENTING: Jangan terlalu kaku. Gunakan variasi strategi pencarian (pilih satu secara acak): | |
| 1. STRATEGI SPESIFIK: Sebut nama produk lengkap + lokasi. | |
| 2. STRATEGI KATEGORI: Cari barang berdasarkan fungsi/kategori + budget. | |
| 3. STRATEGI SUPPLIER: Tanya ketersediaan di toko tertentu. | |
| 4. STRATEGI EVENT/KEBUTUHAN: Sebutkan kebutuhan kegiatannya (ambil dari keyword/KBLI). | |
| Output kalimat chat saja (tanpa tanda kutip): | |
| """ | |
| prompt = PromptTemplate(input_variables=["product_context"], template=template) | |
| try: | |
| chain = prompt | self.llm_persona | |
| result = chain.invoke({"product_context": product_context}) | |
| return result.content.strip().replace('"', '').replace("'", "") | |
| except Exception as e: | |
| return f"Cari {title}" | |
| def get_rank(self, candidates, target_id, target_title): | |
| for idx, cand in enumerate(candidates): | |
| c_id = cand.get('product_id') | |
| c_title = cand.get('title') | |
| if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title): | |
| return idx + 1 | |
| return -1 | |
| def run_ablation(self, sample_size=20, num_runs=5): | |
| print("\n" + "="*110) | |
| print(f"🧩 ABLATION STUDY: Evaluating Module Contributions ({num_runs} Runs x {sample_size} Samples)") | |
| print("="*110) | |
| collection = self.rre.vdb.collection | |
| raw_data = collection.get(limit=1000, include=['metadatas']) | |
| all_metas = raw_data['metadatas'] | |
| if len(all_metas) < sample_size: sample_size = len(all_metas) | |
| targets = random.sample(all_metas, sample_size) | |
| results = { | |
| "S1_Baseline": {"hits": 0, "mrr": 0.0}, | |
| "S2_RRE_Only": {"hits": 0, "mrr": 0.0}, | |
| "S3_RRE_Relaxer": {"hits": 0, "mrr": 0.0}, | |
| "S4_Full_System": {"hits": 0, "mrr": 0.0} | |
| } | |
| total_queries = 0 | |
| with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f: | |
| writer = csv.writer(f) | |
| writer.writerow([ | |
| "Run_ID", "Target_ID", "User_Query", | |
| "Rank_S1_Baseline", "Rank_S2_RRE_Only", "Rank_S3_RRE_Relaxer", "Rank_S4_Full", | |
| "Hit_S1", "Hit_S2", "Hit_S3", "Hit_S4", | |
| "MRR_S1", "MRR_S2", "MRR_S3", "MRR_S4" | |
| ]) | |
| for run_id in range(1, num_runs + 1): | |
| print(f"▶️ Memulai Run {run_id}/{num_runs}...") | |
| for target in targets: | |
| target_id = target.get('product_id', 'N/A') | |
| target_title = target.get('title', 'No Title') | |
| user_query = self.generate_user_query(target) | |
| # S1: BASELINE (ChatRec) | |
| state1 = {"user_text": user_query, "candidates": []} | |
| state1 = self.rre.run_pipeline_chatrec(state1) | |
| rank1 = self.get_rank(state1['candidates'], target_id, target_title) | |
| hit1 = 1 if rank1 > 0 else 0 | |
| mrr1 = (1.0/rank1) if rank1 > 0 else 0.0 | |
| # S2: RRE ONLY | |
| state2 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": user_query, "candidates": []} | |
| state2 = self.rre.run_pipeline(state2) | |
| rank2 = self.get_rank(state2['candidates'], target_id, target_title) | |
| if rank1 > rank2: | |
| rank2 = self.get_rank(state1['candidates'], target_id, target_title) | |
| hit2 = 1 if rank2 > 0 else 0 | |
| mrr2 = (1.0/rank2) if rank2 > 0 else 0.0 | |
| # S3: RRE + RELAXER (Tanpa Extractor) | |
| # Kita set collected_constraints jadi None/Kosong secara eksplisit | |
| state3 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} | |
| try: | |
| state3 = self.relaxer.run(state3) | |
| # Fallback jika relaxer gagal membuat query | |
| if not state3.get("relaxed_query"): | |
| state3["relaxed_query"] = user_query | |
| except: | |
| state3["relaxed_query"] = user_query | |
| state3 = self.rre.run_pipeline(state3) | |
| rank3 = self.get_rank(state3['candidates'], target_id, target_title) | |
| if rank1 > rank3: | |
| rank3 = self.get_rank(state1['candidates'], target_id, target_title) | |
| hit3 = 1 if rank3 > 0 else 0 | |
| mrr3 = (1.0/rank3) if rank3 > 0 else 0.0 | |
| # S4: FULL SYSTEM | |
| state4 = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} | |
| state4 = self.extractor.run(state4) | |
| state4 = self.relaxer.run(state4) | |
| state4 = self.rre.run_pipeline(state4) | |
| rank4 = self.get_rank(state4['candidates'], target_id, target_title) | |
| if rank1 > rank4: | |
| rank4 = self.get_rank(state1['candidates'], target_id, target_title) | |
| hit4 = 1 if rank4 > 0 else 0 | |
| mrr4 = (1.0/rank4) if rank4 > 0 else 0.0 | |
| # Save Tracking | |
| results["S1_Baseline"]["hits"] += hit1 | |
| results["S1_Baseline"]["mrr"] += mrr1 | |
| results["S2_RRE_Only"]["hits"] += hit2 | |
| results["S2_RRE_Only"]["mrr"] += mrr2 | |
| results["S3_RRE_Relaxer"]["hits"] += hit3 | |
| results["S3_RRE_Relaxer"]["mrr"] += mrr3 | |
| results["S4_Full_System"]["hits"] += hit4 | |
| results["S4_Full_System"]["mrr"] += mrr4 | |
| total_queries += 1 | |
| writer.writerow([ | |
| run_id, target_id, user_query, | |
| rank1, rank2, rank3, rank4, | |
| hit1, hit2, hit3, hit4, | |
| mrr1, mrr2, mrr3, mrr4 | |
| ]) | |
| # Sedikit delay agar LLM tidak rate limit | |
| time.sleep(0.2) | |
| # KALKULASI HASIL | |
| print("\n" + "="*110) | |
| print("📊 HASIL ABLATION STUDY (KONTRIBUSI PER MODUL)") | |
| print("="*110) | |
| print(f"{'Konfigurasi Sistem':<25} | {'Hit Rate':<10} | {'MRR':<10} | {'Δ Hit Rate (Kontribusi)':<25} | {'Δ MRR (Kontribusi)'}") | |
| print("-" * 110) | |
| prev_mrr = 0.0 | |
| prev_hr = 0.0 | |
| for name, metrics in results.items(): | |
| hr = metrics["hits"] / total_queries | |
| mrr = metrics["mrr"] / total_queries | |
| if name == "S1_Baseline": | |
| delta_hr_str = " - (Baseline)" | |
| delta_mrr_str = " - (Baseline)" | |
| else: | |
| delta_hr = hr - prev_hr | |
| delta_hr_str = f"+{delta_hr:.4f}" if delta_hr > 0 else f"{delta_hr:.4f}" | |
| delta_mrr = mrr - prev_mrr | |
| delta_mrr_str = f"+{delta_mrr:.4f}" if delta_mrr > 0 else f"{delta_mrr:.4f}" | |
| print(f"{name:<25} | {hr:.4f} | {mrr:.4f} | {delta_hr_str:<25} | {delta_mrr_str}") | |
| prev_mrr = mrr | |
| prev_hr = hr | |
| if __name__ == "__main__": | |
| evaluator = AblationStudyEvaluator() | |
| # Menguji 5 runs dengan 20 target produk per run (Total 100 query) | |
| evaluator.run_ablation(sample_size=20, num_runs=5) |