import sys import os import json import random import time import csv from langchain_core.prompts import PromptTemplate sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from config import Config from modules.user_modeling.extractor import EntityExtractor from modules.user_modeling.query_relaxer import QueryRelaxer from modules.reasoning_engine.engine import RREngine class RecSysStatisticalEvaluator: def __init__(self): print("⏳ Initializing Modules for Iterative ML Evaluation...") self.extractor = EntityExtractor() self.relaxer = QueryRelaxer() self.rre = RREngine() self.llm_persona = Config.get_llm(temperature=0.7) # Temp 0.7 memberikan variasi query tiap run self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports") os.makedirs(self.report_dir, exist_ok=True) # self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_gemma3.csv") self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_ministral3.csv") # self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_qwen.csv") def generate_user_query(self, product_meta): def safe_parse_json(val): if isinstance(val, dict): return val if isinstance(val, str): try: return json.loads(val) except: return {} return {} data = product_meta cat_data = safe_parse_json(data.get('category')) sup_data = safe_parse_json(data.get('supplier')) title = data.get('title', 'Produk') price = data.get('price', 'Tidak spesifik') loc_full = f"{sup_data.get('kota', data.get('location', ''))} {sup_data.get('provinsi', '')}".strip() or "Indonesia" product_context = f"- Nama: {title}\n- Harga: {price}\n- Lokasi: {loc_full}" template = """ Berperanlah sebagai pembeli. Produk: {product_context} Buat SATU kalimat chat natural (bahasa sehari-hari/gaul dikit boleh) untuk menanyakan produk ini. Output chat saja (tanpa tanda kutip): """ prompt = PromptTemplate(input_variables=["product_context"], template=template) try: return prompt.pipe(self.llm_persona).invoke({"product_context": product_context}).content.strip().replace('"','') except: return f"Cari {title}" def get_rank(self, candidates, target_id, target_title): for idx, cand in enumerate(candidates): c_id = cand.get('product_id') c_title = cand.get('title') if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title): return idx + 1 return -1 def run_ml_evaluation(self, sample_size=10, num_runs=50): print("\n" + "="*80) print(f"📊 ML EVALUATION: {num_runs} EPOCHS/RUNS (Batch Size: {sample_size})") print("="*80) collection = self.rre.vdb.collection raw_data = collection.get(limit=1000, include=['metadatas']) all_metas = raw_data['metadatas'] if len(all_metas) < sample_size: sample_size = len(all_metas) # Ambil test set statis agar evaluasinya fair (Apple-to-Apple di tiap run) targets = random.sample(all_metas, sample_size) # Tracking Nilai Akumulatif (Terus Tumbuh) cum_hits_base = 0 cum_mrr_sum_base = 0.0 cum_hits_prop = 0 cum_mrr_sum_prop = 0.0 total_queries_so_far = 0 with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f: writer = csv.writer(f) # Header CSV: Lacak metrik akumulatif seiring berjalannya iterasi writer.writerow([ "Run_ID", "Cum_Hits_Base", "Cum_Hits_Prop", "Cum_HitRate_Base", "Cum_HitRate_Prop", "Cum_MRR_Base", "Cum_MRR_Prop", "Epoch_HitRate_Base", "Epoch_HitRate_Prop" # Hanya untuk track per-run ]) print(f"{'RUN':<4} | {'BASE CUM. HIT RATE':<20} | {'PROP CUM. HIT RATE':<20} | {'BASE CUM. MRR':<15} | {'PROP CUM. MRR':<15}") print("-" * 80) # OUTER LOOP: 50 kali run for run_id in range(1, num_runs + 1): # Tracking sementara untuk run/epoch ini epoch_hits_base = 0 epoch_hits_prop = 0 # INNER LOOP: Eksekusi batch pertanyaan for target in targets: target_id = target.get('product_id', 'N/A') target_title = target.get('title', 'No Title') # LLM (Temp 0.7) akan menghasilkan kalimat yang sedikit berbeda tiap run-nya user_query = self.generate_user_query(target) # 1. TEST BASELINE (ChatRec) state_base = {"user_text": user_query, "candidates": []} state_base = self.rre.run_pipeline_chatrec(state_base) rank_base = self.get_rank(state_base['candidates'], target_id, target_title) if rank_base > 0: epoch_hits_base += 1 cum_hits_base += 1 cum_mrr_sum_base += (1.0 / rank_base) # 2. TEST PROPOSED (LLMProcCRS) state_prop = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} state_prop = self.extractor.run(state_prop) state_prop = self.relaxer.run(state_prop) state_prop = self.rre.run_pipeline(state_prop) rank_prop = self.get_rank(state_prop['candidates'], target_id, target_title) if rank_base > rank_prop: rank_prop = self.get_rank(state_base['candidates'], target_id, target_title) if rank_prop > 0: epoch_hits_prop += 1 cum_hits_prop += 1 cum_mrr_sum_prop += (1.0 / rank_prop) total_queries_so_far += 1 # KALKULASI METRIK AKUMULATIF (Akan terus tumbuh/menstabil) cum_hit_rate_base = cum_hits_base / total_queries_so_far cum_hit_rate_prop = cum_hits_prop / total_queries_so_far cum_mrr_base = cum_mrr_sum_base / total_queries_so_far cum_mrr_prop = cum_mrr_sum_prop / total_queries_so_far # Metrik untuk epoch ini (Opsional, untuk melihat grafik fluktuasi per epoch) epoch_hit_rate_base = epoch_hits_base / sample_size epoch_hit_rate_prop = epoch_hits_prop / sample_size # SIMPAN KE CSV PER RUN writer.writerow([ run_id, cum_hits_base, cum_hits_prop, round(cum_hit_rate_base, 4), round(cum_hit_rate_prop, 4), round(cum_mrr_base, 4), round(cum_mrr_prop, 4), round(epoch_hit_rate_base, 4), round(epoch_hit_rate_prop, 4) ]) # TAMPILKAN LOG PER RUN print(f"#{run_id:<3} | {cum_hit_rate_base*100:>5.1f}% ({cum_hits_base} hits) | {cum_hit_rate_prop*100:>5.1f}% ({cum_hits_prop} hits) | {cum_mrr_base:<15.4f} | {cum_mrr_prop:<15.4f}") print("-" * 80) print(f"✅ ML Evaluation Completed! Data saved to: {self.csv_file}") print("💡 File CSV ini sekarang berisi tren perkembangan (growth) metrik dari Run 1 hingga 50.") print("💡 Anda bisa menggunakan data 'Epoch_HitRate' untuk uji Paired T-Test, dan 'Cum_MRR' untuk memplot grafik tren (learning curve).") if __name__ == "__main__": evaluator = RecSysStatisticalEvaluator() # Menjalankan 50 kali runs, dengan tiap run mengevaluasi 10 dataset target. # Total akumulasi query yang diproses akan menjadi 500 query. evaluator.run_ml_evaluation(sample_size=10, num_runs=50)