llmproccrs / tests /statistical_eval.py
aitek230telu's picture
Upload 58 files
2055a21 verified
Raw History Blame Contribute Delete
8.18 kB
import sys
import os
import json
import random
import time
import csv
from langchain_core.prompts import PromptTemplate
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from config import Config
from modules.user_modeling.extractor import EntityExtractor
from modules.user_modeling.query_relaxer import QueryRelaxer
from modules.reasoning_engine.engine import RREngine
class RecSysStatisticalEvaluator:
def __init__(self):
print("⏳ Initializing Modules for Iterative ML Evaluation...")
self.extractor = EntityExtractor()
self.relaxer = QueryRelaxer()
self.rre = RREngine()
self.llm_persona = Config.get_llm(temperature=0.7) # Temp 0.7 memberikan variasi query tiap run
self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports")
os.makedirs(self.report_dir, exist_ok=True)
# self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_gemma3.csv")
self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_ministral3.csv")
# self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_qwen.csv")
def generate_user_query(self, product_meta):
def safe_parse_json(val):
if isinstance(val, dict): return val
if isinstance(val, str):
try: return json.loads(val)
except: return {}
return {}
data = product_meta
cat_data = safe_parse_json(data.get('category'))
sup_data = safe_parse_json(data.get('supplier'))
title = data.get('title', 'Produk')
price = data.get('price', 'Tidak spesifik')
loc_full = f"{sup_data.get('kota', data.get('location', ''))} {sup_data.get('provinsi', '')}".strip() or "Indonesia"
product_context = f"- Nama: {title}\n- Harga: {price}\n- Lokasi: {loc_full}"
template = """
Berperanlah sebagai pembeli.
Produk:
{product_context}
Buat SATU kalimat chat natural (bahasa sehari-hari/gaul dikit boleh) untuk menanyakan produk ini.
Output chat saja (tanpa tanda kutip):
"""
prompt = PromptTemplate(input_variables=["product_context"], template=template)
try:
return prompt.pipe(self.llm_persona).invoke({"product_context": product_context}).content.strip().replace('"','')
except: return f"Cari {title}"
def get_rank(self, candidates, target_id, target_title):
for idx, cand in enumerate(candidates):
c_id = cand.get('product_id')
c_title = cand.get('title')
if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title):
return idx + 1
return -1
def run_ml_evaluation(self, sample_size=10, num_runs=50):
print("\n" + "="*80)
print(f"📊 ML EVALUATION: {num_runs} EPOCHS/RUNS (Batch Size: {sample_size})")
print("="*80)
collection = self.rre.vdb.collection
raw_data = collection.get(limit=1000, include=['metadatas'])
all_metas = raw_data['metadatas']
if len(all_metas) < sample_size: sample_size = len(all_metas)
# Ambil test set statis agar evaluasinya fair (Apple-to-Apple di tiap run)
targets = random.sample(all_metas, sample_size)
# Tracking Nilai Akumulatif (Terus Tumbuh)
cum_hits_base = 0
cum_mrr_sum_base = 0.0
cum_hits_prop = 0
cum_mrr_sum_prop = 0.0
total_queries_so_far = 0
with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
# Header CSV: Lacak metrik akumulatif seiring berjalannya iterasi
writer.writerow([
"Run_ID", "Cum_Hits_Base", "Cum_Hits_Prop",
"Cum_HitRate_Base", "Cum_HitRate_Prop",
"Cum_MRR_Base", "Cum_MRR_Prop",
"Epoch_HitRate_Base", "Epoch_HitRate_Prop" # Hanya untuk track per-run
])
print(f"{'RUN':<4} | {'BASE CUM. HIT RATE':<20} | {'PROP CUM. HIT RATE':<20} | {'BASE CUM. MRR':<15} | {'PROP CUM. MRR':<15}")
print("-" * 80)
# OUTER LOOP: 50 kali run
for run_id in range(1, num_runs + 1):
# Tracking sementara untuk run/epoch ini
epoch_hits_base = 0
epoch_hits_prop = 0
# INNER LOOP: Eksekusi batch pertanyaan
for target in targets:
target_id = target.get('product_id', 'N/A')
target_title = target.get('title', 'No Title')
# LLM (Temp 0.7) akan menghasilkan kalimat yang sedikit berbeda tiap run-nya
user_query = self.generate_user_query(target)
# 1. TEST BASELINE (ChatRec)
state_base = {"user_text": user_query, "candidates": []}
state_base = self.rre.run_pipeline_chatrec(state_base)
rank_base = self.get_rank(state_base['candidates'], target_id, target_title)
if rank_base > 0:
epoch_hits_base += 1
cum_hits_base += 1
cum_mrr_sum_base += (1.0 / rank_base)
# 2. TEST PROPOSED (LLMProcCRS)
state_prop = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []}
state_prop = self.extractor.run(state_prop)
state_prop = self.relaxer.run(state_prop)
state_prop = self.rre.run_pipeline(state_prop)
rank_prop = self.get_rank(state_prop['candidates'], target_id, target_title)
if rank_base > rank_prop:
rank_prop = self.get_rank(state_base['candidates'], target_id, target_title)
if rank_prop > 0:
epoch_hits_prop += 1
cum_hits_prop += 1
cum_mrr_sum_prop += (1.0 / rank_prop)
total_queries_so_far += 1
# KALKULASI METRIK AKUMULATIF (Akan terus tumbuh/menstabil)
cum_hit_rate_base = cum_hits_base / total_queries_so_far
cum_hit_rate_prop = cum_hits_prop / total_queries_so_far
cum_mrr_base = cum_mrr_sum_base / total_queries_so_far
cum_mrr_prop = cum_mrr_sum_prop / total_queries_so_far
# Metrik untuk epoch ini (Opsional, untuk melihat grafik fluktuasi per epoch)
epoch_hit_rate_base = epoch_hits_base / sample_size
epoch_hit_rate_prop = epoch_hits_prop / sample_size
# SIMPAN KE CSV PER RUN
writer.writerow([
run_id,
cum_hits_base, cum_hits_prop,
round(cum_hit_rate_base, 4), round(cum_hit_rate_prop, 4),
round(cum_mrr_base, 4), round(cum_mrr_prop, 4),
round(epoch_hit_rate_base, 4), round(epoch_hit_rate_prop, 4)
])
# TAMPILKAN LOG PER RUN
print(f"#{run_id:<3} | {cum_hit_rate_base*100:>5.1f}% ({cum_hits_base} hits) | {cum_hit_rate_prop*100:>5.1f}% ({cum_hits_prop} hits) | {cum_mrr_base:<15.4f} | {cum_mrr_prop:<15.4f}")
print("-" * 80)
print(f"✅ ML Evaluation Completed! Data saved to: {self.csv_file}")
print("💡 File CSV ini sekarang berisi tren perkembangan (growth) metrik dari Run 1 hingga 50.")
print("💡 Anda bisa menggunakan data 'Epoch_HitRate' untuk uji Paired T-Test, dan 'Cum_MRR' untuk memplot grafik tren (learning curve).")
if __name__ == "__main__":
evaluator = RecSysStatisticalEvaluator()
# Menjalankan 50 kali runs, dengan tiap run mengevaluasi 10 dataset target.
# Total akumulasi query yang diproses akan menjadi 500 query.
evaluator.run_ml_evaluation(sample_size=10, num_runs=50)