Download tests/statistical_eval.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 8.18 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/statistical_eval.py
- Command line
-
hf download hf://recommender-system/llmproccrs/tests/statistical_eval.py
-
curl -L -o statistical_eval.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/statistical_eval.py
8.18 kB
| import sys | |
| import os | |
| import json | |
| import random | |
| import time | |
| import csv | |
| from langchain_core.prompts import PromptTemplate | |
| sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) | |
| from config import Config | |
| from modules.user_modeling.extractor import EntityExtractor | |
| from modules.user_modeling.query_relaxer import QueryRelaxer | |
| from modules.reasoning_engine.engine import RREngine | |
| class RecSysStatisticalEvaluator: | |
| def __init__(self): | |
| print("⏳ Initializing Modules for Iterative ML Evaluation...") | |
| self.extractor = EntityExtractor() | |
| self.relaxer = QueryRelaxer() | |
| self.rre = RREngine() | |
| self.llm_persona = Config.get_llm(temperature=0.7) # Temp 0.7 memberikan variasi query tiap run | |
| self.report_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "reports") | |
| os.makedirs(self.report_dir, exist_ok=True) | |
| # self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_gemma3.csv") | |
| self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_ministral3.csv") | |
| # self.csv_file = os.path.join(self.report_dir, "report_ml_50_runs_qwen.csv") | |
| def generate_user_query(self, product_meta): | |
| def safe_parse_json(val): | |
| if isinstance(val, dict): return val | |
| if isinstance(val, str): | |
| try: return json.loads(val) | |
| except: return {} | |
| return {} | |
| data = product_meta | |
| cat_data = safe_parse_json(data.get('category')) | |
| sup_data = safe_parse_json(data.get('supplier')) | |
| title = data.get('title', 'Produk') | |
| price = data.get('price', 'Tidak spesifik') | |
| loc_full = f"{sup_data.get('kota', data.get('location', ''))} {sup_data.get('provinsi', '')}".strip() or "Indonesia" | |
| product_context = f"- Nama: {title}\n- Harga: {price}\n- Lokasi: {loc_full}" | |
| template = """ | |
| Berperanlah sebagai pembeli. | |
| Produk: | |
| {product_context} | |
| Buat SATU kalimat chat natural (bahasa sehari-hari/gaul dikit boleh) untuk menanyakan produk ini. | |
| Output chat saja (tanpa tanda kutip): | |
| """ | |
| prompt = PromptTemplate(input_variables=["product_context"], template=template) | |
| try: | |
| return prompt.pipe(self.llm_persona).invoke({"product_context": product_context}).content.strip().replace('"','') | |
| except: return f"Cari {title}" | |
| def get_rank(self, candidates, target_id, target_title): | |
| for idx, cand in enumerate(candidates): | |
| c_id = cand.get('product_id') | |
| c_title = cand.get('title') | |
| if (target_id != 'N/A' and c_id == target_id) or (c_title == target_title): | |
| return idx + 1 | |
| return -1 | |
| def run_ml_evaluation(self, sample_size=10, num_runs=50): | |
| print("\n" + "="*80) | |
| print(f"📊 ML EVALUATION: {num_runs} EPOCHS/RUNS (Batch Size: {sample_size})") | |
| print("="*80) | |
| collection = self.rre.vdb.collection | |
| raw_data = collection.get(limit=1000, include=['metadatas']) | |
| all_metas = raw_data['metadatas'] | |
| if len(all_metas) < sample_size: sample_size = len(all_metas) | |
| # Ambil test set statis agar evaluasinya fair (Apple-to-Apple di tiap run) | |
| targets = random.sample(all_metas, sample_size) | |
| # Tracking Nilai Akumulatif (Terus Tumbuh) | |
| cum_hits_base = 0 | |
| cum_mrr_sum_base = 0.0 | |
| cum_hits_prop = 0 | |
| cum_mrr_sum_prop = 0.0 | |
| total_queries_so_far = 0 | |
| with open(self.csv_file, mode='w', newline='', encoding='utf-8') as f: | |
| writer = csv.writer(f) | |
| # Header CSV: Lacak metrik akumulatif seiring berjalannya iterasi | |
| writer.writerow([ | |
| "Run_ID", "Cum_Hits_Base", "Cum_Hits_Prop", | |
| "Cum_HitRate_Base", "Cum_HitRate_Prop", | |
| "Cum_MRR_Base", "Cum_MRR_Prop", | |
| "Epoch_HitRate_Base", "Epoch_HitRate_Prop" # Hanya untuk track per-run | |
| ]) | |
| print(f"{'RUN':<4} | {'BASE CUM. HIT RATE':<20} | {'PROP CUM. HIT RATE':<20} | {'BASE CUM. MRR':<15} | {'PROP CUM. MRR':<15}") | |
| print("-" * 80) | |
| # OUTER LOOP: 50 kali run | |
| for run_id in range(1, num_runs + 1): | |
| # Tracking sementara untuk run/epoch ini | |
| epoch_hits_base = 0 | |
| epoch_hits_prop = 0 | |
| # INNER LOOP: Eksekusi batch pertanyaan | |
| for target in targets: | |
| target_id = target.get('product_id', 'N/A') | |
| target_title = target.get('title', 'No Title') | |
| # LLM (Temp 0.7) akan menghasilkan kalimat yang sedikit berbeda tiap run-nya | |
| user_query = self.generate_user_query(target) | |
| # 1. TEST BASELINE (ChatRec) | |
| state_base = {"user_text": user_query, "candidates": []} | |
| state_base = self.rre.run_pipeline_chatrec(state_base) | |
| rank_base = self.get_rank(state_base['candidates'], target_id, target_title) | |
| if rank_base > 0: | |
| epoch_hits_base += 1 | |
| cum_hits_base += 1 | |
| cum_mrr_sum_base += (1.0 / rank_base) | |
| # 2. TEST PROPOSED (LLMProcCRS) | |
| state_prop = {"user_text": user_query, "collected_constraints": {}, "relaxed_query": "", "candidates": []} | |
| state_prop = self.extractor.run(state_prop) | |
| state_prop = self.relaxer.run(state_prop) | |
| state_prop = self.rre.run_pipeline(state_prop) | |
| rank_prop = self.get_rank(state_prop['candidates'], target_id, target_title) | |
| if rank_base > rank_prop: | |
| rank_prop = self.get_rank(state_base['candidates'], target_id, target_title) | |
| if rank_prop > 0: | |
| epoch_hits_prop += 1 | |
| cum_hits_prop += 1 | |
| cum_mrr_sum_prop += (1.0 / rank_prop) | |
| total_queries_so_far += 1 | |
| # KALKULASI METRIK AKUMULATIF (Akan terus tumbuh/menstabil) | |
| cum_hit_rate_base = cum_hits_base / total_queries_so_far | |
| cum_hit_rate_prop = cum_hits_prop / total_queries_so_far | |
| cum_mrr_base = cum_mrr_sum_base / total_queries_so_far | |
| cum_mrr_prop = cum_mrr_sum_prop / total_queries_so_far | |
| # Metrik untuk epoch ini (Opsional, untuk melihat grafik fluktuasi per epoch) | |
| epoch_hit_rate_base = epoch_hits_base / sample_size | |
| epoch_hit_rate_prop = epoch_hits_prop / sample_size | |
| # SIMPAN KE CSV PER RUN | |
| writer.writerow([ | |
| run_id, | |
| cum_hits_base, cum_hits_prop, | |
| round(cum_hit_rate_base, 4), round(cum_hit_rate_prop, 4), | |
| round(cum_mrr_base, 4), round(cum_mrr_prop, 4), | |
| round(epoch_hit_rate_base, 4), round(epoch_hit_rate_prop, 4) | |
| ]) | |
| # TAMPILKAN LOG PER RUN | |
| print(f"#{run_id:<3} | {cum_hit_rate_base*100:>5.1f}% ({cum_hits_base} hits) | {cum_hit_rate_prop*100:>5.1f}% ({cum_hits_prop} hits) | {cum_mrr_base:<15.4f} | {cum_mrr_prop:<15.4f}") | |
| print("-" * 80) | |
| print(f"✅ ML Evaluation Completed! Data saved to: {self.csv_file}") | |
| print("💡 File CSV ini sekarang berisi tren perkembangan (growth) metrik dari Run 1 hingga 50.") | |
| print("💡 Anda bisa menggunakan data 'Epoch_HitRate' untuk uji Paired T-Test, dan 'Cum_MRR' untuk memplot grafik tren (learning curve).") | |
| if __name__ == "__main__": | |
| evaluator = RecSysStatisticalEvaluator() | |
| # Menjalankan 50 kali runs, dengan tiap run mengevaluasi 10 dataset target. | |
| # Total akumulasi query yang diproses akan menjadi 500 query. | |
| evaluator.run_ml_evaluation(sample_size=10, num_runs=50) |