import torch from transformers import AutoModelForCausalLM, AutoTokenizer from peft import PeftModel import json from typing import Dict, List, Tuple import numpy as np from tqdm import tqdm from sklearn.metrics import accuracy_score, f1_score import evaluate from datasets import load_dataset import pandas as pd import matplotlib.pyplot as plt import seaborn as sns class CounselorBenchmark: def __init__(self, base_model_path: str, finetuned_model_path: str): """ Initialize benchmark suite for counselor models """ self.base_model_path = base_model_path self.finetuned_model_path = finetuned_model_path self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Load evaluation metrics self.bleu = evaluate.load("sacrebleu") self.rouge = evaluate.load("rouge") self.bertscore = evaluate.load("bertscore") def load_models(self): """Load both base and fine-tuned models for comparison""" # Load base model print("Loading base model...") self.base_tokenizer = AutoTokenizer.from_pretrained(self.base_model_path) self.base_model = AutoModelForCausalLM.from_pretrained( self.base_model_path, torch_dtype=torch.bfloat16, device_map="auto" ) # Load fine-tuned model print("Loading fine-tuned model...") self.ft_tokenizer = AutoTokenizer.from_pretrained(self.finetuned_model_path) self.ft_model = AutoModelForCausalLM.from_pretrained( self.finetuned_model_path, torch_dtype=torch.bfloat16, device_map="auto" ) def generate_response(self, model, tokenizer, prompt: str, max_length: int = 256): """Generate response from model""" inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512) inputs = {k: v.to(self.device) for k, v in inputs.items()} with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=max_length, temperature=0.7, do_sample=True, top_p=0.9, repetition_penalty=1.1 ) response = tokenizer.decode(outputs[0], skip_special_tokens=True) # Extract only the generated part response = response[len(prompt):].strip() return response def evaluate_empathy_score(self, response: str) -> float: """ Evaluate empathy in counselor response Custom metric based on Japanese counseling keywords """ empathy_keywords = [ 'わかります', '理解', '共感', '気持ち', '感じ', 'つらい', '大変', 'お察し', '心配', '支援' ] score = sum(1 for keyword in empathy_keywords if keyword in response) return min(score / len(empathy_keywords), 1.0) def evaluate_response_quality(self, response: str) -> Dict[str, float]: """ Comprehensive response quality evaluation """ metrics = {} # Length appropriateness (not too short, not too long) response_length = len(response) if 50 <= response_length <= 300: metrics['length_score'] = 1.0 elif response_length < 50: metrics['length_score'] = response_length / 50 else: metrics['length_score'] = max(0, 1 - (response_length - 300) / 500) # Question engagement (does counselor ask clarifying questions?) metrics['question_score'] = 1.0 if '？' in response or 'か？' in response else 0.0 # Supportive language support_phrases = ['大丈夫', '一緒に', '支援', 'サポート', '助け'] metrics['support_score'] = sum(1 for phrase in support_phrases if phrase in response) / len(support_phrases) # Empathy score metrics['empathy_score'] = self.evaluate_empathy_score(response) return metrics def benchmark_on_test_set(self, test_data_path: str, num_samples: int = 100): """ Run comprehensive benchmark on test set """ # Load test data test_dataset = load_dataset('json', data_files=test_data_path, split='train') test_samples = test_dataset.select(range(min(num_samples, len(test_dataset)))) results = { 'base_model': {'responses': [], 'metrics': []}, 'finetuned_model': {'responses': [], 'metrics': []} } print(f"Evaluating on {len(test_samples)} test samples...") for sample in tqdm(test_samples): prompt = sample['text'].split('### Response:')[0] + '### Response:' reference = sample['text'].split('### Response:')[1].strip() if '### Response:' in sample['text'] else "" # Generate responses base_response = self.generate_response(self.base_model, self.base_tokenizer, prompt) ft_response = self.generate_response(self.ft_model, self.ft_tokenizer, prompt) # Store responses results['base_model']['responses'].append(base_response) results['finetuned_model']['responses'].append(ft_response) # Evaluate quality base_metrics = self.evaluate_response_quality(base_response) ft_metrics = self.evaluate_response_quality(ft_response) results['base_model']['metrics'].append(base_metrics) results['finetuned_model']['metrics'].append(ft_metrics) return results def calculate_aggregate_metrics(self, results: Dict) -> Dict: """Calculate aggregate metrics for comparison""" aggregate = {} for model_name in ['base_model', 'finetuned_model']: model_metrics = results[model_name]['metrics'] aggregate[model_name] = {} # Calculate average for each metric metric_names = model_metrics[0].keys() if model_metrics else [] for metric in metric_names: values = [m[metric] for m in model_metrics] aggregate[model_name][metric] = { 'mean': np.mean(values), 'std': np.std(values), 'min': np.min(values), 'max': np.max(values) } return aggregate def generate_comparison_report(self, results: Dict, aggregate: Dict): """Generate detailed comparison report""" report = [] report.append("=" * 80) report.append("COUNSELOR MODEL BENCHMARK REPORT") report.append("=" * 80) report.append("") # Overall performance comparison report.append("PERFORMANCE COMPARISON:") report.append("-" * 40) for metric in aggregate['base_model'].keys(): base_score = aggregate['base_model'][metric]['mean'] ft_score = aggregate['finetuned_model'][metric]['mean'] improvement = ((ft_score - base_score) / base_score * 100) if base_score > 0 else 0 report.append(f"\n{metric.upper()}:") report.append(f" Base Model: {base_score:.3f} (±{aggregate['base_model'][metric]['std']:.3f})") report.append(f" Fine-tuned Model: {ft_score:.3f} (±{aggregate['finetuned_model'][metric]['std']:.3f})") report.append(f" Improvement: {improvement:+.1f}%") # Calculate overall score base_overall = np.mean([aggregate['base_model'][m]['mean'] for m in aggregate['base_model']]) ft_overall = np.mean([aggregate['finetuned_model'][m]['mean'] for m in aggregate['finetuned_model']]) overall_improvement = ((ft_overall - base_overall) / base_overall * 100) if base_overall > 0 else 0 report.append("\n" + "=" * 40) report.append("OVERALL PERFORMANCE:") report.append(f" Base Model: {base_overall:.3f}") report.append(f" Fine-tuned Model: {ft_overall:.3f}") report.append(f" Overall Improvement: {overall_improvement:+.1f}%") report.append("=" * 40) return "\n".join(report) def visualize_results(self, aggregate: Dict): """Create visualization of benchmark results""" # Prepare data for plotting metrics = list(aggregate['base_model'].keys()) base_scores = [aggregate['base_model'][m]['mean'] for m in metrics] ft_scores = [aggregate['finetuned_model'][m]['mean'] for m in metrics] # Create comparison plot fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6)) # Bar plot comparison x = np.arange(len(metrics)) width = 0.35 ax1.bar(x - width/2, base_scores, width, label='Base Model', color='lightblue') ax1.bar(x + width/2, ft_scores, width, label='Fine-tuned Model', color='darkblue') ax1.set_xlabel('Metrics') ax1.set_ylabel('Score') ax1.set_title('Model Performance Comparison') ax1.set_xticks(x) ax1.set_xticklabels(metrics, rotation=45, ha='right') ax1.legend() ax1.grid(True, alpha=0.3) # Improvement percentage plot improvements = [((ft - base) / base * 100) if base > 0 else 0 for base, ft in zip(base_scores, ft_scores)] colors = ['green' if imp > 0 else 'red' for imp in improvements] ax2.bar(metrics, improvements, color=colors, alpha=0.7) ax2.set_xlabel('Metrics') ax2.set_ylabel('Improvement (%)') ax2.set_title('Fine-tuning Improvement over Base Model') ax2.axhline(y=0, color='black', linestyle='-', linewidth=0.5) ax2.set_xticklabels(metrics, rotation=45, ha='right') ax2.grid(True, alpha=0.3) plt.tight_layout() plt.savefig('benchmark_results.png', dpi=300, bbox_inches='tight') plt.show() print("Visualization saved as 'benchmark_results.png'") # Run benchmarking if __name__ == "__main__": # Initialize benchmark benchmark = CounselorBenchmark( base_model_path="./models/LFM2-2.6B", finetuned_model_path="./merged_counselor_mode_2b" ) # Load models benchmark.load_models() # Run benchmark print("Running benchmark evaluation...") results = benchmark.benchmark_on_test_set("./processed_data_score80/test.jsonl", num_samples=100) # Calculate aggregate metrics aggregate = benchmark.calculate_aggregate_metrics(results) # Generate report report = benchmark.generate_comparison_report(results, aggregate) print(report) # Save report with open("benchmark_report_2b.txt", "w") as f: f.write(report) # Visualize results benchmark.visualize_results(aggregate) print("\nBenchmarking completed! Check 'benchmark_report.txt' for detailed results.") #################### # import torch # from transformers import AutoModelForCausalLM, AutoTokenizer # from peft import PeftModel, PeftConfig # import numpy as np # from typing import List, Dict, Tuple, Optional # import json # from tqdm import tqdm # import os # import gc # import warnings # from datetime import datetime # import pandas as pd # import matplotlib.pyplot as plt # import seaborn as sns # from nltk.translate.bleu_score import sentence_bleu, corpus_bleu, SmoothingFunction # from rouge_score import rouge_scorer # import nltk # from collections import defaultdict # # Download required NLTK data # try: # nltk.download('punkt', quiet=True) # except: # pass # warnings.filterwarnings('ignore') # class AdvancedCounselorBenchmark: # def __init__(self, # base_model_name: str = "LiquidAI/LFM2-1.2B", # finetuned_model_path: str = "./counselor_model/best_model", # merged_model_path: str = "./merged_counselor_model", # test_data_path: str = "./processed_data_score70/test.jsonl", # device: str = None): # """ # Initialize advanced benchmark suite with BLEU and ROUGE metrics # Args: # base_model_name: Name/path of base model # finetuned_model_path: Path to fine-tuned LoRA adapter # merged_model_path: Path to save/load merged model # test_data_path: Path to test dataset with reference responses # device: Device to run on (cuda/cpu) # """ # self.base_model_name = base_model_name # self.finetuned_model_path = finetuned_model_path # self.merged_model_path = merged_model_path # self.test_data_path = test_data_path # self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") # print(f"🔧 Initializing Advanced Benchmark Suite") # print(f" Device: {self.device}") # if self.device == "cuda": # print(f" GPU: {torch.cuda.get_device_name(0)}") # print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB") # # Initialize ROUGE scorer # self.rouge_scorer = rouge_scorer.RougeScorer( # ['rouge1', 'rouge2', 'rougeL'], # use_stemmer=False, # Set to False for Japanese # lang='japanese' # ) # # Smoothing function for BLEU scores # self.smoothing = SmoothingFunction().method1 # self.results = {} # def load_test_data(self) -> List[Dict]: # """Load test dataset with reference responses""" # print(f"\n📚 Loading test data from {self.test_data_path}") # test_data = [] # if os.path.exists(self.test_data_path): # with open(self.test_data_path, 'r', encoding='utf-8') as f: # for line in f: # data = json.loads(line) # test_data.append(data) # print(f" Loaded {len(test_data)} test examples") # else: # print(f"⚠️ Test data not found. Creating synthetic test data...") # test_data = self.create_synthetic_test_data() # return test_data # def create_synthetic_test_data(self) -> List[Dict]: # """Create synthetic test data if real data is not available""" # synthetic_data = [ # { # "text": "### Input:\n最近ストレスを感じています。\n\n### Response:\nストレスを感じているのですね。それは大変つらいことだと思います。どのような状況でストレスを感じることが多いですか？お話を聞かせていただければ、一緒に対処法を考えることができます。", # "input": "最近ストレスを感じています。", # "reference": "ストレスを感じているのですね。それは大変つらいことだと思います。どのような状況でストレスを感じることが多いですか？お話を聞かせていただければ、一緒に対処法を考えることができます。" # }, # { # "text": "### Input:\n仕事がうまくいかなくて悩んでいます。\n\n### Response:\n仕事でお悩みなのですね。うまくいかないと感じると、本当に辛いですよね。具体的にどのような点で困難を感じていらっしゃいますか？一緒に整理してみましょう。", # "input": "仕事がうまくいかなくて悩んでいます。", # "reference": "仕事でお悩みなのですね。うまくいかないと感じると、本当に辛いですよね。具体的にどのような点で困難を感じていらっしゃいますか？一緒に整理してみましょう。" # }, # { # "text": "### Input:\n人間関係で困っています。\n\n### Response:\n人間関係の悩みは本当に心が疲れますよね。お気持ちお察しします。どのような関係性でお困りでしょうか？職場、家族、友人関係など、もう少し詳しくお聞かせいただけますか？", # "input": "人間関係で困っています。", # "reference": "人間関係の悩みは本当に心が疲れますよね。お気持ちお察しします。どのような関係性でお困りでしょうか？職場、家族、友人関係など、もう少し詳しくお聞かせいただけますか？" # }, # { # "text": "### Input:\n将来が不安です。\n\n### Response:\n将来への不安を抱えていらっしゃるのですね。先が見えない不安は、とても重く感じられることと思います。特にどのような点について不安を感じていらっしゃいますか？", # "input": "将来が不安です。", # "reference": "将来への不安を抱えていらっしゃるのですね。先が見えない不安は、とても重く感じられることと思います。特にどのような点について不安を感じていらっしゃいますか？" # }, # { # "text": "### Input:\n自信が持てません。\n\n### Response:\n自信が持てないというお気持ち、よくわかります。多くの方が同じような悩みを抱えています。どのような場面で特に自信が持てないと感じますか？あなたの強みも一緒に見つけていきましょう。", # "input": "自信が持てません。", # "reference": "自信が持てないというお気持ち、よくわかります。多くの方が同じような悩みを抱えています。どのような場面で特に自信が持てないと感じますか？あなたの強みも一緒に見つけていきましょう。" # } # ] # return synthetic_data # def merge_and_save_model(self, force_merge: bool = False): # """Merge LoRA weights with base model and save""" # if os.path.exists(self.merged_model_path) and not force_merge: # print(f"✅ Merged model already exists at {self.merged_model_path}") # return # print("\n🔄 Merging LoRA adapter with base model...") # try: # # Load base model # print(" Loading base model...") # base_model = AutoModelForCausalLM.from_pretrained( # self.base_model_name, # torch_dtype=torch.float16, # device_map="auto" if self.device == "cuda" else None, # trust_remote_code=True, # low_cpu_mem_usage=True # ) # # Check if adapter exists # adapter_config_path = os.path.join(self.finetuned_model_path, "adapter_config.json") # if not os.path.exists(adapter_config_path): # print(f"⚠️ No LoRA adapter found at {self.finetuned_model_path}") # model = base_model # else: # # Load LoRA adapter # print(" Loading LoRA adapter...") # model = PeftModel.from_pretrained( # base_model, # self.finetuned_model_path, # torch_dtype=torch.float16 # ) # # Merge weights # print(" Merging weights...") # model = model.merge_and_unload() # # Save merged model # print(f" Saving merged model to {self.merged_model_path}...") # model.save_pretrained(self.merged_model_path) # # Save tokenizer # tokenizer = AutoTokenizer.from_pretrained( # self.finetuned_model_path # if os.path.exists(os.path.join(self.finetuned_model_path, "tokenizer_config.json")) # else self.base_model_name # ) # tokenizer.save_pretrained(self.merged_model_path) # print("✅ Model merged and saved successfully!") # # Clean up memory # del base_model, model # gc.collect() # torch.cuda.empty_cache() # except Exception as e: # print(f"❌ Error during merging: {e}") # raise # def load_models(self): # """Load base and fine-tuned models for comparison""" # print("\n📚 Loading models for benchmarking...") # # Load tokenizer # self.tokenizer = AutoTokenizer.from_pretrained(self.base_model_name) # if self.tokenizer.pad_token is None: # self.tokenizer.pad_token = self.tokenizer.eos_token # # Load base model # print(" Loading base model...") # self.base_model = AutoModelForCausalLM.from_pretrained( # self.base_model_name, # torch_dtype=torch.float16, # device_map="auto" if self.device == "cuda" else None, # trust_remote_code=True, # low_cpu_mem_usage=True # ) # self.base_model.eval() # # Load merged fine-tuned model # if os.path.exists(self.merged_model_path): # print(" Loading merged fine-tuned model...") # self.finetuned_model = AutoModelForCausalLM.from_pretrained( # self.merged_model_path, # torch_dtype=torch.float16, # device_map="auto" if self.device == "cuda" else None, # trust_remote_code=True, # low_cpu_mem_usage=True # ) # else: # print(" Loading fine-tuned model (attempting PEFT)...") # try: # base_for_peft = AutoModelForCausalLM.from_pretrained( # self.base_model_name, # torch_dtype=torch.float16, # device_map="auto" if self.device == "cuda" else None, # trust_remote_code=True, # low_cpu_mem_usage=True # ) # self.finetuned_model = PeftModel.from_pretrained( # base_for_peft, # self.finetuned_model_path, # torch_dtype=torch.float16 # ) # except: # self.finetuned_model = AutoModelForCausalLM.from_pretrained( # self.finetuned_model_path, # torch_dtype=torch.float16, # device_map="auto" if self.device == "cuda" else None, # trust_remote_code=True, # low_cpu_mem_usage=True # ) # self.finetuned_model.eval() # print("✅ Models loaded successfully!") # def generate_response(self, model, prompt: str, max_length: int = 150) -> str: # """Generate response from model""" # inputs = self.tokenizer( # prompt, # return_tensors="pt", # truncation=True, # max_length=512 # ) # if self.device == "cuda": # inputs = {k: v.cuda() for k, v in inputs.items()} # with torch.no_grad(): # outputs = model.generate( # **inputs, # max_new_tokens=max_length, # temperature=0.7, # do_sample=True, # top_p=0.9, # pad_token_id=self.tokenizer.pad_token_id, # eos_token_id=self.tokenizer.eos_token_id # ) # response = self.tokenizer.decode(outputs[0], skip_special_tokens=True) # # Extract only the generated response # if "### Response:" in response: # response = response.split("### Response:")[-1].strip() # elif "Response:" in response: # response = response.split("Response:")[-1].strip() # else: # # Remove the input prompt from response # response = response[len(prompt):].strip() # return response # def tokenize_japanese(self, text: str) -> List[str]: # """Tokenize Japanese text for BLEU calculation""" # # Simple character-based tokenization for Japanese # # In production, use MeCab or similar for better tokenization # import re # # Remove special characters and split # text = re.sub(r'[。、！？\n]', ' ', text) # tokens = text.strip().split() # # Character-level tokenization as fallback # if not tokens: # tokens = list(text.strip()) # return tokens # def calculate_bleu_scores(self, reference: str, hypothesis: str) -> Dict[str, float]: # """Calculate BLEU-1, BLEU-2, BLEU-3, BLEU-4 scores""" # # Tokenize texts # ref_tokens = self.tokenize_japanese(reference) # hyp_tokens = self.tokenize_japanese(hypothesis) # # Calculate BLEU scores with different n-grams # scores = {} # # BLEU-1 (unigram) # scores['BLEU-1'] = sentence_bleu( # [ref_tokens], hyp_tokens, # weights=(1.0, 0, 0, 0), # smoothing_function=self.smoothing # ) # # BLEU-2 (bigram) # scores['BLEU-2'] = sentence_bleu( # [ref_tokens], hyp_tokens, # weights=(0.5, 0.5, 0, 0), # smoothing_function=self.smoothing # ) # # BLEU-3 (trigram) # scores['BLEU-3'] = sentence_bleu( # [ref_tokens], hyp_tokens, # weights=(0.33, 0.33, 0.34, 0), # smoothing_function=self.smoothing # ) # # BLEU-4 (4-gram) # scores['BLEU-4'] = sentence_bleu( # [ref_tokens], hyp_tokens, # weights=(0.25, 0.25, 0.25, 0.25), # smoothing_function=self.smoothing # ) # return scores # def calculate_rouge_scores(self, reference: str, hypothesis: str) -> Dict[str, float]: # """Calculate ROUGE-1, ROUGE-2, ROUGE-L scores""" # scores = self.rouge_scorer.score(reference, hypothesis) # return { # 'ROUGE-1': scores['rouge1'].fmeasure, # 'ROUGE-2': scores['rouge2'].fmeasure, # 'ROUGE-L': scores['rougeL'].fmeasure # } # def run_bleu_rouge_benchmark(self, num_samples: int = None): # """Run comprehensive BLEU and ROUGE benchmark""" # print("\n" + "="*70) # print("🏃 RUNNING BLEU & ROUGE BENCHMARK") # print("="*70) # # Load test data # test_data = self.load_test_data() # if num_samples: # test_data = test_data[:num_samples] # print(f" Using {num_samples} samples for benchmarking") # # Initialize score collectors # base_scores = defaultdict(list) # finetuned_scores = defaultdict(list) # # Metrics to calculate # metrics = ['BLEU-1', 'BLEU-2', 'BLEU-3', 'BLEU-4', # 'ROUGE-1', 'ROUGE-2', 'ROUGE-L'] # print(f"\n📊 Evaluating {len(test_data)} test examples...") # print("-" * 70) # detailed_results = [] # for i, example in enumerate(tqdm(test_data, desc="Evaluating")): # # Extract input and reference # if 'input' in example: # input_text = example['input'] # else: # # Try to extract from text field # if "### Input:" in example['text']: # input_text = example['text'].split("### Input:")[1].split("### Response:")[0].strip() # else: # input_text = example['text'].split("\n")[0].strip() # if 'reference' in example: # reference = example['reference'] # else: # # Try to extract from text field # if "### Response:" in example['text']: # reference = example['text'].split("### Response:")[1].strip() # else: # parts = example['text'].split("\n") # reference = parts[1] if len(parts) > 1 else parts[0] # # Format input for models # formatted_input = f"### Instruction:\nあなたは思いやりのある心理カウンセラーです。\n\n### Input:\n{input_text}\n\n### Response:\n" # # Generate responses # base_response = self.generate_response(self.base_model, formatted_input) # finetuned_response = self.generate_response(self.finetuned_model, formatted_input) # # Calculate BLEU scores # base_bleu = self.calculate_bleu_scores(reference, base_response) # finetuned_bleu = self.calculate_bleu_scores(reference, finetuned_response) # # Calculate ROUGE scores # base_rouge = self.calculate_rouge_scores(reference, base_response) # finetuned_rouge = self.calculate_rouge_scores(reference, finetuned_response) # # Combine scores # base_all_scores = {**base_bleu, **base_rouge} # finetuned_all_scores = {**finetuned_bleu, **finetuned_rouge} # # Collect scores # for metric in metrics: # base_scores[metric].append(base_all_scores[metric]) # finetuned_scores[metric].append(finetuned_all_scores[metric]) # # Store detailed results # detailed_results.append({ # 'input': input_text, # 'reference': reference, # 'base_response': base_response, # 'finetuned_response': finetuned_response, # 'base_scores': base_all_scores, # 'finetuned_scores': finetuned_all_scores # }) # # Print sample results # if i < 3: # Show first 3 examples # print(f"\n📝 Example {i+1}:") # print(f" Input: {input_text[:50]}...") # print(f" Reference: {reference[:50]}...") # print(f" Base response: {base_response[:50]}...") # print(f" Fine-tuned response: {finetuned_response[:50]}...") # print(f" Base BLEU-4: {base_bleu['BLEU-4']:.3f}") # print(f" Fine-tuned BLEU-4: {finetuned_bleu['BLEU-4']:.3f}") # # Calculate aggregate statistics # print("\n" + "="*70) # print("📈 BENCHMARK RESULTS") # print("="*70) # self.results = { # 'detailed_results': detailed_results, # 'aggregate_scores': {}, # 'improvements': {} # } # # Print and store results # print("\n" + "-"*70) # print(f"{'Metric':<12} {'Base Model':<20} {'Fine-tuned Model':<20} {'Improvement':<15}") # print("-"*70) # for metric in metrics: # base_mean = np.mean(base_scores[metric]) # base_std = np.std(base_scores[metric]) # finetuned_mean = np.mean(finetuned_scores[metric]) # finetuned_std = np.std(finetuned_scores[metric]) # # Calculate improvement # if base_mean > 0: # improvement = ((finetuned_mean - base_mean) / base_mean) * 100 # else: # improvement = 0 # # Store results # self.results['aggregate_scores'][metric] = { # 'base_mean': base_mean, # 'base_std': base_std, # 'finetuned_mean': finetuned_mean, # 'finetuned_std': finetuned_std # } # self.results['improvements'][metric] = improvement # # Print results # base_str = f"{base_mean:.3f} (±{base_std:.3f})" # finetuned_str = f"{finetuned_mean:.3f} (±{finetuned_std:.3f})" # imp_str = f"{improvement:+.1f}%" # # Color code improvement # if improvement > 0: # imp_str = f"✅ {imp_str}" # elif improvement < 0: # imp_str = f"⚠️ {imp_str}" # else: # imp_str = f"➖ {imp_str}" # print(f"{metric:<12} {base_str:<20} {finetuned_str:<20} {imp_str:<15}") # # Calculate overall scores # print("\n" + "="*70) # print("🎯 OVERALL PERFORMANCE") # print("="*70) # # Average BLEU score # bleu_metrics = ['BLEU-1', 'BLEU-2', 'BLEU-3', 'BLEU-4'] # base_bleu_avg = np.mean([np.mean(base_scores[m]) for m in bleu_metrics]) # finetuned_bleu_avg = np.mean([np.mean(finetuned_scores[m]) for m in bleu_metrics]) # bleu_improvement = ((finetuned_bleu_avg - base_bleu_avg) / base_bleu_avg) * 100 if base_bleu_avg > 0 else 0 # # Average ROUGE score # rouge_metrics = ['ROUGE-1', 'ROUGE-2', 'ROUGE-L'] # base_rouge_avg = np.mean([np.mean(base_scores[m]) for m in rouge_metrics]) # finetuned_rouge_avg = np.mean([np.mean(finetuned_scores[m]) for m in rouge_metrics]) # rouge_improvement = ((finetuned_rouge_avg - base_rouge_avg) / base_rouge_avg) * 100 if base_rouge_avg > 0 else 0 # # Overall average # base_overall = np.mean([np.mean(base_scores[m]) for m in metrics]) # finetuned_overall = np.mean([np.mean(finetuned_scores[m]) for m in metrics]) # overall_improvement = ((finetuned_overall - base_overall) / base_overall) * 100 if base_overall > 0 else 0 # self.results['summary'] = { # 'bleu_average': { # 'base': base_bleu_avg, # 'finetuned': finetuned_bleu_avg, # 'improvement': bleu_improvement # }, # 'rouge_average': { # 'base': base_rouge_avg, # 'finetuned': finetuned_rouge_avg, # 'improvement': rouge_improvement # }, # 'overall': { # 'base': base_overall, # 'finetuned': finetuned_overall, # 'improvement': overall_improvement # } # } # print(f"\n📊 Average BLEU Score:") # print(f" Base Model: {base_bleu_avg:.3f}") # print(f" Fine-tuned Model: {finetuned_bleu_avg:.3f}") # print(f" Improvement: {bleu_improvement:+.1f}%") # print(f"\n📊 Average ROUGE Score:") # print(f" Base Model: {base_rouge_avg:.3f}") # print(f" Fine-tuned Model: {finetuned_rouge_avg:.3f}") # print(f" Improvement: {rouge_improvement:+.1f}%") # print(f"\n🎯 Overall Average:") # print(f" Base Model: {base_overall:.3f}") # print(f" Fine-tuned Model: {finetuned_overall:.3f}") # print(f" Improvement: {overall_improvement:+.1f}%") # print("="*70) # return self.results # def visualize_results(self, save_path: str = "bleu_rouge_benchmark.png"): # """Create comprehensive visualization of BLEU and ROUGE results""" # if 'aggregate_scores' not in self.results: # print("❌ No results to visualize. Run benchmark first.") # return # print("\n📊 Creating visualizations...") # fig, axes = plt.subplots(2, 3, figsize=(18, 12)) # # Color scheme # base_color = '#3498db' # finetuned_color = '#e74c3c' # improvement_positive = '#27ae60' # improvement_negative = '#c0392b' # # 1. BLEU Scores Comparison # bleu_metrics = ['BLEU-1', 'BLEU-2', 'BLEU-3', 'BLEU-4'] # bleu_base = [self.results['aggregate_scores'][m]['base_mean'] for m in bleu_metrics] # bleu_finetuned = [self.results['aggregate_scores'][m]['finetuned_mean'] for m in bleu_metrics] # x = np.arange(len(bleu_metrics)) # width = 0.35 # axes[0, 0].bar(x - width/2, bleu_base, width, label='Base Model', # color=base_color, alpha=0.8) # axes[0, 0].bar(x + width/2, bleu_finetuned, width, label='Fine-tuned Model', # color=finetuned_color, alpha=0.8) # axes[0, 0].set_xlabel('BLEU Metrics') # axes[0, 0].set_ylabel('Score') # axes[0, 0].set_title('BLEU Score Comparison') # axes[0, 0].set_xticks(x) # axes[0, 0].set_xticklabels(bleu_metrics) # axes[0, 0].legend() # axes[0, 0].grid(True, alpha=0.3) # axes[0, 0].set_ylim([0, max(max(bleu_base), max(bleu_finetuned)) * 1.2]) # # 2. ROUGE Scores Comparison # rouge_metrics = ['ROUGE-1', 'ROUGE-2', 'ROUGE-L'] # rouge_base = [self.results['aggregate_scores'][m]['base_mean'] for m in rouge_metrics] # rouge_finetuned = [self.results['aggregate_scores'][m]['finetuned_mean'] for m in rouge_metrics] # x = np.arange(len(rouge_metrics)) # axes[0, 1].bar(x - width/2, rouge_base, width, label='Base Model', # color=base_color, alpha=0.8) # axes[0, 1].bar(x + width/2, rouge_finetuned, width, label='Fine-tuned Model', # color=finetuned_color, alpha=0.8) # axes[0, 1].set_xlabel('ROUGE Metrics') # axes[0, 1].set_ylabel('Score') # axes[0, 1].set_title('ROUGE Score Comparison') # axes[0, 1].set_xticks(x) # axes[0, 1].set_xticklabels(rouge_metrics) # axes[0, 1].legend() # axes[0, 1].grid(True, alpha=0.3) # axes[0, 1].set_ylim([0, max(max(rouge_base), max(rouge_finetuned)) * 1.2]) # # 3. Improvement Percentages # all_metrics = bleu_metrics + rouge_metrics # improvements = [self.results['improvements'][m] for m in all_metrics] # colors = [improvement_positive if imp > 0 else improvement_negative for imp in improvements] # axes[0, 2].barh(range(len(all_metrics)), improvements, color=colors, alpha=0.7) # axes[0, 2].set_yticks(range(len(all_metrics))) # axes[0, 2].set_yticklabels(all_metrics) # axes[0, 2].set_xlabel('Improvement (%)') # axes[0, 2].set_title('Performance Improvement by Metric') # axes[0, 2].axvline(x=0, color='black', linestyle='-', linewidth=0.5) # axes[0, 2].grid(True, alpha=0.3, axis='x') # # 4. Line plot showing progression # axes[1, 0].plot(bleu_metrics, bleu_base, 'o-', label='Base Model', # color=base_color, linewidth=2, markersize=8) # axes[1, 0].plot(bleu_metrics, bleu_finetuned, 's-', label='Fine-tuned Model', # color=finetuned_color, linewidth=2, markersize=8) # axes[1, 0].set_xlabel('BLEU N-gram') # axes[1, 0].set_ylabel('Score') # axes[1, 0].set_title('BLEU Score Progression') # axes[1, 0].legend() # axes[1, 0].grid(True, alpha=0.3) # # 5. Summary Statistics # ax5 = axes[1, 1] # ax5.axis('off') # summary_text = f""" # BENCHMARK SUMMARY # {'='*30} # BLEU Average: # Base: {self.results['summary']['bleu_average']['base']:.3f} # Fine-tuned: {self.results['summary']['bleu_average']['finetuned']:.3f} # Improvement: {self.results['summary']['bleu_average']['improvement']:+.1f}% # ROUGE Average: # Base: {self.results['summary']['rouge_average']['base']:.3f} # Fine-tuned: {self.results['summary']['rouge_average']['finetuned']:.3f} # Improvement: {self.results['summary']['rouge_average']['improvement']:+.1f}% # Overall Performance: # Base: {self.results['summary']['overall']['base']:.3f} # Fine-tuned: {self.results['summary']['overall']['finetuned']:.3f} # Improvement: {self.results['summary']['overall']['improvement']:+.1f}% # Best Improvements: # """ # # Find best improvements # sorted_metrics = sorted(all_metrics, # key=lambda m: self.results['improvements'][m], # reverse=True) # for m in sorted_metrics[:2]: # summary_text += f" • {m}: {self.results['improvements'][m]:+.1f}%\n" # if any(self.results['improvements'][m] < 0 for m in all_metrics): # summary_text += f"\nNeeds Attention:\n" # for m in sorted_metrics[-2:]: # if self.results['improvements'][m] < 0: # summary_text += f" • {m}: {self.results['improvements'][m]:+.1f}%\n" # ax5.text(0.1, 0.9, summary_text, transform=ax5.transAxes, # fontsize=10, verticalalignment='top', fontfamily='monospace') # # 6. Heatmap of all scores # metrics_for_heatmap = all_metrics # models = ['Base', 'Fine-tuned'] # heatmap_data = [] # for metric in metrics_for_heatmap: # heatmap_data.append([ # self.results['aggregate_scores'][metric]['base_mean'], # self.results['aggregate_scores'][metric]['finetuned_mean'] # ]) # im = axes[1, 2].imshow(heatmap_data, cmap='YlOrRd', aspect='auto') # axes[1, 2].set_xticks(np.arange(len(models))) # axes[1, 2].set_yticks(np.arange(len(metrics_for_heatmap))) # axes[1, 2].set_xticklabels(models) # axes[1, 2].set_yticklabels(metrics_for_heatmap) # axes[1, 2].set_title('Score Heatmap') # # Add text annotations # for i in range(len(metrics_for_heatmap)): # for j in range(len(models)): # text = axes[1, 2].text(j, i, f'{heatmap_data[i][j]:.3f}', # ha="center", va="center", color="black", fontsize=8) # plt.colorbar(im, ax=axes[1, 2]) # plt.suptitle('BLEU & ROUGE Benchmark Results', fontsize=16, fontweight='bold') # plt.tight_layout() # plt.savefig(save_path, dpi=300, bbox_inches='tight') # print(f"✅ Visualization saved to {save_path}") # plt.show() # def save_results(self, output_path: str = "bleu_rouge_results.json"): # """Save benchmark results to JSON""" # # Convert numpy types to Python native types for JSON serialization # def convert_to_native(obj): # if isinstance(obj, np.floating): # return float(obj) # elif isinstance(obj, np.integer): # return int(obj) # elif isinstance(obj, np.ndarray): # return obj.tolist() # elif isinstance(obj, dict): # return {k: convert_to_native(v) for k, v in obj.items()} # elif isinstance(obj, list): # return [convert_to_native(item) for item in obj] # return obj # results_native = convert_to_native(self.results) # with open(output_path, 'w', encoding='utf-8') as f: # json.dump(results_native, f, ensure_ascii=False, indent=2) # print(f"✅ Results saved to {output_path}") # def generate_detailed_report(self, output_path: str = "bleu_rouge_report.md"): # """Generate detailed markdown report""" # if not self.results: # print("❌ No results to report. Run benchmark first.") # return # report = f"""# BLEU & ROUGE Benchmark Report # Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} # ## Executive Summary # Comprehensive evaluation of the fine-tuned counseling model using BLEU and ROUGE metrics. # ### Overall Performance # - **Base Model Score**: {self.results['summary']['overall']['base']:.3f} # - **Fine-tuned Model Score**: {self.results['summary']['overall']['finetuned']:.3f} # - **Overall Improvement**: {self.results['summary']['overall']['improvement']:+.1f}% # ## Detailed Metrics # ### BLEU Scores # | Metric | Base Model | Fine-tuned Model | Improvement | # |--------|------------|------------------|-------------| # """ # for metric in ['BLEU-1', 'BLEU-2', 'BLEU-3', 'BLEU-4']: # scores = self.results['aggregate_scores'][metric] # report += f"| {metric} | {scores['base_mean']:.3f} (±{scores['base_std']:.3f}) | " # report += f"{scores['finetuned_mean']:.3f} (±{scores['finetuned_std']:.3f}) | " # report += f"{self.results['improvements'][metric]:+.1f}% |\n" # report += f""" # **BLEU Average**: {self.results['summary']['bleu_average']['improvement']:+.1f}% improvement # ### ROUGE Scores # | Metric | Base Model | Fine-tuned Model | Improvement | # |--------|------------|------------------|-------------| # """ # for metric in ['ROUGE-1', 'ROUGE-2', 'ROUGE-L']: # scores = self.results['aggregate_scores'][metric] # report += f"| {metric} | {scores['base_mean']:.3f} (±{scores['base_std']:.3f}) | " # report += f"{scores['finetuned_mean']:.3f} (±{scores['finetuned_std']:.3f}) | " # report += f"{self.results['improvements'][metric]:+.1f}% |\n" # report += f""" # **ROUGE Average**: {self.results['summary']['rouge_average']['improvement']:+.1f}% improvement # ## Sample Outputs # """ # # Add sample outputs # for i, result in enumerate(self.results['detailed_results'][:3]): # report += f"""### Example {i+1} # **Input**: {result['input']} # **Reference**: {result['reference'][:200]}... # **Base Model Response**: {result['base_response'][:200]}... # **Fine-tuned Model Response**: {result['finetuned_response'][:200]}... # **Scores**: # - Base BLEU-4: {result['base_scores']['BLEU-4']:.3f}, ROUGE-L: {result['base_scores']['ROUGE-L']:.3f} # - Fine-tuned BLEU-4: {result['finetuned_scores']['BLEU-4']:.3f}, ROUGE-L: {result['finetuned_scores']['ROUGE-L']:.3f} # --- # """ # report += """## Analysis & Recommendations # """ # overall_imp = self.results['summary']['overall']['improvement'] # if overall_imp < -10: # report += """### ⚠️ Significant Performance Degradation # The fine-tuned model shows significant degradation in BLEU/ROUGE scores. This indicates: # 1. **Catastrophic Forgetting**: The model has lost its language generation capabilities # 2. **Overfitting**: The model memorized training data instead of learning patterns # 3. **Format Mismatch**: Training and inference formats may differ # **Immediate Actions Required**: # - ✅ Ensure proper model merging (LoRA weights with base model) # - ✅ Reduce learning rate (try 1e-5 or 2e-5) # - ✅ Use smaller LoRA rank (r=4 or r=8) # - ✅ Mix general conversation data with counseling data (80/20 ratio) # - ✅ Implement regularization (weight decay=0.1, dropout=0.1) # - ✅ Use early stopping with patience=3 # """ # elif overall_imp < 0: # report += """### ⚠️ Minor Performance Degradation # The model shows slight degradation. Common causes: # 1. **Aggressive Fine-tuning**: Parameters changed too much # 2. **Limited Training Data**: Not enough diverse examples # 3. **Domain Shift**: Counseling domain too different from base training # **Recommended Actions**: # - ✅ Fine-tune for fewer epochs (1-2 instead of 3) # - ✅ Use gradient accumulation for larger effective batch size # - ✅ Implement knowledge distillation from base model # - ✅ Add more diverse training examples # """ # elif overall_imp < 10: # report += """### 📊 Modest Improvement # The model shows small but positive improvements. # **To Further Improve**: # - ✅ Increase training data quality and quantity # - ✅ Experiment with different generation parameters # - ✅ Fine-tune on domain-specific pre-training # - ✅ Use ensemble methods with base model # """ # else: # report += """### ✅ Significant Improvement # Excellent results! The fine-tuned model shows substantial improvements. # **Next Steps**: # - ✅ Deploy for A/B testing with users # - ✅ Monitor performance on edge cases # - ✅ Consider model compression for deployment # - ✅ Collect user feedback for iterative improvement # """ # with open(output_path, 'w', encoding='utf-8') as f: # f.write(report) # print(f"✅ Detailed report saved to {output_path}") # # Main execution # if __name__ == "__main__": # import argparse # parser = argparse.ArgumentParser(description='Advanced BLEU & ROUGE Benchmark') # parser.add_argument('--base_model', type=str, default='LiquidAI/LFM2-2.6B', # help='Base model name') # parser.add_argument('--finetuned_path', type=str, default='./counselor_model/best_model', # help='Path to fine-tuned model') # parser.add_argument('--merged_path', type=str, default='./merged_counselor_mode_2b', # help='Path to save/load merged model') # parser.add_argument('--test_data', type=str, default='./processed_data_score80/test.jsonl', # help='Path to test data') # parser.add_argument('--num_samples', type=int, default=None, # help='Number of samples to evaluate (None for all)') # parser.add_argument('--force_merge', action='store_true', # help='Force re-merge even if merged model exists') # parser.add_argument('--skip_merge', action='store_true', # help='Skip merging step') # parser.add_argument('--output_dir', type=str, default='./benchmark_results', # help='Directory to save results') # args = parser.parse_args() # # Create output directory # os.makedirs(args.output_dir, exist_ok=True) # try: # # Initialize benchmark # print("🚀 Initializing Advanced BLEU & ROUGE Benchmark") # benchmark = AdvancedCounselorBenchmark( # base_model_name=args.base_model, # finetuned_model_path=args.finetuned_path, # merged_model_path=args.merged_path, # test_data_path=args.test_data # ) # # Merge models if needed # if not args.skip_merge: # benchmark.merge_and_save_model(force_merge=args.force_merge) # # Load models # benchmark.load_models() # # Run BLEU & ROUGE benchmark # results = benchmark.run_bleu_rouge_benchmark(num_samples=args.num_samples) # # Save results # benchmark.save_results(os.path.join(args.output_dir, "bleu_rouge_results_2b.json")) # # Generate visualizations # benchmark.visualize_results(os.path.join(args.output_dir, "bleu_rouge_visualization_2b.png")) # # Generate detailed report # benchmark.generate_detailed_report(os.path.join(args.output_dir, "bleu_rouge_report_2b.md")) # print("\n✅ BLEU & ROUGE Benchmarking completed successfully!") # print(f"📁 Results saved to {args.output_dir}/") # except Exception as e: # print(f"\n❌ Error during benchmarking: {e}") # import traceback # traceback.print_exc()