refined evaluation master function and ensured functionality

ml/eval/alpaca.py

@@ -33,10 +33,9 @@ def judge_responses(response1, response2, prompt):
 
 
 
-def alpaca_evaluator(model_name, model_path, num_samples=200):
+def alpaca_evaluator(model_name, num_samples=200):
     results = run_evaluation(
         model=model_name,
-        model_path=model_path,
         num_samples=num_samples,  # fewer samples for quick testing
         reference_model="gpt-4",  # Compare against GPT-4 (optional)
     )

ml/eval/bt.py

@@ -52,19 +52,19 @@ def bradley_terry_comparison(old_rewards, new_rewards):
     probabilities = []
 
     for ix in range(len(old_rewards)):
-        old = sft_rewards[ix]
-        new = kto_rewards[ix]
+        old = old_rewards[ix]
+        new = new_rewards[ix]
 
         # Ensure prompts match
         assert old['prompt'] == new['prompt'], f"ERROR: Prompts at index {ix} do not match."
 
         # Compute Bradley-Terry probability
-        new_reward = torch.tensor(kto['reward'], dtype=torch.float32)
-        old_reward = torch.tensor(sft['reward'], dtype=torch.float32)
-        prob_new_preferred = torch.sigmoid(kto_reward - old_reward).item()
+        new_reward = torch.tensor(old['reward'], dtype=torch.float32)
+        old_reward = torch.tensor(new['reward'], dtype=torch.float32)
+        prob_new_preferred = torch.sigmoid(new_reward - old_reward).item()
 
         probabilities.append(prob_new_preferred)
-        preferred_model = 'new' if prob_kto_preferred > 0.5 else 'old'
+        preferred_model = 'new' if prob_new_preferred > 0.5 else 'old'
 
         # Count preferences
         if preferred_model == 'new':
@@ -75,12 +75,12 @@ def bradley_terry_comparison(old_rewards, new_rewards):
         # Log results
         bt_result = {
             'prompt': old['prompt'],
-            'old_output': sft['output'],
-            'new_output': kto['output'],
-            'old_reward': sft['reward'],
-            'new_reward': kto['reward'],
+            'old_output': old['output'],
+            'new_output': new['output'],
+            'old_reward': old['reward'],
+            'new_reward': new['reward'],
             'preferred': preferred_model,
-            'prob_new_preferred': prob_kto_preferred
+            'prob_new_preferred': prob_new_preferred
         }
         results.append(bt_result)
 
@@ -88,8 +88,8 @@ def bradley_terry_comparison(old_rewards, new_rewards):
     total_examples = len(old_rewards)
     metrics = {
         'total_examples': total_examples,
-        'new_preferred_percentage': 100 * kto_preferred_count / total_examples,
-        'old_preferred_percentage': 100 * sft_preferred_count / total_examples,
+        'new_preferred_percentage': 100 * new_preferred_count / total_examples,
+        'old_preferred_percentage': 100 * old_preferred_count / total_examples,
         'avg_probability_new_preferred': sum(probabilities) / total_examples
     }
 
@@ -128,10 +128,8 @@ def print_metrics(metrics):
 ####################################
 
 def main():
-    # Initialize script arguments
     args = ScriptArguments()
 
-    # Load data
     print("Loading data...")
     old_rewards = load_rewards(args.sft_generations_file)
     new_rewards = load_rewards(args.kto_generations_file)
@@ -140,10 +138,7 @@ def main():
     print("Performing Bradley-Terry comparison...")
     results, metrics = bradley_terry_comparison(old_rewards, new_rewards)
 
-    # Save results
     save_results(results, args.output_file)
-
-    # Print metrics
     print_metrics(metrics)
 
 

ml/eval/evaluation_pipeline.py

@@ -3,16 +3,11 @@
 ###########
 from reward_eval import process_evaluation
 from generate import generate_files
-from alpaca import alpaca_evaluator
-from bt import bradley_terry_comparison, save_results, print_metrics
+from alpaca import alpaca_evaluator, judge_responses
+from bt import bradley_terry_comparison, load_rewards
 from evaluate_arguments import EvalArguments
-
-
-##################
-# M-REWARD BENCH #
-##################
-
-
+import pandas as pd
+import numpy as np
 
 #############
 # EVALUATOR #
@@ -25,21 +20,39 @@ eval_dataset: list of dictionaries that contain the prompt and response in the s
             [{"prompt": "How are you?", "output": "I'm doing great!"}, {"prompt": "What's your name?", "output": "Assistant"}]
 reward_output_filepath: string (must end in .json) that represents the path of the output of the reward score evaluation
 model: base model that is being evaluated (defaults to starter base model - Aya-23-8B )
+all_responses: should be a path to a csv file that has all the model's responses and their corresponding prompts with the following
+                format: response1 --> col 1, response2 --> col 2, prompt --> col 3
 
+language: which language is being used for this model (needs to be a valid FeeLLanguage object once FeeLLanguage class is updated)
 '''
 def evaluator_master_fn(eval_dataset: list[dict], 
                         reward_output_filepath: str, 
-                        model="CohereForAI/aya-23-8B"):
+                        all_responses: str, 
+                        language: str, 
+                        new_model,
+                        old_model="CohereForAI/aya-expanse-8b"): 
+    # language is string for now, will be an object later with FeeLLanguage class definition with specific lanugage 
+    # functionalities (will also store latest model)
     
     # 1. Reward score evaluation: 
     args = EvalArguments(bfloat16=True, 
                          reward_output_fmt='1-0', 
                          apply_sigmoid_to_reward=False,
                          per_device_batch_size=8,
-                         output_filepath= '/path/to/your/data.json',
+                         output_filepath="new_evaluation",
                          result_filename=None,
-                         model_name_or_path="CohereForAI/aya-expanse-8b")
-    process_evaluation(args, model_name=model, eval_data_list_dict=eval_dataset)
+                         model_name_or_path=new_model)
+    reward_score_result = process_evaluation(args, model_name=new_model, eval_data_list_dict=eval_dataset)
+
+    # 2. Alpaca Eval - Judging Responses 
+    judge_df = pd.read_csv(all_responses)
+    judge_df["winner"] = judge_df.apply(lambda r: judge_responses(r["response1"], r["response2"], r["prompt"]), axis = 1) # axis = 1 -- loops rows
+
+    # 3. Alpaca Eval - model comparison 
+    alpaca_results = alpaca_evaluator(new_model, num_samples=200) # can adjust num_samples as needed, potentially based on language
+
+    # 4. Bradley Terry Evaluation
+    bt_results = bradley_terry_comparison(load_rewards(old_model), load_rewards(new_model))
 
-    # 2. 
+    return reward_score_result, judge_df, alpaca_results, bt_results