Upload 7 files

datasets_loader.py

@@ -0,0 +1,329 @@
+from abc import ABC, abstractmethod
+import re
+from math_verify import parse, verify
+import pandas
+from datasets import load_dataset
+import random
+ANSWER_PATTERN_MULTICHOICE = r"(?:\$\$\s*)?\\boxed\{[^}]*?([A-Z])[^}]*\}(?:\s*\$\$)?|(?:\*{0,2}\s*)?(?:Final|Correct)\s*Answer:\s*([A-Z])\."
+ANSWER_PATTERN = r"(?i)Answer\s*:\s*([^\n]+)"
+ANSWER_PATTERN_BOXED = r"(?i)\\boxed\s*{([^\n]+)}"
+
+class DatasetHandler(ABC):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED, num_examples: int = None):
+        self.answer_pattern = answer_pattern
+        self.num_examples = num_examples if num_examples is not None else 1
+
+    @abstractmethod
+    def load_data(self):
+        """
+        Load the dataset and return a tuple: (splits_dict, answer_type).
+
+        splits_dict: A dictionary where each key is a split name (e.g., 'train', 'test')
+                     and the value is the corresponding dataset or data structure.
+        answer_type: A string describing the type of the answer, e.g.:
+                     'number', 'text', 'option letter', etc.
+        """
+        pass
+
+    def extract_answer(self, response: str) -> str:
+        try:
+            return re.search(self.answer_pattern, response).group(1)
+        except:
+            return None
+
+    def compare_answer(self, response: str, answer: str) -> bool:
+        response_answer = self.extract_answer(response)
+        answer = str(answer)
+        response_answer = str(response_answer)    
+        if response_answer is None:
+            return False
+        if self.answer_pattern == ANSWER_PATTERN_MULTICHOICE:
+            return response_answer == answer
+        return verify(parse(answer), parse(response_answer))
+
+    def get_score(self, responses: str, answers: str) -> float:
+        scores = []
+        for r,a in zip(responses, answers):
+            if self.compare_answer(r,a):
+                scores.append(1)
+            else:
+                scores.append(0)
+        return scores, sum(scores)/len(scores)
+
+class MathDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+
+    def load_data(self):
+        df = pandas.read_csv(
+            f"https://openaipublic.blob.core.windows.net/simple-evals/math_500_test.csv"
+        )
+        examples = [row.to_dict() for _, row in df.iterrows()]
+        questions = [example['Question'] for example in examples]
+        answers = [example['Answer'] for example in examples]
+
+        return questions, answers
+
+class Gsm8kDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("openai/gsm8k", 'main', split='test')
+        examples = [row for row in dataset]
+        questions = [example['question'] for example in examples]
+        answers = [example["answer"].split('#### ')[-1] for example in examples]
+        return questions, answers
+
+class AmcDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("zwhe99/amc23", split='test')
+        examples = [row for row in dataset]
+        questions = [example['question'] for example in examples]  *32
+        answers = [example['answer'] for example in examples]  *32
+
+        return questions, answers
+
+class MinervaDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("zwhe99/simplerl-minerva-math", split='test')
+        examples = [row for row in dataset]
+        questions = [example['problem'] for example in examples]
+        answers = [example['answer'] for example in examples]
+
+        return questions, answers
+
+class OlympiadDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("zwhe99/simplerl-OlympiadBench", split='test')
+        examples = [row for row in dataset]
+        questions = [example['question'] for example in examples]
+        answers = [example['final_answer'][0] for example in examples]
+
+        return questions, answers
+
+class Aime2024DatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("HuggingFaceH4/aime_2024", split='train')
+        examples = [row for row in dataset]
+        questions = [example['problem'] for example in examples]*32
+        answers = [example['answer'] for example in examples]*32
+
+        return questions, answers
+
+
+class Aime2025DatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("yentinglin/aime_2025", 'default')['train']
+        examples = [row for row in dataset]
+        questions = [example['problem'] for example in examples]*32
+        answers = [example['answer'] for example in examples]*32
+
+        return questions, answers
+
+class MmluProDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_MULTICHOICE):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset('TIGER-Lab/MMLU-Pro', split='test')
+        examples = []
+        for row in dataset:
+            example = {
+                'question': row['question'],
+                'options': row['options'],
+                'answer': row['answer'],
+                'answer_index': row['answer_index'],
+                'category': row['category'],
+                'cot_content': row['cot_content'],
+                'src': row['src']
+            }
+            examples.append(example)
+        random.shuffle(examples)
+        examples = examples[:1000]
+        questions = []
+        answers = []
+        for example in examples:
+            # Format question with options
+            question = example['question'] + "\n\nOptions:\n"
+            for i, opt in enumerate(example['options']):
+                question += f"{chr(65+i)}. {opt}\n"
+            
+            questions.append(question)
+            answers.append(example['answer'])
+            
+        return questions, answers
+
+class bbehDatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("MrLight/bbeh-eval", split='train')
+        examples = [row for row in dataset]
+        random.shuffle(examples)
+        examples = examples[:1000]
+        questions = [example['question'] for example in examples]
+        answers = [example['answer'] for example in examples]
+
+        return questions, answers
+
+class SuperGPQADatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_MULTICHOICE):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset('m-a-p/SuperGPQA')
+        examples = []
+        for row in dataset['train']:
+            example = {
+                'question': row['question'],
+                'options': row['options'],
+                'answer': row['answer_letter']
+            }
+            examples.append(example)        
+        random.shuffle(examples)
+        examples = examples[:1000]
+        
+        questions = []
+        answers = []
+        for example in examples:
+            # Format question with options
+            question = example['question'] + "\n\nOptions:\n"
+            for i, opt in enumerate(example['options']):
+                question += f"{chr(65+i)}. {opt}\n"
+            
+            questions.append(question)
+            answers.append(example['answer'])
+            
+        return questions, answers
+
+class GPQA_DatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str = ANSWER_PATTERN_MULTICHOICE):
+        super().__init__(answer_pattern)
+    
+    def load_data(self):
+        dataset = load_dataset("Idavidrein/gpqa", "gpqa_diamond",'train')
+        examples = []
+        
+        for row in dataset:
+            # Get the question and answers
+            question = row['Question']
+            options = [
+                row['Correct Answer'],
+                row['Incorrect Answer 1'],
+                row['Incorrect Answer 2'],
+                row['Incorrect Answer 3']
+            ]
+            # Shuffle options to randomize correct answer position
+            random.shuffle(options)
+            # Find the index of correct answer after shuffling
+            correct_index = options.index(row['Correct Answer'])
+            correct_option = chr(65 + correct_index)
+            
+            example = {
+                'question': question,
+                'options': options,
+                'answer': correct_option
+            }
+            examples.append(example)
+        
+        # Shuffle and limit to 1000 examples like other handlers
+        random.shuffle(examples)
+        examples = examples[:1000]
+        
+        questions = []
+        answers = []
+        for example in examples:
+            # Format question with options
+            question = example['question'] + "\n\nOptions:\n"
+            for i, opt in enumerate(example['options']):
+                question += f"{chr(65+i)}. {opt}\n"
+            
+            questions.append(question)
+            answers.append(example['answer'])
+            
+        return questions, answers
+
+
+class Mydataset_DatasetHandler(DatasetHandler):
+    def __init__(self, answer_pattern: str =  ANSWER_PATTERN_BOXED, name: str = "qwen3_frequent_solver_v1"):
+        super().__init__(answer_pattern)
+        self.name = name
+    def load_data(self):
+        dataset = load_dataset(self.name)['train']
+        examples = []
+        
+        for row in dataset:
+            example = {
+                'question': row['problem'],
+                'answer': row['answer']
+            }
+            examples.append(example)
+        
+        # Shuffle and limit to 1000 examples like other handlers
+        random.shuffle(examples)
+        # examples = examples[:1000]
+        
+        questions = []
+        answers = []
+        for example in examples:
+
+            questions.append(example['question'])
+            answers.append(example['answer'])
+            
+        return questions, answers
+
+def get_dataset_handler(dataset_name: str,name: str = None) -> DatasetHandler:
+    if dataset_name == "math":
+        return MathDatasetHandler()
+    elif dataset_name == "gsm8k":
+        return Gsm8kDatasetHandler()
+    elif dataset_name == "amc":
+        return AmcDatasetHandler()
+    elif dataset_name == "minerva":
+        return MinervaDatasetHandler()
+    elif dataset_name == "olympiad":
+        return OlympiadDatasetHandler()
+    elif dataset_name == "aime2024":
+        return Aime2024DatasetHandler()
+    elif dataset_name == "aime2025":
+        return Aime2025DatasetHandler()
+    elif dataset_name == "mmlu_pro":
+        return MmluProDatasetHandler()
+    elif dataset_name == "bbeh":
+        return bbehDatasetHandler()
+    elif dataset_name == "super_gpqa":
+        return SuperGPQADatasetHandler()
+    elif dataset_name == "gpqa":
+        return GPQA_DatasetHandler()
+    elif dataset_name == "mydataset":
+        return Mydataset_DatasetHandler(name=name)
+    else:
+        raise ValueError(f"Dataset {dataset_name} not found")
+    
+
+if __name__ == "__main__":
+    print("mmlu_pro")
+    for dataset_name in ["gpqa"]:
+        print(f"Loading {dataset_name} dataset")
+        handler = get_dataset_handler(dataset_name)
+        questions, answers = handler.load_data()
+        print(questions[0])
+        print('-'*100)
+        print(answers[0])

eval_bbeh.py

@@ -0,0 +1,185 @@
+import datasets
+import json
+import re
+import random
+import argparse
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+
+def extract_last_boxed(text):
+    pattern = r'\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}'
+    matches = list(re.finditer(pattern, text))
+    if matches:
+        return matches[-1].group(1)
+    return None
+
+def extract_last_final_answer(text):
+    pattern1 = r'Final Answer:((?:[^<]|<[^<])*?)\n'
+    pattern2 = r'The answer is:((?:[^<]|<[^<])*?)\n'
+    matches1 = list(re.finditer(pattern1, text))
+    matches2 = list(re.finditer(pattern2, text))
+    if matches1:
+        return matches1[-1].group(1)
+    elif matches2:
+        return matches2[-1].group(1)
+    return None
+
+def extract_solution(solution_str):
+    if '<|im_start|>user' in solution_str:
+        model_output = re.sub(r'^.*?<\|im_start\|>assistant', '<|im_start|>assistant', solution_str, flags=re.DOTALL, count=1)
+    elif 'Assistant:' in solution_str:
+        model_output = solution_str.split('Assistant:')[-1].strip()
+    else:
+        model_output = solution_str
+
+    stop_words = ["</s>", "<|im_end|>", "<|endoftext|>"] 
+    for stop_word in stop_words:
+        if stop_word in model_output:
+            model_output = model_output.split(stop_word)[0].strip()
+    
+    extract_boxed_answer = extract_last_boxed(model_output)
+    if extract_boxed_answer:
+        return extract_boxed_answer
+    else:
+        return extract_last_final_answer(model_output)
+
+def strip_latex(response: str) -> str:
+  if response.startswith("$") and response.endswith("$"):
+    response = response[1:-1]
+  if "boxed{" in response and response.endswith("}"):
+    response = response[0:-1].split("boxed{")[1]
+  if "text{" in response and response.endswith("}"):
+    response = response[0:-1].split("text{")[1]
+  if "texttt{" in response and response.endswith("}"):
+    response = response[0:-1].split("texttt{")[1]
+  return response
+
+
+def extract_answer(sample: str) -> str:
+  if sample is None:
+     sample = ""
+  """Extracts the final answer from the sample."""
+  answer_prefixes = [
+      "The answer is:",
+      "The final answer is ",
+      "The final answer is: ",
+      "The answer is "
+  ]
+  answer = sample
+  for answer_prefix in answer_prefixes:
+    if answer_prefix in answer:
+      answer = answer.split(answer_prefix)[-1].strip()
+  if answer.endswith("."):
+    answer = answer[:-1]
+  return strip_latex(answer)
+
+
+def fuzzy_match(prediction: str, reference: str) -> bool:
+  """Fuzzy match function for BigBench Extra Hard."""
+  if prediction == reference:
+    return True
+
+  # (a) vs a
+  if len(prediction) == 3 and prediction[0] == "(" and prediction[-1] == ")":
+    return prediction[1] == reference
+  if len(reference) == 3 and reference[0] == "(" and reference[-1] == ")":
+    return reference[1] == prediction
+
+  # Numbers
+  try:
+    if float(prediction) == float(reference):
+      return True
+  except ValueError:
+    pass
+
+  # quote issues
+  if prediction.replace("'", "") == reference.replace("'", ""):
+    return True
+
+  # Bracket issues
+  if f"[{reference}]" == prediction or f"[{prediction}]" == reference:
+    return True
+
+  # Question mark issues
+  if prediction.endswith("?") and prediction[:-1] == reference:
+    return True
+
+  return False
+
+
+def preprocess_sample(sample: str) -> str:
+    if sample is None:
+        sample = ""
+    prediction = extract_answer(sample.strip()).lower()
+    prediction = prediction.replace(", ", ",").replace("**", "")
+    prediction = prediction.split("\n")[0]
+    prediction = prediction[0:-1] if prediction.endswith(".") else prediction
+    return prediction
+
+
+def preprocess_reference(reference: str) -> str:
+    reference = reference.strip().lower()
+    reference = reference.replace(", ", ",")
+    return reference
+
+
+def evaluate_correctness(sample: str, reference: str) -> bool:
+    prediction = preprocess_sample(sample)
+    reference = preprocess_reference(reference)
+    return fuzzy_match(prediction, reference)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", type=str, required=True, help="Path to the model directory")
+    parser.add_argument("--output_file", type=str, default="outputs.json", help="File to save results")
+    args = parser.parse_args()
+    
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+    llm = LLM(model=args.model_path, tensor_parallel_size=4,gpu_memory_utilization=0.85)
+    dataset = datasets.load_dataset('MrLight/bbeh-eval')
+    categories = sorted(list(set(dataset['train']['task'])))
+    print("Categories:", categories)
+    per_category_accuracy = {c: [0, 0] for c in categories}
+    success, fail = 0, 0
+    answers = []
+    
+    print('----------------- Start Answering -------------------')
+    
+    for category in categories:
+        category_entries = [entry for entry in dataset['train'] if entry['task'] == category]
+        prompts = []
+        for entry in category_entries:
+            query = entry['question'] + '\n'
+            messages = [{
+                "role": "user",
+                "content": query + '\nPlease reason step by step, and put your final answer option within \\boxed{}.'
+            }]
+            if tokenizer.chat_template:
+                prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+            else:
+                prompt = "user: " + query + '\nPlease reason step by step, and put your final answer option within \\boxed{}. Only put the letter in the box, e.g. \\boxed{A}. There is only one correct answer.'
+            prompts.append(prompt)
+        
+        sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=8192)
+        outputs = llm.generate(prompts, sampling_params)
+        
+        for entry, output in zip(category_entries, outputs):
+            answer = output.outputs[0].text
+            entry['solution'] = answer
+            answers.append(entry)
+            answer = extract_solution(answer)
+            if evaluate_correctness(answer, entry['answer']):
+                success += 1
+                per_category_accuracy[category][0] += 1
+            else:
+                fail += 1
+                per_category_accuracy[category][1] += 1
+            
+        print(f"{category}: {per_category_accuracy[category][0] / (per_category_accuracy[category][0] + per_category_accuracy[category][1]):.4f}")
+    
+    with open(args.output_file, 'w') as f:
+        json.dump(answers, f, indent=2)
+    with open('final_results.jsonl', 'a') as f:
+        json.dump({"dataset": "bbeh", "model": args.model_path, "accuracy": round(success / (success + fail)*100, 2)}, f, indent=2)
+    print("Overall Accuracy:", success / (success + fail))

eval_mmlupro.py

@@ -0,0 +1,145 @@
+import datasets
+import json
+import re
+import random
+import argparse
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+
+def extract_last_boxed(text):
+    pattern = r'\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}'
+    matches = list(re.finditer(pattern, text))
+    if matches:
+        return matches[-1].group(1)
+    return None
+
+def extract_last_final_answer(text):
+    pattern1 = r'Final Answer:((?:[^<]|<[^<])*?)\n'
+    pattern2 = r'The answer is:((?:[^<]|<[^<])*?)\n'
+    matches1 = list(re.finditer(pattern1, text))
+    matches2 = list(re.finditer(pattern2, text))
+    if matches1:
+        return matches1[-1].group(1)
+    elif matches2:
+        return matches2[-1].group(1)
+    return None
+
+def extract_solution(solution_str):
+    if '<|im_start|>user' in solution_str:
+        model_output = re.sub(r'^.*?<\|im_start\|>assistant', '<|im_start|>assistant', solution_str, flags=re.DOTALL, count=1)
+    elif 'Assistant:' in solution_str:
+        model_output = solution_str.split('Assistant:')[-1].strip()
+    else:
+        model_output = solution_str
+
+    stop_words = ["</s>", "<|im_end|>", "<|endoftext|>"]
+    for stop_word in stop_words:
+        if stop_word in model_output:
+            model_output = model_output.split(stop_word)[0].strip()
+    
+    extract_boxed_answer = extract_last_boxed(model_output)
+    if extract_boxed_answer:
+        return extract_boxed_answer
+    else:
+        return extract_last_final_answer(model_output)
+
+def form_options(options: list):
+    option_str = 'Options are:\n'
+    opts = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+    for opt, o in zip(options, opts):
+        option_str += f'({o}): {opt}\n'
+    return option_str
+
+def get_prediction(output):
+    solution = extract_solution(output)
+    if solution is None:
+        return random.choice(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
+    for option in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']:
+        if option in solution:
+            return option
+    return random.choice(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", type=str, required=True, help="Path to the model directory")
+    parser.add_argument("--output_file", type=str, default="outputs.json", help="File to save results")
+    args = parser.parse_args()
+    
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+    llm = LLM(model=args.model_path, tensor_parallel_size=4,gpu_memory_utilization=0.85)
+    dataset = datasets.load_dataset('TIGER-Lab/MMLU-Pro')
+    
+    categories = ['computer science', 'math', 'chemistry', 'engineering', 'law', 'biology',
+                  'health', 'physics', 'business', 'philosophy', 'economics', 'other',
+                  'psychology', 'history']
+    # For each category store [correct_count, incorrect_count]
+    per_category_accuracy = {c: [0, 0] for c in categories}
+    success, fail = 0, 0
+    answers = []
+    
+    print('----------------- Start Answering -------------------')
+    
+    for category in categories:
+        category_entries = [entry for entry in dataset['test'] if entry['category'] == category]
+        prompts = []
+        for entry in category_entries:
+            query = entry['question'] + '\n' + form_options(entry['options']) + '\n'
+            messages = [{
+                "role": "user",
+                "content": query + '\nPlease reason step by step, and put your final answer option within \\boxed{}. Only put the option letter in the box, e.g. \\boxed{A}. There is only one correct answer.'
+            }]
+            if tokenizer.chat_template:
+                prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+            else:
+                prompt = "user: " + query + '\nPlease reason step by step, and put your final answer option within \\boxed{}. Only put the letter in the box, e.g. \\boxed{A}. There is only one correct answer.'
+            prompts.append(prompt)
+        
+        sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=8192)
+        outputs = llm.generate(prompts, sampling_params)
+        
+        for entry, output in zip(category_entries, outputs):
+            answer = output.outputs[0].text
+            entry['solution'] = answer
+            answers.append(entry)
+            
+            prediction = get_prediction(answer)
+            if entry["answer"] == prediction:
+                success += 1
+                per_category_accuracy[category][0] += 1
+            else:
+                fail += 1
+                per_category_accuracy[category][1] += 1
+            
+        # Print category accuracy as soon as it's computed
+        total_cat = per_category_accuracy[category][0] + per_category_accuracy[category][1]
+        cat_accuracy = per_category_accuracy[category][0] / total_cat if total_cat > 0 else 0.0
+        print(f"{category}: {cat_accuracy:.4f}")
+    
+    # Save all the answers in a JSON file
+    with open(args.output_file, 'w') as f:
+        json.dump(answers, f, indent=2)
+    
+    # Calculate per-category report, micro average, and macro average
+    print("\n----- Accuracy Report -----")
+    category_accuracy_report = {}
+    for category in categories:
+        correct, incorrect = per_category_accuracy[category]
+        total = correct + incorrect
+        if total > 0:
+            accuracy = correct / total
+        else:
+            accuracy = 0.0
+        category_accuracy_report[category] = accuracy
+        print(f"{category}: {correct}/{total} -> {accuracy*100:.2f}% accuracy")
+        
+    total_predictions = success + fail
+    micro_avg = success / total_predictions if total_predictions > 0 else 0.0
+    print(f"\nMicro Average Accuracy: {micro_avg*100:.2f}%")
+    with open('final_results.jsonl', 'a') as f:
+        json.dump({"dataset": "mmlupro", "model": args.model_path, "accuracy": round(micro_avg*100, 2)}, f, indent=2)
+    valid_categories = [cat for cat in categories if (per_category_accuracy[cat][0] + per_category_accuracy[cat][1] > 0)]
+    if valid_categories:
+        macro_avg = sum(category_accuracy_report[cat] for cat in valid_categories) / len(valid_categories)
+    else:
+        macro_avg = 0.0
+    print(f"Macro Average Accuracy: {macro_avg*100:.2f}%")

eval_supergpqa.py

@@ -0,0 +1,116 @@
+import datasets
+import json
+import re
+import random
+import argparse
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+
+def extract_last_boxed(text):
+    pattern = r'\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}'
+    matches = list(re.finditer(pattern, text))
+    if matches:
+        return matches[-1].group(1)
+    return None
+
+def extract_last_final_answer(text):
+    pattern1 = r'Final Answer:((?:[^<]|<[^<])*?)\n'
+    pattern2 = r'The answer is:((?:[^<]|<[^<])*?)\n'
+    matches1 = list(re.finditer(pattern1, text))
+    matches2 = list(re.finditer(pattern2, text))
+    if matches1:
+        return matches1[-1].group(1)
+    elif matches2:
+        return matches2[-1].group(1)
+    return None
+
+def extract_solution(solution_str):
+    if '<|im_start|>user' in solution_str:
+        model_output = re.sub(r'^.*?<\|im_start\|>assistant', '<|im_start|>assistant', solution_str, flags=re.DOTALL, count=1)
+    elif 'Assistant:' in solution_str:
+        model_output = solution_str.split('Assistant:')[-1].strip()
+    else:
+        model_output = solution_str
+
+    stop_words = ["</s>", "<|im_end|>", "<|endoftext|>"] 
+    for stop_word in stop_words:
+        if stop_word in model_output:
+            model_output = model_output.split(stop_word)[0].strip()
+    
+    extract_boxed_answer = extract_last_boxed(model_output)
+    if extract_boxed_answer:
+        return extract_boxed_answer
+    else:
+        return extract_last_final_answer(model_output)
+
+def form_options(options: list):
+    option_str = 'Options are:\n'
+    opts = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+    for opt, o in zip(options, opts):
+        option_str += f'({o}): {opt}\n'
+    return option_str
+
+def get_prediction(output):
+    solution = extract_solution(output)
+    if solution is None:
+        return random.choice(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
+    for option in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']:
+        if option in solution:
+            return option
+    return random.choice(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", type=str, required=True, help="Path to the model directory")
+    parser.add_argument("--output_file", type=str, default="outputs.json", help="File to save results")
+    args = parser.parse_args()
+    
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+    llm = LLM(model=args.model_path, tensor_parallel_size=4,gpu_memory_utilization=0.85)
+    print('start loading dataset')
+    dataset = datasets.load_dataset('m-a-p/SuperGPQA')
+    categories = ['Engineering', 'Medicine', 'Science', 'Philosophy', 'Military Science', 'Economics', 'Management', 'Sociology', 'Literature and Arts', 'History', 'Agronomy', 'Law', 'Education']
+    per_category_accuracy = {c: [0, 0] for c in categories}
+    success, fail = 0, 0
+    answers = []
+    
+    print('----------------- Start Answering -------------------')
+    
+    for category in categories:
+        category_entries = [entry for entry in dataset['train'] if entry['discipline'] == category]
+        prompts = []
+        for entry in category_entries:
+            query = entry['question'] + '\n' + form_options(entry['options']) + '\n'
+            messages = [{
+                "role": "user",
+                "content": query + '\nPlease reason step by step, and put your final answer option within \\boxed{}. Only put the letter in the box, e.g. \\boxed{A}. There is only one correct answer.'
+            }]
+            if tokenizer.chat_template:
+                prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+            else:
+                prompt = "user: " + query + '\nPlease reason step by step, and put your final answer option within \\boxed{}. Only put the letter in the box, e.g. \\boxed{A}. There is only one correct answer.'
+            prompts.append(prompt)
+        
+        sampling_params = SamplingParams(temperature=0, top_p=1, max_tokens=8192)
+        outputs = llm.generate(prompts, sampling_params)
+        
+        for entry, output in zip(category_entries, outputs):
+            answer = output.outputs[0].text
+            entry['solution'] = answer
+            answers.append(entry)
+            
+            prediction = get_prediction(answer)
+            if entry["answer_letter"] == prediction:
+                success += 1
+                per_category_accuracy[category][0] += 1
+            else:
+                fail += 1
+                per_category_accuracy[category][1] += 1
+            
+        print(f"{category}: {per_category_accuracy[category][0] / (per_category_accuracy[category][0] + per_category_accuracy[category][1]):.4f}")
+    
+    with open(args.output_file, 'w') as f:
+        json.dump(answers, f, indent=2)
+    with open('final_results.jsonl', 'a') as f:
+        json.dump({"dataset": "supergpqa", "model": args.model_path, "accuracy": round(success / (success + fail)*100, 2)}, f, indent=2)
+    print("Overall Accuracy:", success / (success + fail))

evaluate.bash

@@ -0,0 +1,79 @@
+#!/bin/bash
+export VLLM_DISABLE_COMPILE_CACHE=1
+model_name=$1
+
+MODEL_NAMES=(
+  $model_name
+)
+
+TASKS=(
+  "math"
+  "gsm8k" 
+  "amc"
+  "minerva"
+  "olympiad"
+  "aime2024"
+  "aime2025"
+)
+
+GPU_QUEUE=($(nvidia-smi --query-gpu=index --format=csv,noheader))
+echo "Available GPUs: ${GPU_QUEUE[@]}"
+
+declare -A pids
+
+start_job() {
+  local gpu_id="$1"
+  local model="$2"
+  local task="$3"
+
+  echo "==> [$(date '+%Y-%m-%d %H:%M:%S')] Start task [${task}] with model [${model}] on GPU [${gpu_id}] ..."
+
+  CUDA_VISIBLE_DEVICES="${gpu_id}" \
+  python evaluation/generate.py --model "${model}" --dataset "${task}" &
+
+  pids["${gpu_id}"]=$!
+}
+
+for MODEL_NAME in "${MODEL_NAMES[@]}"; do
+    echo "==> Processing model: ${MODEL_NAME}"
+    TASK_INDEX=0
+    NUM_TASKS=${#TASKS[@]}
+
+    while :; do
+        while [ ${#GPU_QUEUE[@]} -gt 0 ] && [ ${TASK_INDEX} -lt ${NUM_TASKS} ]; do
+            gpu_id="${GPU_QUEUE[0]}"
+            GPU_QUEUE=("${GPU_QUEUE[@]:1}")
+
+            task="${TASKS[${TASK_INDEX}]}"
+            ((TASK_INDEX++))
+
+            start_job "$gpu_id" "$MODEL_NAME" "$task"
+        done
+
+        if [ ${TASK_INDEX} -ge ${NUM_TASKS} ] && [ ${#pids[@]} -eq 0 ]; then
+            break
+        fi
+
+        for gpu_id in "${!pids[@]}"; do
+            pid="${pids[$gpu_id]}"
+            if ! kill -0 "$pid" 2>/dev/null; then
+                echo "==> [$(date '+%Y-%m-%d %H:%M:%S')] GPU [${gpu_id}] job finished with PID [${pid}]."
+                unset pids["$gpu_id"]
+                GPU_QUEUE+=("$gpu_id")
+            fi
+        done
+
+        sleep 1
+    done
+done
+
+python evaluation/results_recheck.py --model_name $model_name &
+
+python evaluation/eval_supergpqa.py --model_path $model_name
+python evaluation/eval_bbeh.py --model_path $model_name
+python evaluation/eval_mmlupro.py --model_path $model_name
+
+
+python evaluation/test.py --model_name $model_name
+
+echo "==> All tasks have finished!"

generate.py

@@ -0,0 +1,51 @@
+import vllm
+import argparse
+import  evaluation.datasets_loader as datasets_loader
+from transformers import AutoTokenizer
+import json
+import os
+
+STORAGE_PATH = os.getenv("STORAGE_PATH")
+
+def main(args):
+    print("STORAGE_PATH")
+    print(STORAGE_PATH)
+    with open('tokens.json','r') as f: 
+        tokens = json.load(f)
+    print(args.model, args.dataset)
+    tokenizer = AutoTokenizer.from_pretrained(args.model)
+    model = vllm.LLM(
+        model=args.model,
+        tokenizer=args.model,
+        gpu_memory_utilization=0.85
+    )
+    sample_params = vllm.SamplingParams(
+        max_tokens=4096,
+        temperature=0.0,
+        stop_token_ids=[tokenizer.eos_token_id],
+    )
+    handler = datasets_loader.get_dataset_handler(args.dataset,args.name)
+    questions, answers = handler.load_data()
+    chats=[[{"role": "system", "content": "Please reason step by step, and put your final answer within \\boxed{}."},{"role": "user", "content": question}] for question in questions]
+    if tokenizer.chat_template:
+        prompts = [tokenizer.apply_chat_template(chat, tokenize=False,add_generation_prompt=True, add_special_tokens=True, enable_thinking=False) for chat in chats]
+    else:
+        prompts = ["system: " + chat[0]["content"] + '\n' + "user: " + chat[1]["content"] + '\nPlease reason step by step, and put your final answer within \\boxed{}.' for chat in chats]
+    responses = model.generate(prompts, sampling_params=sample_params,use_tqdm=True)
+    responses = [response.outputs[0].text for response in responses]
+    scores,average_score = handler.get_score(responses, answers)
+    results = [{"question": question, "answer": answer, "response": response, "score": score} for question, answer, response, score in zip(questions, answers, responses, scores)]
+    print(f"Average score: {average_score}")
+    results.append({"average_score": average_score})
+    os.makedirs(f"{STORAGE_PATH}/evaluation/{args.model.replace('/', '_')}", exist_ok=True)
+    with open(f"{STORAGE_PATH}/evaluation/{args.model.replace('/', '_')}/results_{args.dataset}.json", "w") as f:
+        json.dump(results, f, indent=4)
+
+    
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=str, default="Qwen/Qwen3-4B")
+    parser.add_argument("--dataset", type=str, default="math")
+    parser.add_argument("--name", type=str, default=None)
+    args = parser.parse_args()
+    main(args)

results_recheck.py

@@ -0,0 +1,74 @@
+import json
+from mathruler.grader import extract_boxed_content, grade_answer
+import openai
+import requests
+from tqdm import tqdm
+import random
+import argparse
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model_name", type=str, default="Qwen/Qwen2.5-7B-Instruct")
+args = parser.parse_args()
+
+STORAGE_PATH = os.getenv("STORAGE_PATH")
+api_urls = []
+api_keys=[]
+
+
+
+def process_example(answer, response):
+    try:
+        example = {
+            "model": "gpt-4o",
+            "messages": [
+                {"role": "system", "content": "You are a math answer checker."},
+                {"role": "user", "content": f"Hi, there is a answer: {answer}\n\n, and the ground truth answer is: {response}\n\n, please check whether the answer is correct or not, and return the **only** Yes or No."}
+            ],
+            "temperature": 0.1
+        }
+        api_index = random.randint(0, len(api_urls)-1)
+        api_url = api_urls[api_index]
+        api_key = api_keys[api_index]
+        response = requests.post(api_url, headers={"api-key": api_key,"Content-Type": "application/json"}, json=example, timeout=20)
+        return response.json()['choices'][0]['message']['content']
+    except Exception as e:
+        print(e)
+        return "No"
+new_results = []
+for model_name in [args.model_name]:
+    for dataset in [
+    "math",
+    "gsm8k", 
+    "amc",
+    "minerva",
+    "olympiad",
+    "aime2024",
+    "aime2025",
+    ]:
+        with open(f'{STORAGE_PATH}/evaluation/{model_name.replace("/","_")}/results_{dataset}.json', 'r') as f:
+            results = json.load(f)
+
+        for i in tqdm(range(len(results)-1)):
+                if results[i]['score'] < 0.5:
+                    gpt_check = process_example(results[i]['answer'],results[i]['response'])
+                    if "yes" in gpt_check.lower():
+                        results[i]['score']=1
+        new_results.append({
+            'model': model_name,
+            'dataset': dataset,
+            'score': round(sum([result['score'] for result in results[:-1]])/len(results[:-1])*100, 2)
+        })
+        print(new_results)
+        with open(f'final_results.jsonl', 'a') as f:
+            json.dump({
+                'model': model_name,
+                'dataset': dataset,
+                'score': round(sum([result['score'] for result in results[:-1]])/len(results[:-1])*100, 2)
+            }, f)
+            f.write('\n')
+
+
+
+
+