Chagne formatting to use boxed

benchmarking/llm_providers/medrax_provider.py

@@ -70,7 +70,7 @@ class MedRAXProvider(LLMProvider):
             agent, tools_dict = initialize_agent(
                 prompt_file="medrax/docs/system_prompts.txt",
                 tools_to_use=selected_tools,
-                model_dir="model-weights",
+                model_dir="/model-weights",
                 temp_dir="temp",  # Change this to the path of the temporary directory
                 device="cuda:0",
                 model=self.model_name,  # Change this to the model you want to use, e.g. gpt-4.1-2025-04-14, gemini-2.5-pro

benchmarking/runner.py

@@ -259,9 +259,15 @@ class BenchmarkRunner:
         Returns:
             str: The extracted answer
         """
-        # First, look for the '<|A|>' format
-        final_answer_pattern = r'\s*<\|([A-F])\|>'
-        match = re.search(final_answer_pattern, response_text)
+        # Look for the '\boxed{A}' format
+        boxed_pattern = r'\\boxed\{([A-Fa-f])\}'
+        match = re.search(boxed_pattern, response_text)
+        if match:
+            return match.group(1).upper()
+        
+        # Fallback: look for the '<|A|>' format (legacy code, will remove later on)
+        legacy_pattern = r'\s*<\|([A-F])\|>'
+        match = re.search(legacy_pattern, response_text)
         if match:
             return match.group(1).upper()
         

medrax/docs/system_prompts.txt

@@ -22,5 +22,5 @@ Solve using your own vision and reasoning and use tools (if available) to comple
 You can make multiple tool calls in parallel or in sequence as needed for comprehensive answers.
 Think critically about and criticize the tool outputs.
 If you need to look up some information before asking a follow up question, you are allowed to do that.
-When encountering a multiple-choice question, your final response should end with "Final answer: <|A|>" from list of possible choices A, B, C, D, E, F.
+When encountering a multiple-choice question, your final response should end with "Final answer: \boxed{A}" from list of possible choices A, B, C, D, E, F.
 It is extremely important that you strictly answer in the format mentioned above.