verbose mode

tasks.py

@@ -126,9 +126,10 @@ def multichoice(responses: Any, references: list[str]):
     else:
         responses = decode_choice(responses)
 
-    return [
-        int(response == reference) for reference, response in zip(references, responses)
-    ]
+    # return [
+    #     int(response == reference) for reference, response in zip(references, responses)
+    # ]
+    return responses, references
 
 
 class Metrics:
@@ -136,12 +137,18 @@ class Metrics:
     mmlu = multichoice
 
     def gsm8k(responses: list[str], answers: list[str | int]):
-        scores = []
-        for response, answer in zip(responses, answers):
-            pred = extract_numeric(response)
-            gold = extract_numeric(answer) if isinstance(answer, str) else str(answer)
-            scores.append(1.0 * (pred == gold))
-        return scores
+        # scores = []
+        # for response, answer in zip(responses, answers):
+        #     pred = extract_numeric(response)
+        #     gold = extract_numeric(answer) if isinstance(answer, str) else str(answer)
+        #     scores.append(1.0 * (pred == gold))
+        responses = [extract_numeric(response) for response in responses]
+        answers = [
+            extract_numeric(answer) if isinstance(answer, str) else str(answer)
+            for answer in answers
+        ]
+
+        return responses, answers
 
     def MATH(responses: list[str], answers: list[str]):
         scores = []
@@ -445,7 +452,7 @@ class MMLU:
                         label_column=cls.label_column,
                         prompt=partial(cls.prompt_mmlu, chat=chat),
                         few_shot=0 if chat else 5,
-                        few_shot_from="validation"
+                        few_shot_from="validation",
                     )
                     for subcategories in finer_categories[subject]
                 ]

tlem.py

@@ -6,6 +6,7 @@ except Exception as e:
     import logging
 
 from typing import Any, Optional, Protocol, Iterable, Callable
+from numpy.lib import extract
 from tqdm.auto import tqdm
 from evaluate.evaluation_suite import EvaluationSuite
 import evaluate
@@ -70,10 +71,26 @@ class ReasoningMetric(evaluate.Metric):
             reference_urls=["http://path.to.reference.url/new_module"],
         )
 
-    def _compute(self, responses, references):
-        scores = getattr(Metrics, self.config_name)(responses, references)
-        results = {"Accuracy": np.nanmean(scores)}
+    def _compute(self, responses, references, verbose=False):
+        extract_responses, extract_references = getattr(Metrics, self.config_name)(
+            responses, references
+        )
+        df = pd.DataFrame(
+            {
+                "responses": responses,
+                "references": references,
+            }
+        )
+        df["extract_responses"] = extract_responses
+        df["extract_references"] = extract_references
+        results = {
+            "Accuracy": (df["extract_references"] == df["extract_responses"])
+            .astype(int)
+            .mean(),
+        }
         logging.info(results)
+        if verbose:
+            results["df"] = df
         return results