Update bc_eval.py

bc_eval.py

@@ -3,7 +3,7 @@ import itertools
 import os
 import re
 import tempfile
-from collections import defaultdict
+from collections import defaultdict, Counter
 from pathlib import Path
 
 import datasets
@@ -204,13 +204,13 @@ class BabelCodeEval(evaluate.Metric):
                 garbage_collection_freq=gc_freq,
             )
 
-        all_results, q_passes, q_pct = _eval_predictions(results, question_map)
+        all_results, q_passes, q_pct, o_count = _eval_predictions(results, question_map)
 
         assert len(q_passes) == len(q_pct)
         metrics = {}
         for lang in q_passes:
             metrics.update(
-                _calculate_metrics(lang, q_passes[lang], q_pct[lang], k_vals=k)
+                _calculate_metrics(lang, q_passes[lang], q_pct[lang], o_count[lang], k_vals=k)
             )
         return metrics, all_results
 
@@ -258,7 +258,7 @@ def _eval_predictions(pred_results, question_map):
     out = []
     question_results = defaultdict(lambda: defaultdict(list))
     question_pct_pass = defaultdict(lambda: defaultdict(list))
-
+    outcome_counts = defaultdict(Counter)
     for p in pred_results:
         question = question_map[p["qid"]]
         test_cases = question["test_case_ids"]
@@ -275,13 +275,13 @@ def _eval_predictions(pred_results, question_map):
         lang = question["language"]
         question_results[lang][p["qid"]].append(num_passed == len(test_case_results))
         question_pct_pass[lang][p["qid"]].append(num_passed / len(test_case_results))
-
+        outcome_counts[lang][outcome] += 1
         out.append(p)
 
-    return out, question_results, question_pct_pass
+    return out, question_results, question_pct_pass, outcome_counts
 
 
-def _calculate_metrics(lang, q_passed, q_pcts, k_vals):
+def _calculate_metrics(lang, q_passed, q_pcts, o_count, k_vals):
     assert len(q_passed) == len(q_pcts)
 
     num_samples = np.zeros(len(q_passed))
@@ -298,7 +298,9 @@ def _calculate_metrics(lang, q_passed, q_pcts, k_vals):
     }
     out[f"{lang}/mean_pct_pass"] = np.mean(pcts_passed)
     out[f"{lang}/median_pct_pass"] = np.median(pcts_passed)
-    
+
+    for outcome, val in o_count.items():
+        out[f"{lang}/pct_{outcome}"] = val/len(q_passed)
 
     return out