Fix

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "python.formatting.provider": "none"
+}

README.md

@@ -42,7 +42,7 @@ for row in ds:
     question_infos.append(row['question_info'])
     # Replace this with however you generate and postprocess predictions.
     predictions.append(model.generate(row['signature_with_docstring']))
-metric = evaluate.load("bc_eval")
+metric = evaluate.load("gabeorlanski/bc_eval")
 metrics, results = metric.compute(
     predictions=predictions, languages=languages, question_dicts=question_infos, k=[1]
 )
@@ -94,7 +94,7 @@ import os
 os.environ["HF_ALLOW_CODE_EVAL"] = "1"
 ds = load_dataset("gabeorlanski/bc-humaneval", split="test")
 example = ds[0]
-metric = evaluate.load("bc_eval")
+metric = evaluate.load("gabeorlanski/bc_eval")
 languages = ["Python"]
 question_infos = [example["question_info"]]
 predictions = [["""def has_close_elements(numbers: List[float], threshold: float) -> bool:
@@ -116,7 +116,35 @@ metrics, results = metric.compute(
 ```
 `results` is:
 ```
-[{"qid": 0, "idx": "0", "file_path": ".../tmpqt_p3dwn/0", "results": [{"return_code": 0, "runtime": 0.076369, "stdout": "TEST-0...PASSED\r\nTEST-1...PASSED\r\nTEST-2...PASSED\r\nTEST-3...PASSED\r\nTEST-4...PASSED\r\nTEST-5...PASSED\r\nTEST-6...PASSED\r\n", "stderr": "", "timed_out": false}], "failed": false, "timed_out": false, "test_cases": {"0": "PASSED", "1": "PASSED", "2": "PASSED", "3": "PASSED", "4": "PASSED", "5": "PASSED", "6": "PASSED"}, "outcome": "PASSED"}]
+
+[
+    {
+        "qid": 0,
+        "idx": "0",
+        "file_path": ".../tmpqt_p3dwn/0",
+        "results": [
+            {
+                "return_code": 0,
+                "runtime": 0.076369,
+                "stdout": "TEST-0...PASSED\r\nTEST-1...PASSED\r\nTEST-2...PASSED\r\nTEST-3...PASSED\r\nTEST-4...PASSED\r\nTEST-5...PASSED\r\nTEST-6...PASSED\r\n",
+                "stderr": "",
+                "timed_out": false,
+            }
+        ],
+        "failed": false,
+        "timed_out": false,
+        "test_cases": {
+            "0": "PASSED",
+            "1": "PASSED",
+            "2": "PASSED",
+            "3": "PASSED",
+            "4": "PASSED",
+            "5": "PASSED",
+            "6": "PASSED",
+        },
+        "outcome": "PASSED",
+    }
+]
 ```
 
 
@@ -131,7 +159,7 @@ ds = load_dataset(
         "gabeorlanski/bc-humaneval", "Python", split="test"
     )
 example = ds[0]
-metric = evaluate.load("bc_eval")
+metric = evaluate.load("gabeorlanski/bc_eval")
 languages = ["Python"]
 question_infos = [example["question_info"]]
 predictions = [["""def has_close_elements(numbers: List[float], threshold: float) -> bool:
@@ -170,7 +198,7 @@ ds = load_dataset(
         "gabeorlanski/bc-humaneval", "Python", split="test"
     )
 example = ds[0]
-metric = evaluate.load("bc_eval")
+metric = evaluate.load("gabeorlanski/bc_eval")
 languages = ["Python"]
 question_infos = [example["question_info"]]
 predictions = [["""import time
@@ -203,7 +231,7 @@ ds = load_dataset(
         "gabeorlanski/bc-humaneval", "Python", split="test"
     )
 example = ds[0]
-metric = evaluate.load("bc_eval")
+metric = evaluate.load("gabeorlanski/bc_eval")
 languages = ["Python"]
 question_infos = [example["question_info"]]
 predictions = [["""import time
@@ -223,8 +251,7 @@ metrics, results = metric.compute(
 {"Python/pass@1": 0.0, "Python/mean_pct_pass": 0.0}
 ```
 `results` is:
-```
-[{"qid": 0, "idx": "0", "file_path": "/tmpjdn51aaa/0", "results": [{"return_code": 0, "runtime": 0.102855, "stdout": "TEST-0...ValueError\r\nTEST-1...ValueError\r\nTEST-2...ValueError\r\nTEST-3...ValueError\r\nTEST-4...ValueError\r\nTEST-5...ValueError\r\nTEST-6...ValueError\r\n", "stderr": "", "timed_out": false}], "failed": false, "timed_out": false, "test_cases": {"0": "ValueError", "1": "ValueError", "2": "ValueError", "3": "ValueError", "4": "ValueError", "5": "ValueError", "6": "ValueError"}, "outcome": "HAD_ERROR"}, 
+```[{"qid": 0, "idx": "0", "file_path": "/tmpjdn51aaa/0", "results": [{"return_code": 0, "runtime": 0.102855, "stdout": "TEST-0...ValueError\r\nTEST-1...ValueError\r\nTEST-2...ValueError\r\nTEST-3...ValueError\r\nTEST-4...ValueError\r\nTEST-5...ValueError\r\nTEST-6...ValueError\r\n", "stderr": "", "timed_out": false}], "failed": false, "timed_out": false, "test_cases": {"0": "ValueError", "1": "ValueError", "2": "ValueError", "3": "ValueError", "4": "ValueError", "5": "ValueError", "6": "ValueError"}, "outcome": "HAD_ERROR"}, 
 {"qid": 0, "idx": "1", "file_path": "/tmpjdn51aaa/1", "results": [{"return_code": 0, "runtime": 0.094347, "stdout": "TEST-0...NameError\r\nTEST-1...NameError\r\nTEST-2...NameError\r\nTEST-3...NameError\r\nTEST-4...NameError\r\nTEST-5...NameError\r\nTEST-6...NameError\r\n", "stderr": "", "timed_out": false}], "failed": false, "timed_out": false, "test_cases": {"0": "NameError", "1": "NameError", "2": "NameError", "3": "NameError", "4": "NameError", "5": "NameError", "6": "NameError"}, "outcome": "HAD_ERROR"}]
 ```
 

bc_eval.py

@@ -82,9 +82,7 @@ _QUESTION_INFO_KEYS = {
 }
 
 
-def make_file_and_command(
-    qid, idx, pred, question, working_dir, timeout_override=None
-):
+def make_file_and_command(qid, idx, pred, question, working_dir, timeout_override=None):
     file_name = f"pred.{question['extension']}"
     pred_dir = working_dir.joinpath(idx)
     pred_dir.mkdir(parents=True)
@@ -101,9 +99,7 @@ def make_file_and_command(
         commands.append(
             {
                 "timeout": t if timeout_override is None else timeout_override,
-                "command": [
-                    c if c != "__FILENAME__" else file_name for c in cmd
-                ],
+                "command": [c if c != "__FILENAME__" else file_name for c in cmd],
             }
         )
 
@@ -124,7 +120,7 @@ def _write_preds(
         zip(preds, languages, question_dicts), desc="Setup", total=len(preds)
     ):
         qid = len(question_id_to_dict)
-        q_dict['language'] = l
+        q_dict["language"] = l
         question_id_to_dict[qid] = q_dict
         for p in pred_list:
             commands.append(
@@ -141,9 +137,7 @@ def _write_preds(
     return question_id_to_dict, commands
 
 
-@evaluate.utils.file_utils.add_start_docstrings(
-    _DESCRIPTION, _KWARGS_DESCRIPTION
-)
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class BabelCodeEval(evaluate.Metric):
     def _info(self):
         list_keys = ["timeouts", "commands", "test_case_ids"]
@@ -153,12 +147,8 @@ class BabelCodeEval(evaluate.Metric):
             if k not in list_keys
         }
         question_info_type["test_case_ids"] = datasets.Value("string")
-        question_info_type["commands"] = datasets.Sequence(
-            datasets.Value("string")
-        )
-        question_info_type["timeouts"] = datasets.Sequence(
-            datasets.Value("int32")
-        )
+        question_info_type["commands"] = datasets.Sequence(datasets.Value("string"))
+        question_info_type["timeouts"] = datasets.Sequence(datasets.Value("int32"))
 
         return evaluate.MetricInfo(
             # This is the description that will appear on the metrics page.
@@ -170,7 +160,7 @@ class BabelCodeEval(evaluate.Metric):
                 {
                     "predictions": datasets.Sequence(datasets.Value("string")),
                     "languages": datasets.Value("string"),
-                    "question_dicts": question_info_type
+                    "question_dicts": question_info_type,
                 }
             ),
             homepage="https://github.com/google-research/babelcode",
@@ -211,18 +201,17 @@ class BabelCodeEval(evaluate.Metric):
                 garbage_collection_freq=500,
             )
 
-        
-        all_results, q_passes, q_pct = _eval_predictions(
-            results, question_map
-        )
-        
-    
+        all_results, q_passes, q_pct = _eval_predictions(results, question_map)
+
         assert len(q_passes) == len(q_pct)
         metrics = {}
         for lang in q_passes:
-            metrics.update(_calculate_metrics(lang, q_passes[lang], q_pct[lang],k_vals=k))
+            metrics.update(
+                _calculate_metrics(lang, q_passes[lang], q_pct[lang], k_vals=k)
+            )
         return metrics, all_results
 
+
 def _eval_single_pred(result, test_ids, num_expected_commands):
     test_case_results = {k: "MISSING" for k in test_ids}
     if len(result["results"]) != num_expected_commands:
@@ -279,39 +268,35 @@ def _eval_predictions(pred_results, question_map):
         p["results"] = [dataclasses.asdict(r) for r in p["results"]]
         p["test_cases"] = test_case_results
         p["outcome"] = outcome
-        
-        lang = question['language']
-        question_results[lang][p["qid"]].append(
-            num_passed == len(test_case_results)
-        )
-        question_pct_pass[lang][p["qid"]].append(
-            num_passed / len(test_case_results)
-        )
+
+        lang = question["language"]
+        question_results[lang][p["qid"]].append(num_passed == len(test_case_results))
+        question_pct_pass[lang][p["qid"]].append(num_passed / len(test_case_results))
 
         out.append(p)
 
     return out, question_results, question_pct_pass
 
 
-def _calculate_metrics(lang,q_passed, q_pcts, k_vals):
+def _calculate_metrics(lang, q_passed, q_pcts, k_vals):
     assert len(q_passed) == len(q_pcts)
-    
+
     num_samples = np.zeros(len(q_passed))
     num_correct = np.zeros(len(q_passed))
     pcts_passed = np.zeros(len(q_passed))
-    for i, (k,v) in enumerate(q_passed.items()):
+    for i, (k, v) in enumerate(q_passed.items()):
         num_samples[i] = len(v)
         num_correct[i] = sum(v)
         pcts_passed[i] = np.mean(q_pcts[k])
-        
-        
-    out = {f'{lang}/pass@{k}': estimate_pass_at_k(num_samples, num_correct, k).mean() for k in k_vals}
-    out[f'{lang}/mean_pct_pass'] = np.mean(pcts_passed)
-    
-    
+
+    out = {
+        f"{lang}/pass@{k}": estimate_pass_at_k(num_samples, num_correct, k).mean()
+        for k in k_vals
+    }
+    out[f"{lang}/mean_pct_pass"] = np.mean(pcts_passed)
+
     return out
-        
-    
+
 
 def estimate_pass_at_k(num_samples, num_correct, k):
     """Estimates pass@k of each problem and returns them in an array."""
@@ -329,8 +314,5 @@ def estimate_pass_at_k(num_samples, num_correct, k):
         num_samples_it = iter(num_samples)
 
     return np.array(
-        [
-            estimator(int(n), int(c), k)
-            for n, c in zip(num_samples_it, num_correct)
-        ]
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
     )