.

code_eval.py

@@ -180,12 +180,12 @@ class CodeEval(evaluate.Metric):
             results = {}
             for tid, pred, ref in zip(task_ids, predictions, references):
                 results[tid] = []
-                for candidate in pred:
-                    result = Result(task_id=tid, completion_id=len(results))
+                for cid, candidate in enumerate(pred):
+                    result = Result(task_id=tid, completion_id=cid)
                     for test_case in ref:
                         assert isinstance(test_case, str)
                         test_program = candidate + "\n" + test_case
-                        args = (test_program, timeout, tid)
+                        args = (test_program, timeout, tid, cid)
                         future = executor.submit(check_correctness, *args)
                         result.add(future)
                     results[tid].append(result)
@@ -278,7 +278,7 @@ class Result(BaseModel):
                             future.cancel()
 
         if all(r is not None for r in self.result):
-            self.passed = all(r["passed"] for r in self.result)
+            self.passed = all(r == "passed" for r in self.result)
 
     def done(self):
         return self.passed is not None