File size: 2,314 Bytes
df41178 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/usr/bin/env python3
import json
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "LocalAI改良/tools/ops/humaneval"))
try:
import he_eval # リポジトリの外（LocalAI改良/tools/ops/humaneval）にある。無い所では飛ばす
except ImportError:
he_eval = None
@unittest.skipIf(he_eval is None, "he_eval がこの機械に無い")
class HumanEvalTests(unittest.TestCase):
def _problem(self, task_id="HumanEval/0"):
return {
"task_id": task_id,
"prompt": "def add(a, b):\n \"\"\"add\"\"\"\n",
"entry_point": "add",
"test": "def check(candidate):\n assert candidate(2, 3) == 5\n",
}
def test_missing_generation_stays_in_denominator(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
data = root / "data.jsonl"
gen = root / "gen.jsonl"
data.write_text(json.dumps(self._problem()) + "\n" +
json.dumps(self._problem("HumanEval/1")) + "\n",
encoding="utf-8")
gen.write_text(json.dumps({
"task_id": "HumanEval/0",
"raw": "```python\ndef add(a, b):\n return a+b\n```",
}) + "\n", encoding="utf-8")
summary, rows = he_eval.evaluate(
data, gen, timeout=3, memory_mb=256, output_mb=1,
sandbox=False)
self.assertEqual(summary["data_n"], 2)
self.assertEqual(summary["pass"], 1)
self.assertEqual(summary["missing_generation_n"], 1)
self.assertEqual(summary["accuracy_over_data"], 0.5)
self.assertEqual(rows[1]["why"], "missing_generation")
def test_timeout_is_reported(self):
problem = self._problem()
generation = {"task_id": problem["task_id"],
"raw": "```python\ndef add(a,b):\n while True: pass\n```"}
result = he_eval.run_case(problem, generation, timeout=0.2,
memory_mb=256, output_mb=1, sandbox=False)
self.assertFalse(result["pass"])
self.assertEqual(result["why"], "timeout")
if __name__ == "__main__":
unittest.main()
|