""" Sample test cases for the /evaluate/ endpoint. Sends requests to the deployed evaluator at: https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space/evaluate/ Each test case constructs a payload with: - task_id, res_id, solution, test (a unittest.TestCase), entry_point, code_prompt and validates the response status ("pass", "fail", or "timeout"). """ import httpx import json import sys BASE_URL = "https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space" EVALUATE_URL = f"{BASE_URL}/evaluate/" TIMEOUT = 300 # seconds def make_sample(task_id, res_id, code_prompt, solution, test, entry_point): return { "task_id": task_id, "res_id": res_id, "code_prompt": code_prompt, "solution": solution, "test": test, "entry_point": entry_point, } # --------------------------------------------------------------------------- # Test case definitions # --------------------------------------------------------------------------- # 1. Passing: correct add function CASE_PASS_ADD = make_sample( task_id="test/add_pass", res_id=0, code_prompt="def add(a, b):", solution="def add(a, b):\n return a + b\n", test=""" import unittest class TestCases(unittest.TestCase): def test_add_positive(self): self.assertEqual(add(2, 3), 5) def test_add_negative(self): self.assertEqual(add(-1, -1), -2) def test_add_zero(self): self.assertEqual(add(0, 0), 0) """, entry_point="add", ) # 2. Failing: wrong multiply implementation CASE_FAIL_MULTIPLY = make_sample( task_id="test/multiply_fail", res_id=1, code_prompt="def multiply(a, b):", solution="def multiply(a, b):\n return a + b\n", # bug: adds instead of multiplies test=""" import unittest class TestCases(unittest.TestCase): def test_multiply_basic(self): self.assertEqual(multiply(3, 4), 12) def test_multiply_zero(self): self.assertEqual(multiply(0, 5), 0) """, entry_point="multiply", ) # 3. Passing: fibonacci function CASE_PASS_FIB = make_sample( task_id="test/fibonacci_pass", res_id=2, code_prompt="def fibonacci(n):", solution="""def fibonacci(n): if n <= 0: return 0 elif n == 1: return 1 a, b = 0, 1 for _ in range(2, n + 1): a, b = b, a + b return b """, test=""" import unittest class TestCases(unittest.TestCase): def test_fib_zero(self): self.assertEqual(fibonacci(0), 0) def test_fib_one(self): self.assertEqual(fibonacci(1), 1) def test_fib_ten(self): self.assertEqual(fibonacci(10), 55) def test_fib_twenty(self): self.assertEqual(fibonacci(20), 6765) """, entry_point="fibonacci", ) # 4. Failing: function raises an exception CASE_FAIL_EXCEPTION = make_sample( task_id="test/divide_exception", res_id=3, code_prompt="def safe_divide(a, b):", solution="""def safe_divide(a, b): return a / b """, test=""" import unittest class TestCases(unittest.TestCase): def test_divide_normal(self): self.assertEqual(safe_divide(10, 2), 5.0) def test_divide_by_zero(self): self.assertEqual(safe_divide(1, 0), 0) """, entry_point="safe_divide", ) # 5. Passing: string reversal CASE_PASS_REVERSE = make_sample( task_id="test/reverse_pass", res_id=4, code_prompt="def reverse_string(s):", solution="def reverse_string(s):\n return s[::-1]\n", test=""" import unittest class TestCases(unittest.TestCase): def test_reverse_word(self): self.assertEqual(reverse_string("hello"), "olleh") def test_reverse_empty(self): self.assertEqual(reverse_string(""), "") def test_reverse_palindrome(self): self.assertEqual(reverse_string("racecar"), "racecar") """, entry_point="reverse_string", ) # 6. Passing: list flattening CASE_PASS_FLATTEN = make_sample( task_id="test/flatten_pass", res_id=5, code_prompt="def flatten(lst):", solution="""def flatten(lst): result = [] for item in lst: if isinstance(item, list): result.extend(flatten(item)) else: result.append(item) return result """, test=""" import unittest class TestCases(unittest.TestCase): def test_flat(self): self.assertEqual(flatten([1, 2, 3]), [1, 2, 3]) def test_nested(self): self.assertEqual(flatten([1, [2, 3], [4, [5]]]), [1, 2, 3, 4, 5]) def test_empty(self): self.assertEqual(flatten([]), []) """, entry_point="flatten", ) ALL_CASES = [ ("PASS add", CASE_PASS_ADD, "pass"), ("FAIL multiply", CASE_FAIL_MULTIPLY, "fail"), ("PASS fibonacci", CASE_PASS_FIB, "pass"), ("FAIL divide_by_zero",CASE_FAIL_EXCEPTION, "fail"), ("PASS reverse_string",CASE_PASS_REVERSE, "pass"), ("PASS flatten", CASE_PASS_FLATTEN, "pass"), ] def run_tests(): print(f"Sending {len(ALL_CASES)} samples to {EVALUATE_URL}") print(f"(calibrate=False so solutions are evaluated as-is)\n") samples = [case for _, case, _ in ALL_CASES] expected = {case["task_id"]: exp for _, case, exp in ALL_CASES} # Send all samples in one batch with httpx.Client(timeout=TIMEOUT) as client: resp = client.post( EVALUATE_URL, json=samples, params={"calibrate": False, "min_time_limit": 1}, ) if resp.status_code != 200: print(f"ERROR: HTTP {resp.status_code}") print(resp.text) sys.exit(1) results = resp.json() print(f"Date: {results['date']}\n") print(f"{'Test Case':<25} {'Expected':<10} {'Actual':<10} {'Result'}") print("-" * 60) passed = 0 total = 0 for label, case, exp_status in ALL_CASES: task_id = case["task_id"] task_results = results["eval"].get(task_id, []) if not task_results: actual = "MISSING" ok = False else: actual = task_results[0]["status"] ok = actual == exp_status status_str = "OK" if ok else "MISMATCH" print(f"{label:<25} {exp_status:<10} {actual:<10} {status_str}") if not ok and task_results: details = task_results[0].get("details", {}) if details: for k, v in details.items(): # Print first line of detail only first_line = str(v).split("\n")[0][:80] print(f" detail[{k}]: {first_line}") total += 1 if ok: passed += 1 print("-" * 60) print(f"\n{passed}/{total} test cases matched expected status.") if passed == total: print("\nAll tests produced expected results.") else: print("\nSome tests did not match. Review details above.") sys.exit(1) if __name__ == "__main__": run_tests()