test_evaluate.py

"""
Sample test cases for the /evaluate/ endpoint.

Sends requests to the deployed evaluator at:
  https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space/evaluate/

Each test case constructs a payload with:
  - task_id, res_id, solution, test (a unittest.TestCase), entry_point, code_prompt
and validates the response status ("pass", "fail", or "timeout").
"""

import httpx
import json
import sys

BASE_URL = "https://thinkahead-oe-eval-bcb-lite-evaluator.hf.space"
EVALUATE_URL = f"{BASE_URL}/evaluate/"
TIMEOUT = 300  # seconds


def make_sample(task_id, res_id, code_prompt, solution, test, entry_point):
    return {
        "task_id": task_id,
        "res_id": res_id,
        "code_prompt": code_prompt,
        "solution": solution,
        "test": test,
        "entry_point": entry_point,
    }


# ---------------------------------------------------------------------------
# Test case definitions
# ---------------------------------------------------------------------------

# 1. Passing: correct add function
CASE_PASS_ADD = make_sample(
    task_id="test/add_pass",
    res_id=0,
    code_prompt="def add(a, b):",
    solution="def add(a, b):\n    return a + b\n",
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_add_positive(self):
        self.assertEqual(add(2, 3), 5)
    def test_add_negative(self):
        self.assertEqual(add(-1, -1), -2)
    def test_add_zero(self):
        self.assertEqual(add(0, 0), 0)
""",
    entry_point="add",
)

# 2. Failing: wrong multiply implementation
CASE_FAIL_MULTIPLY = make_sample(
    task_id="test/multiply_fail",
    res_id=1,
    code_prompt="def multiply(a, b):",
    solution="def multiply(a, b):\n    return a + b\n",  # bug: adds instead of multiplies
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_multiply_basic(self):
        self.assertEqual(multiply(3, 4), 12)
    def test_multiply_zero(self):
        self.assertEqual(multiply(0, 5), 0)
""",
    entry_point="multiply",
)

# 3. Passing: fibonacci function
CASE_PASS_FIB = make_sample(
    task_id="test/fibonacci_pass",
    res_id=2,
    code_prompt="def fibonacci(n):",
    solution="""def fibonacci(n):
    if n <= 0:
        return 0
    elif n == 1:
        return 1
    a, b = 0, 1
    for _ in range(2, n + 1):
        a, b = b, a + b
    return b
""",
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_fib_zero(self):
        self.assertEqual(fibonacci(0), 0)
    def test_fib_one(self):
        self.assertEqual(fibonacci(1), 1)
    def test_fib_ten(self):
        self.assertEqual(fibonacci(10), 55)
    def test_fib_twenty(self):
        self.assertEqual(fibonacci(20), 6765)
""",
    entry_point="fibonacci",
)

# 4. Failing: function raises an exception
CASE_FAIL_EXCEPTION = make_sample(
    task_id="test/divide_exception",
    res_id=3,
    code_prompt="def safe_divide(a, b):",
    solution="""def safe_divide(a, b):
    return a / b
""",
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_divide_normal(self):
        self.assertEqual(safe_divide(10, 2), 5.0)
    def test_divide_by_zero(self):
        self.assertEqual(safe_divide(1, 0), 0)
""",
    entry_point="safe_divide",
)

# 5. Passing: string reversal
CASE_PASS_REVERSE = make_sample(
    task_id="test/reverse_pass",
    res_id=4,
    code_prompt="def reverse_string(s):",
    solution="def reverse_string(s):\n    return s[::-1]\n",
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_reverse_word(self):
        self.assertEqual(reverse_string("hello"), "olleh")
    def test_reverse_empty(self):
        self.assertEqual(reverse_string(""), "")
    def test_reverse_palindrome(self):
        self.assertEqual(reverse_string("racecar"), "racecar")
""",
    entry_point="reverse_string",
)

# 6. Passing: list flattening
CASE_PASS_FLATTEN = make_sample(
    task_id="test/flatten_pass",
    res_id=5,
    code_prompt="def flatten(lst):",
    solution="""def flatten(lst):
    result = []
    for item in lst:
        if isinstance(item, list):
            result.extend(flatten(item))
        else:
            result.append(item)
    return result
""",
    test="""
import unittest
class TestCases(unittest.TestCase):
    def test_flat(self):
        self.assertEqual(flatten([1, 2, 3]), [1, 2, 3])
    def test_nested(self):
        self.assertEqual(flatten([1, [2, 3], [4, [5]]]), [1, 2, 3, 4, 5])
    def test_empty(self):
        self.assertEqual(flatten([]), [])
""",
    entry_point="flatten",
)


ALL_CASES = [
    ("PASS  add",           CASE_PASS_ADD,          "pass"),
    ("FAIL  multiply",      CASE_FAIL_MULTIPLY,     "fail"),
    ("PASS  fibonacci",     CASE_PASS_FIB,          "pass"),
    ("FAIL  divide_by_zero",CASE_FAIL_EXCEPTION,    "fail"),
    ("PASS  reverse_string",CASE_PASS_REVERSE,      "pass"),
    ("PASS  flatten",       CASE_PASS_FLATTEN,      "pass"),
]


def run_tests():
    print(f"Sending {len(ALL_CASES)} samples to {EVALUATE_URL}")
    print(f"(calibrate=False so solutions are evaluated as-is)\n")

    samples = [case for _, case, _ in ALL_CASES]
    expected = {case["task_id"]: exp for _, case, exp in ALL_CASES}

    # Send all samples in one batch
    with httpx.Client(timeout=TIMEOUT) as client:
        resp = client.post(
            EVALUATE_URL,
            json=samples,
            params={"calibrate": False, "min_time_limit": 1},
        )

    if resp.status_code != 200:
        print(f"ERROR: HTTP {resp.status_code}")
        print(resp.text)
        sys.exit(1)

    results = resp.json()

    print(f"Date: {results['date']}\n")
    print(f"{'Test Case':<25} {'Expected':<10} {'Actual':<10} {'Result'}")
    print("-" * 60)

    passed = 0
    total = 0
    for label, case, exp_status in ALL_CASES:
        task_id = case["task_id"]
        task_results = results["eval"].get(task_id, [])
        if not task_results:
            actual = "MISSING"
            ok = False
        else:
            actual = task_results[0]["status"]
            ok = actual == exp_status

        status_str = "OK" if ok else "MISMATCH"
        print(f"{label:<25} {exp_status:<10} {actual:<10} {status_str}")

        if not ok and task_results:
            details = task_results[0].get("details", {})
            if details:
                for k, v in details.items():
                    # Print first line of detail only
                    first_line = str(v).split("\n")[0][:80]
                    print(f"  detail[{k}]: {first_line}")

        total += 1
        if ok:
            passed += 1

    print("-" * 60)
    print(f"\n{passed}/{total} test cases matched expected status.")

    if passed == total:
        print("\nAll tests produced expected results.")
    else:
        print("\nSome tests did not match. Review details above.")
        sys.exit(1)


if __name__ == "__main__":
    run_tests()