File size: 1,571 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | import unittest
from agent_harness.interactive_experiment import (
InteractiveExperimentError,
parse_query,
score_selection_response,
)
class InteractiveExperimentTests(unittest.TestCase):
def test_invalid_query_response_is_rejected_for_runner_fallback(self) -> None:
response = {"choices": [{"message": {"content": "unfinished reasoning"}}]}
with self.assertRaises(InteractiveExperimentError):
parse_query(response)
def test_invalid_candidate_selection_is_scored_as_protocol_violation(self) -> None:
response = {
"choices": [
{
"message": {
"content": '{"files":["not-offered.go"],"reasoning":"guess"}'
}
}
]
}
selection, violation = score_selection_response(response, {"offered.go"})
self.assertEqual(selection, {"files": [], "reasoning": ""})
self.assertIsNotNone(violation)
self.assertIn("outside the candidates", violation or "")
def test_valid_candidate_selection_has_no_protocol_violation(self) -> None:
response = {
"choices": [
{
"message": {
"content": '{"files":["offered.go"],"reasoning":"direct match"}'
}
}
]
}
selection, violation = score_selection_response(response, {"offered.go"})
self.assertEqual(selection["files"], ["offered.go"])
self.assertIsNone(violation)
|