Spaces:
Sleeping
Sleeping
File size: 3,637 Bytes
869af79 f18b8fb 869af79 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | """Test grader functions the way the OpenEnv validator would call them."""
from guardian_openenv.task_graders import (
grade_value_hotel_budget_guard,
grade_airline_seat_upsell_gauntlet,
grade_marketplace_ghost_checkout,
)
GRADERS = [
("value_hotel_budget_guard", grade_value_hotel_budget_guard),
("airline_seat_upsell_gauntlet", grade_airline_seat_upsell_gauntlet),
("marketplace_ghost_checkout", grade_marketplace_ghost_checkout),
]
print("=== Test 1: Called with no arguments ===")
for task_id, fn in GRADERS:
result = fn()
score = result["score"]
valid = 0 < score < 1
print(f" {task_id}: score={score} valid={valid}")
if not valid:
print(f" *** FAIL: score is out of (0,1) range!")
# Check breakdown
for k, v in result.get("grader_breakdown", {}).items():
if isinstance(v, float) and (v <= 0.0 or v >= 1.0):
print(f" *** breakdown {k}={v} OUT OF RANGE")
print()
print("=== Test 2: Called with trajectory=[] ===")
for task_id, fn in GRADERS:
result = fn(trajectory=[])
score = result["score"]
valid = 0 < score < 1
print(f" {task_id}: score={score} valid={valid}")
print()
print("=== Test 3: Called with environment=None, trajectory=[] ===")
for task_id, fn in GRADERS:
result = fn(environment=None, trajectory=[])
score = result["score"]
valid = 0 < score < 1
print(f" {task_id}: score={score} valid={valid}")
print()
print("=== Test 4: Run full episode then grade ===")
from guardian_openenv.environment import GuardianReviewEnvironment
from guardian_openenv.models import GuardianAction, ActionType
env = GuardianReviewEnvironment()
for task_id, fn in GRADERS:
env.reset(task_id)
# Take a few actions
env.step(GuardianAction(action_type=ActionType.INSPECT_SECTION, section_id=env._task.sections[0].section_id))
env.step(GuardianAction(action_type=ActionType.SUBMIT_DECISION))
# Now grade using the environment
result = fn(environment=env)
score = result["score"]
valid = 0 < score < 1
print(f" {task_id}: score={score} valid={valid}")
if not valid:
print(f" *** FAIL: score is out of (0,1) range!")
for k, v in result.get("grader_breakdown", {}).items():
if isinstance(v, float) and (v <= 0.0 or v >= 1.0):
print(f" *** breakdown {k}={v} OUT OF RANGE")
print()
print("=== Test 5: Grade with state dict (as validator might pass) ===")
for task_id, fn in GRADERS:
env.reset(task_id)
state_dict = env.state().model_dump()
result = fn(environment=state_dict)
score = result["score"]
valid = 0 < score < 1
print(f" {task_id}: score={score} valid={valid}")
print()
print("=== Test 6: inference_scores.json check ===")
import json
from pathlib import Path
scores_path = Path("outputs/inference_scores.json")
if scores_path.exists():
data = json.loads(scores_path.read_text())
for task in data.get("tasks", []):
tid = task["task_id"]
fs = task.get("score")
if fs is not None:
valid = 0 < fs < 1
print(f" {tid}: score={fs} valid={valid}")
else:
print(f" {tid}: MISSING score FIELD")
valid = False
if not valid:
print(f" *** FAIL!")
for k, v in task.get("grader_breakdown", {}).items():
if isinstance(v, float) and (v <= 0.0 or v >= 1.0):
print(f" *** breakdown {k}={v} OUT OF RANGE")
ms = data.get("mean_score")
print(f" mean_score={ms} valid={0 < ms < 1}")
else:
print(" (file not found)")
print()
print("ALL TESTS COMPLETE")
|