""" Task and Grader Registry for OpenEnv Validation Provides enumerable access to all tasks with their corresponding graders """ from server.tasks import TASKS from server.graders import GRADERS def get_tasks_with_graders(): """ Returns a list of tasks that have graders. Used by validators to ensure all tasks have scoring functions. Returns: dict: Mapping of task_name -> {"task_config": ..., "grader": ...} """ tasks_with_graders = {} for task_name, task_config in TASKS.items(): if task_name in GRADERS: tasks_with_graders[task_name] = { "name": task_name, "description": task_config.get("description", ""), "difficulty": task_config.get("difficulty", ""), "content_type": task_config.get("content_type", ""), "num_items": len(task_config.get("items", [])), "grader": GRADERS[task_name], "grader_name": GRADERS[task_name].__name__, } return tasks_with_graders def validate_tasks(): """ Validates that all tasks have proper graders. Returns: dict: Validation results """ tasks_with_graders = get_tasks_with_graders() validation = { "total_tasks": len(TASKS), "tasks_with_graders": len(tasks_with_graders), "tasks": list(tasks_with_graders.keys()), "graders": list(GRADERS.keys()), "valid": len(tasks_with_graders) >= 3, } # Check for mismatches for task_name in TASKS: if task_name not in GRADERS: validation[f"missing_grader_for_{task_name}"] = True for grader_name in GRADERS: if grader_name not in TASKS: validation[f"extra_grader_{grader_name}"] = True return validation if __name__ == "__main__": import json print("=" * 60) print("Task and Grader Validation") print("=" * 60) validation = validate_tasks() print(json.dumps(validation, indent=2, default=str)) print("\n" + "=" * 60) print("Tasks with Graders:") print("=" * 60) for task_name, task_info in get_tasks_with_graders().items(): print(f"\n✓ {task_name}") print(f" Description: {task_info['description']}") print(f" Difficulty: {task_info['difficulty']}") print(f" Items: {task_info['num_items']}") print(f" Grader: {task_info['grader_name']}")