server/registry.py

"""
Task and Grader Registry for OpenEnv Validation
Provides enumerable access to all tasks with their corresponding graders
"""

from server.tasks import TASKS
from server.graders import GRADERS


def get_tasks_with_graders():
    """
    Returns a list of tasks that have graders.
    Used by validators to ensure all tasks have scoring functions.
    
    Returns:
        dict: Mapping of task_name -> {"task_config": ..., "grader": ...}
    """
    tasks_with_graders = {}
    
    for task_name, task_config in TASKS.items():
        if task_name in GRADERS:
            tasks_with_graders[task_name] = {
                "name": task_name,
                "description": task_config.get("description", ""),
                "difficulty": task_config.get("difficulty", ""),
                "content_type": task_config.get("content_type", ""),
                "num_items": len(task_config.get("items", [])),
                "grader": GRADERS[task_name],
                "grader_name": GRADERS[task_name].__name__,
            }
    
    return tasks_with_graders


def validate_tasks():
    """
    Validates that all tasks have proper graders.
    
    Returns:
        dict: Validation results
    """
    tasks_with_graders = get_tasks_with_graders()
    
    validation = {
        "total_tasks": len(TASKS),
        "tasks_with_graders": len(tasks_with_graders),
        "tasks": list(tasks_with_graders.keys()),
        "graders": list(GRADERS.keys()),
        "valid": len(tasks_with_graders) >= 3,
    }
    
    # Check for mismatches
    for task_name in TASKS:
        if task_name not in GRADERS:
            validation[f"missing_grader_for_{task_name}"] = True
    
    for grader_name in GRADERS:
        if grader_name not in TASKS:
            validation[f"extra_grader_{grader_name}"] = True
    
    return validation


if __name__ == "__main__":
    import json
    
    print("=" * 60)
    print("Task and Grader Validation")
    print("=" * 60)
    
    validation = validate_tasks()
    print(json.dumps(validation, indent=2, default=str))
    
    print("\n" + "=" * 60)
    print("Tasks with Graders:")
    print("=" * 60)
    
    for task_name, task_info in get_tasks_with_graders().items():
        print(f"\n✓ {task_name}")
        print(f"  Description: {task_info['description']}")
        print(f"  Difficulty: {task_info['difficulty']}")
        print(f"  Items: {task_info['num_items']}")
        print(f"  Grader: {task_info['grader_name']}")