"""Template task — scene-level. **Copy this directory** and edit. Steps: 1. Replace TASK_NAME, display strings. 2. Update validate_scene / evaluate_scene / load_gt_scene. 3. Choose primary_metric. 4. Register in src/tasks/__init__.py. 5. Declare the task scope in the **Worker repo only** at ``PhysInOne-Leaderboard-Worker/task_manifest/.json`` and upload ground truth to ``/data/Test//_trajectory/``. """ from __future__ import annotations import json import os from typing import Any, Dict from src.tasks.base import TaskPlugin TASK_NAME = "template" def validate_scene(scene_dir: str) -> None: if not os.path.isfile(os.path.join(scene_dir, "prediction.json")): raise ValueError("prediction.json is missing from the scene ZIP root") def evaluate_scene(scene_dir: str, gt_scene: Any) -> Dict[str, float]: with open(os.path.join(scene_dir, "prediction.json"), "r", encoding="utf-8") as f: pred = json.load(f) true_label = gt_scene.get("label") if isinstance(gt_scene, dict) else gt_scene return {"accuracy": 1.0 if pred.get("label") == true_label else 0.0} def load_gt_scene(scene_id: str, gt_dir: str) -> Any: with open(os.path.join(gt_dir, "label.json"), "r", encoding="utf-8") as f: return json.load(f) TASK = TaskPlugin( name=TASK_NAME, display_name="Template Task", description="Copy this directory to `src/tasks//` and then customize it.", expected_scene_layout=( "```\n" ".zip\n" "└── prediction.json # {\"label\": ...}\n" "```\n" "GT is stored at `ground_truth//label.json` in the private dataset." ), validate_scene_fn=validate_scene, evaluate_scene_fn=evaluate_scene, load_gt_scene_fn=load_gt_scene, primary_metric="accuracy", higher_is_better=True, leaderboard_columns=["accuracy"], )