challenges/answer_therapy/config.py

PHASES = [
    {"label": "Dev (test-dev2025)",             "codename": "test-dev2025"},
    {"label": "Standard (test-standard2025)",   "codename": "test-standard2025"},
    {"label": "Challenge (test-challenge2025)", "codename": "test-challenge2025"},
]

LEADERBOARD_METRICS = [
    "overall_f1", "overall_precision", "overall_recall",
    "vizwiz_f1",  "vizwiz_precision",  "vizwiz_recall",
    "vqav2_f1",   "vqa_precision",     "vqa_recall",
]
DEFAULT_SORT_METRIC = "overall_f1"
LEADERBOARD_FILE    = "leaderboards/answer-therapy.jsonl"
CHALLENGE_PHASE     = "test-challenge2025"
SUBFOLDER           = "answer-therapy"
CHALLENGE_TYPE      = "VQA Answer Therapy"

EVAL_DETAILS_MD = """
### How is the Score Calculated?

Each entry is a binary classification: does the visual question produce answers that all
share the **same image region** (single grounding), or do different answers point to
**different regions** (multiple groundings)?

Your `single_grounding` confidence score is thresholded at **0.5** for evaluation.

| Metric | Description |
|--------|-------------|
| `Overall F1` | F1 score across all questions *(default ranking metric)* |
| `Overall Precision` | Precision across all questions |
| `Overall Recall` | Recall across all questions |
| `VizWiz F1` | F1 on questions from the VizWiz dataset |
| `VQAv2 F1` | F1 on questions from the VQAv2 dataset |

Scores are reported as percentages (0–100).
"""

FORMAT_MD = """
### Submission Format

Your JSON file must be a **list of result objects**, one per visual question:

```json
[
  {
    "question_id": "VizWiz_test_000000020000.jpg",
    "single_grounding": 0.85
  },
  {
    "question_id": "249549029",
    "single_grounding": 0.12
  },
  ...
]
```

- **`question_id`** — string. Use the image filename for VizWiz questions (e.g. `VizWiz_test_00002183.jpg`) and the numeric string ID for VQAv2 questions.
- **`single_grounding`** — float between 0.0 and 1.0. Confidence that all answers share the same grounding region. `1` = single grounding, `0` = multiple groundings.
"""