Spaces:
Sleeping
Sleeping
File size: 2,056 Bytes
7ee2ab0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | PHASES = [
{"label": "Dev (test-dev2025)", "codename": "test-dev2025"},
{"label": "Standard (test-standard2025)", "codename": "test-standard2025"},
{"label": "Challenge (test-challenge2025)", "codename": "test-challenge2025"},
]
LEADERBOARD_METRICS = [
"overall_f1", "overall_precision", "overall_recall",
"vizwiz_f1", "vizwiz_precision", "vizwiz_recall",
"vqav2_f1", "vqa_precision", "vqa_recall",
]
DEFAULT_SORT_METRIC = "overall_f1"
LEADERBOARD_FILE = "leaderboards/answer-therapy.jsonl"
CHALLENGE_PHASE = "test-challenge2025"
SUBFOLDER = "answer-therapy"
CHALLENGE_TYPE = "VQA Answer Therapy"
EVAL_DETAILS_MD = """
### How is the Score Calculated?
Each entry is a binary classification: does the visual question produce answers that all
share the **same image region** (single grounding), or do different answers point to
**different regions** (multiple groundings)?
Your `single_grounding` confidence score is thresholded at **0.5** for evaluation.
| Metric | Description |
|--------|-------------|
| `Overall F1` | F1 score across all questions *(default ranking metric)* |
| `Overall Precision` | Precision across all questions |
| `Overall Recall` | Recall across all questions |
| `VizWiz F1` | F1 on questions from the VizWiz dataset |
| `VQAv2 F1` | F1 on questions from the VQAv2 dataset |
Scores are reported as percentages (0–100).
"""
FORMAT_MD = """
### Submission Format
Your JSON file must be a **list of result objects**, one per visual question:
```json
[
{
"question_id": "VizWiz_test_000000020000.jpg",
"single_grounding": 0.85
},
{
"question_id": "249549029",
"single_grounding": 0.12
},
...
]
```
- **`question_id`** — string. Use the image filename for VizWiz questions (e.g. `VizWiz_test_00002183.jpg`) and the numeric string ID for VQAv2 questions.
- **`single_grounding`** — float between 0.0 and 1.0. Confidence that all answers share the same grounding region. `1` = single grounding, `0` = multiple groundings.
"""
|