File size: 1,736 Bytes
256c9c2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """
{{PAPER_TITLE}} — Evaluation Metrics
Paper: https://arxiv.org/abs/{{ARXIV_ID}}
Implements: Metric computation for {{METRICS_LISTED}}
Section references:
{{§SECTION}} — {{evaluation methodology description}}
NOTE: This file provides metric computation functions only.
It does NOT handle data loading or model inference.
Pass predictions and targets to compute metrics.
"""
from typing import Dict, List, Optional
import torch
import numpy as np
def compute_{{METRIC_1}}(
predictions: torch.Tensor,
targets: torch.Tensor,
) -> float:
"""§{{SECTION}} — Compute {{metric name}}.
"{{Quote from paper about evaluation metric}}"
Args:
predictions: model output — shape: (batch, {{dims}})
targets: ground truth — shape: (batch, {{dims}})
Returns:
{{metric_name}} score as a float
NOTE: {{any caveats about metric implementation, e.g.,
"Different BLEU implementations give different numbers.
We use sacrebleu for reproducibility."}}
"""
# §{{SECTION}} — metric computation
pass # REPLACE with actual metric
def compute_all_metrics(
predictions: torch.Tensor,
targets: torch.Tensor,
) -> Dict[str, float]:
"""Compute all evaluation metrics reported in the paper.
§{{SECTION}} — "We report {{metric_1}}, {{metric_2}}, and {{metric_3}}"
Args:
predictions: model output — shape: (batch, {{dims}})
targets: ground truth — shape: (batch, {{dims}})
Returns:
Dict mapping metric name to value
"""
return {
"{{metric_1}}": compute_{{METRIC_1}}(predictions, targets),
# Add more metrics as needed
}
|