benchmark/metrics/vqa.py

from pathlib import Path
from typing import Dict

import t2v_metrics

class VQAMetric:
    def __init__(self):
        self.metric = t2v_metrics.VQAScore(model="clip-flant5-xxl")
    
    @property
    def name(self) -> str:
        return "vqa_score"
    
    def compute_score(
        self,
        image_path: Path,
        prompt: str,
    ) -> Dict[str, float]:
        score = self.metric(images=[str(image_path)], texts=[prompt])
        return {"vqa": score[0][0].item()}