scorer.py

import json
import re
import string
import warnings
import pandas as pd
import numpy as np
import os

def instruction_scorer(data, judgment_file, model_name):

    df = data
    img_dict = {}
    for j in range(len(df)):
        row = df.iloc[j]
        img_dict[row['image_url']] = {'category': row['category']}
    
    with open(judgment_file, 'r') as f:
        judgements = json.load(f)
    
    model_data = judgements[model_name]

    model_analysis = {}

    cat = {'time': [0,0], 'shopping': [0,0], 'navigation-transportation': [0,0], 'abstract': [0,0], 'app': [0,0], 'web': [0,0], 'infographics': [0,0], 'stvqa': [0,0], 'estvqa': [0,0]}

    count, total = 0, 0
    
    for key in model_data:
        if key in img_dict:
            img_data = img_dict[key]
            rating = model_data[key]
            count += rating
            total += 1
            cat[img_data['category']][1] += 1
            cat[img_data['category']][0] += rating

    model_analysis[model_name] = {'category': cat}
    
    x = model_analysis[model_name]['category']
    
    output_dict = {}

    for h in x:
        output_dict[h]=100*x[h][0]/x[h][1]

    output_dict["misc"]= 100 * (x['stvqa'][0] + x['estvqa'][0])/(x['stvqa'][1] + x['stvqa'][1])

    output_dict["average"] = (output_dict["time"]+output_dict["shopping"]+output_dict["navigation-transportation"]+output_dict["abstract"]+output_dict["app"]+output_dict["web"]+output_dict["infographics"]+output_dict["misc"])/8

    return output_dict