Spaces:
Sleeping
Sleeping
| import json | |
| import random | |
| import os | |
| import time | |
| from random import randint | |
| from os import listdir | |
| import numpy as np | |
| from sklearn.metrics import average_precision_score, f1_score,recall_score | |
| #delete cv2 | |
| index_list = [728, 542, 519, 1357, 1007, 1380, 783, 733, 1327, 760, 684, 795, 1075, 917, 638, 1020, 1180, 1225, 914, 1236, 871, 1231, 1330, 602, 870, 639, 1187, 934, 626, 1255, 1064, 1281, 689, 1078, 1043, 735, 1159, 1189, 1140, 579, 554, 752, 1095, 574, 841, 1054, 622, 547, 908, 1267, 767, 1215, 739, 664, 1365, 984, 1243, 1165, 634, 972, 1329, 804, 855, 1249, 311, 415, 448, 319, 14, 472, 356, 407, 265, 241, 309, 176, 105, 383, 32, 349, 149, 436, 343, 21, 147, 31, 240, 314, 9, 100, 190, 11, 218, 20, 346, 89, 25, 112, 243, 357] | |
| # | |
| current_dir_path = os.path.dirname(os.path.realpath(__file__)) | |
| def All_scores(): | |
| All_scores = F1_score(All_gt_binary,All_predict_binary) | |
| VQA_scores = F1_score(VQA_gt_binary,VQA_predict_binary) | |
| VizWiz_scores = F1_score(VizWiz_gt_binary,VizWiz_predict_binary) | |
| return All_scores, VizWiz_scores, VQA_scores | |
| def F1_score(gt_binary,predict_binary): | |
| # predict_binary/gt_binary is np.array | |
| length_diff = len(gt_binary)-len(predict_binary) | |
| assert len(gt_binary) == len(predict_binary) | |
| acc = 1- np.mean(abs(gt_binary - predict_binary)) | |
| print("accuracy: " + str(acc) + "\n") | |
| # consider positive is diff, negative is same | |
| print("consider positive is diff, negative is same") | |
| relevant_set = [gt_binary[i] for i in range(len(gt_binary)) if predict_binary[i]==1] | |
| precision = np.mean(relevant_set) | |
| print("precision: " + str(precision)) | |
| relevant_element = [predict_binary[i] for i in range(len(gt_binary)) if gt_binary[i]==1] | |
| recall = np.mean(relevant_element) | |
| print("recall: " + str(recall)) | |
| scores={} | |
| scores["Overall_f1_score"] = 2*precision*recall/(precision+recall) | |
| scores["Oveall_precision"]=precision | |
| scores["Overall_recall"] = recall | |
| print("scores",scores) | |
| # scores["VizWiz_f1_score"] = | |
| # scores["VizWiz_precision"]= | |
| # scores["VizWiz_recall"] = | |
| # scores["VQA_f1_score"] = | |
| # scores["VQA_precision"]= | |
| # scores["VQA_recall"] = | |
| return scores | |
| def evaluate_unanswerability(self, imgs=None): | |
| if imgs == None: | |
| imgs = [img for img in self.params['images']] | |
| pred = [] | |
| gt_labels = [] | |
| for img in imgs: | |
| gt_labels.append(self.vqa.imgToQA[img]['answerable']) | |
| pred.append(self.vqaRes.imgToQA[img]['answerable']) | |
| gt_labels = np.array(gt_labels) | |
| pred = np.array(pred) | |
| gt_labels_n = 1 - gt_labels | |
| pred_n = 1.0 - pred | |
| average_precision = average_precision_score(gt_labels_n, pred_n) | |
| one_f1_score = f1_score(gt_labels_n, pred_n > 0.5) | |
| self.unanswerability['average_precision'] = round(100*average_precision, self.n) | |
| self.unanswerability['f1_score'] = round(100*one_f1_score, self.n) | |
| def get_overlap_index(list1, list2): | |
| index_list=[] | |
| for i in range(len(list1)): | |
| if list1[i] in list2: | |
| index_list.append(list2.index(list1[i])) | |
| return index_list | |
| def GroundingDifference(annFile, resFile): | |
| Anns_vqa = [] | |
| Anns_vizwiz = [] | |
| Ress_vqa=[] | |
| Ress_vizwiz=[] | |
| with open(annFile,'r') as annF: | |
| with open(resFile,'r') as resF: | |
| anns = json.load(annF) | |
| test_ress = json.load(resF) | |
| if len(anns) == 100: | |
| # print(get_overlap_index(anns,test_ress)) | |
| ress = [test_ress[i] for i in index_list] | |
| else: | |
| ress = test_ress | |
| Anns_labels = np.array([ann["single_grounding"] for ann in anns]) | |
| Res_labels = np.array([res["single_grounding"] for res in ress]) | |
| for ann in anns: | |
| if ann["question_id"].startswith("Viz"): | |
| Anns_vizwiz.append(ann) | |
| else: | |
| Anns_vqa.append(ann) | |
| for res in ress: | |
| if res["question_id"].startswith("Viz"): | |
| Ress_vizwiz.append(res) | |
| else: | |
| Ress_vqa.append(res) | |
| # All_Ress = Ress_vizwiz+Ress_vqa | |
| Anns_vizwiz_labels = np.array([ann["single_grounding"] for ann in Anns_vizwiz]) | |
| Anns_vqa_labels = np.array([ann["single_grounding"] for ann in Anns_vqa]) | |
| Ress_vizwiz_labels = np.array([res["single_grounding"] for res in Ress_vizwiz]) | |
| Ress_vqa_labels = np.array([res["single_grounding"] for res in Ress_vqa]) | |
| if len(Anns_labels) !=len(Res_labels): | |
| print("unsucessful submission! The number of files you generated is not equal to the number of ground-truth files.") | |
| else: | |
| results ={} | |
| # F1_score(Anns_labels, Res_labels) | |
| # F1_score(Anns_vizwiz_labels, Ress_vizwiz_labels) | |
| # F1_score(Anns_vqa_labels, Ress_vqa_labels) | |
| results["overall_f1"] = round(100*f1_score(Anns_labels,Res_labels>0.5),2) | |
| results['overall_precision'] = round(100*average_precision_score(Anns_labels,Res_labels>0.5), 2) | |
| results['overall_recall'] = round(100*recall_score(Anns_labels,Res_labels>0.5), 2) | |
| results['vqav2_f1'] = round(100*f1_score(Anns_vqa_labels,Ress_vqa_labels>0.5),2) | |
| results['vqa_precision'] = round(100*average_precision_score(Anns_vqa_labels,Ress_vqa_labels>0.5), 2) | |
| results['vqa_recall'] = round(100*recall_score(Anns_vqa_labels,Ress_vqa_labels>0.5), 2) | |
| results['vizwiz_f1'] = round(100*f1_score(Anns_vizwiz_labels,Ress_vizwiz_labels>0.5),2) | |
| results['vizwiz_precision'] = round(100*average_precision_score(Anns_vizwiz_labels,Ress_vizwiz_labels>0.5), 2) | |
| results['vizwiz_recall'] = round(100*recall_score(Anns_vizwiz_labels,Ress_vizwiz_labels>0.5), 2) | |
| return results | |
| phase_splits = { | |
| "test-dev2025": ["test-dev"], | |
| "test-standard2025": ["test"], | |
| "test-challenge2025": ["test"] | |
| } | |
| def evaluate(test_annotation_file, resFile, phase_codename, **kwargs): | |
| result = [] | |
| splits = phase_splits[phase_codename] | |
| for split in splits: | |
| annFile = os.path.join(current_dir_path, "Annotations", split + ".json") | |
| # if "dev" in phase_codename: | |
| print(phase_codename) | |
| result.append({split: GroundingDifference(annFile,resFile)}) # return a dict from groundingDifference() | |
| output = {"result": result} | |
| output["submission_result"] = result | |
| # output["submission_result"] = output["result"][0] | |
| print(result) | |
| print("Completed evaluation for Test Phase") | |
| return output | |
| if __name__=="__main__": | |
| evaluate("t","Results/test.json","test-dev2025") |