Update evaluation/evaluator.py

evaluation/evaluator.py

@@ -2,8 +2,8 @@ import json
 from evaluation.evaluate_utils.evaluate_factory import get_evaluator
 import numpy as np
 
-def find_isnan(samp):
 
+def find_isnan(samp):
     try:
         if np.isnan(samp):
             return True
@@ -11,10 +11,9 @@ def find_isnan(samp):
             return False
     except:
         return False
-        
 
-def fix_ans(answer):
 
+def fix_ans(answer):
     try:
         answer = answer.replace("{'", '{"').replace("', '", '", "').replace("': '", '": "').replace("'}", '"}')
         answer = answer.replace("': ", '": ')
@@ -24,7 +23,6 @@ def fix_ans(answer):
 
 
 def parse_answer(answer):
-
     if len(answer) == 1:
         ans, is_num = fix_number(answer[0])
         if is_num:
@@ -47,7 +45,6 @@ def parse_answer(answer):
 
 
 def fix_number(number):
-
     if type(number) == str:
         copy_ans = number
         copy_ans = ' '.join(' '.join(' '.join(copy_ans.split('$')).split('%')).split('sqft')).strip()
@@ -64,8 +61,8 @@ def fix_number(number):
 
 
 def fix_prediction(prediction, gold_answer, evaluator):
-
-    if type(prediction) == list and len(prediction) == 1 and (type(prediction[0]) == int or ((type(prediction[0]) == str) and prediction[0].isnumeric())):
+    if type(prediction) == list and len(prediction) == 1 and (
+            type(prediction[0]) == int or ((type(prediction[0]) == str) and prediction[0].isnumeric())):
         prediction = fix_number(prediction[0])
 
     if type(prediction) != list:
@@ -86,20 +83,24 @@ def fix_prediction(prediction, gold_answer, evaluator):
 
 
 def question_scorer(prediction, gold_answer):
-
     try:
         prediction = json.loads(prediction)
     except:
         prediction = prediction
 
-    answer_list = [x for x in gold_answer.split("\n") if len(x.strip()) > 0] if type(gold_answer) != list else gold_answer
+    answer_list = [x for x in gold_answer.split("\n") if len(x.strip()) > 0] if type(
+        gold_answer) != list else gold_answer
     gold_answer, evaluator = parse_answer(answer_list)
     prediction, run_eval = fix_prediction(prediction, gold_answer, evaluator)
 
     has_ans = 1.
     if (type(prediction) != float and len(prediction) == 0) or find_isnan(prediction):
         has_ans = 0.
-        
+
+    if type(prediction) == list:
+        if all((type(pred) not in {float, int} and len(pred) == 0) or find_isnan(pred) for pred in prediction):
+            has_ans = 0
+
     if not run_eval:
         return 0., has_ans