File size: 3,108 Bytes
80e6c74 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | import json
def evaluate_vqa_answers(groundtruth_file, test_file, output_file):
"""
Compares the chosen answers from a test file with ground truth answers.
Votes as correct (1) if any test answer is in the ground truth answer list, otherwise incorrect (0).
Parameters:
groundtruth_file (str): Path to the ground truth JSON file.
test_file (str): Path to the test JSON file.
output_file (str): Path to save the evaluation results.
"""
# Load the JSON files
with open(groundtruth_file, "r", encoding="utf-8") as f:
groundtruth_data = json.load(f)
with open(test_file, "r", encoding="utf-8") as f:
test_data = json.load(f)
evaluation_results = {} # Stores evaluation per image
total_correct = 0
total_questions = 0
# Iterate through images in the test file
for image_path, test_questions in test_data.items():
if image_path not in groundtruth_data:
print(f"Skipping {image_path}, not found in ground truth.")
continue
groundtruth_questions = groundtruth_data[image_path]
image_results = []
# Ensure both lists have the same number of questions
min_length = min(len(test_questions), len(groundtruth_questions))
for i in range(min_length):
test_qa = test_questions[i]
groundtruth_qa = groundtruth_questions[i]
test_answer = set(test_qa.get("chosen answer", [])) # Get list, default to empty
groundtruth_answer = set(groundtruth_qa.get("chosen answer", [])) # Default empty
if not groundtruth_answer:
print ("Groundtruth empty case, skip this case")
total_questions -= 1
is_correct = 0
elif test_answer & groundtruth_answer:
is_correct = 1 # Correct match
else:
is_correct = 0 # Incorrect match
image_results.append({
"question": test_qa.get("question", ""),
"test answer": list(test_answer),
"groundtruth answer": list(groundtruth_answer),
"correct": is_correct
})
total_correct += is_correct
total_questions += 1
evaluation_results[image_path] = image_results
# Calculate overall accuracy
accuracy = (total_correct / total_questions) * 100 if total_questions > 0 else 0
print(f"Total Correct: {total_correct}/{total_questions} | Accuracy: {accuracy:.2f}%")
# Save evaluation results to JSON
with open(output_file, "w", encoding="utf-8") as f:
json.dump(evaluation_results, f, indent=4)
print(f"Evaluation results saved to {output_file}")
# Example Usage
evaluate_vqa_answers(
groundtruth_file="../dataset/Gut-VLM/train_test_split/VQA_format_testset_only.json",
test_file="../results/qwen_caption_hal_aware_caption2vqa_parsed.json",
output_file="../results/qwen_hal_aware_QAAS_results.json"
) |