evaluation_vdc/utils.py

def get_qa_generation_prompt(ground_truth_caption):
    system_qa_generation = """
You are an intelligent chatbot designed for generating 10 question-answer pairs given a detailed
description of a video or image. You are describing the video.
Here's how you can accomplish the task: INSTRUCTIONS: - Cover the main objects and actions in the
video or image.
- The questions should be open-ended and start with What, Who, Where, When, Why, How, etc.
- The answer should be a short sentence or phrase.
- Generate 10 question-answer pairs.
"""
    prompt_qa_generation = f"""
Please generate 10 question-answer pairs given a detailed description of a video or image: detailed description: 
{ground_truth_caption}
Please generate the response in the form of a Python list of tuple with the question and the corresponding answer. DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python list of tuple. 
For example, your response should look like this: [("the question 1", "the answer 1"), ("the question 2", "the answer 2"), ...].
"""
    message = [
        {"role": "system", "content": system_qa_generation},
        {"role": "user", "content": prompt_qa_generation}
    ]
#     prompt = f"""
# You are an intelligent chatbot designed for generating 5 question-answer pairs based on a detailed description of a video.

# INSTRUCTIONS:
# - Cover all key visual aspects, such as main characters, their actions, appearance, clothing, surroundings, and camera work.
# - Questions should be open-ended, beginning with words like 'What', 'Who', 'Where', 'When', 'Why', 'How', etc.
# - Answers must be short phrases or sentences, and grounded in the given caption.
# - Only include information that is directly mentioned or strongly implied.
# - Output your answer as a Python list of tuples (question, answer).
# - DO NOT provide any other output or explanation, only the list.

# Here is the ground truth caption:

# {ground_truth_caption}

# Please generate 5 question-answer pairs in the following format:

# [["question1", "answer1"], ["question2", "answer2"], ..., ["question5", "answer5"]]

# """
    return message


def extract_answer_from_prediction(question, predicted_caption):
    
    system_extract_answer = """
You are an intelligent chatbot designed for providing accurate answers to questions related to the
content based on a detailed description of a video or image.
Here's how you can accomplish the task:
——
##INSTRUCTIONS:
- Read the detailed description carefully.
- Answer the question only based on the detailed description.
- The answer should be a short sentence or phrase.
"""
    prompt_extract_answer = f"""
Please provide accurate answers to questions related to the content based on a detailed description of a video or image:
detailed description: {predicted_caption}
question: {question}
DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide short but accurate answer.
"""
    message = [
        {"role": "system", "content": system_extract_answer},
        {"role": "user", "content": prompt_extract_answer}
    ]
    return message


def correct_evaluation_prompt(question, correct_answer, predicted_answer):
#     prompt = f"""
# You are an intelligent chatbot designed to evaluate the accuracy and quality of predicted answers in a video-based question-answering task.

# ## INSTRUCTIONS:
# - Compare the **predicted answer** to the **correct answer** for a given question.
# - Focus on **meaningful correctness**:
#   - 1 = The predicted answer meaningfully matches the correct answer (even with paraphrasing or synonyms)
#   - 0 = The predicted answer does not match meaningfully
# - Also rate **quality (0 to 5):**
#   - Based on clarity, fluency, and coherence of the predicted answer

# ## OUTPUT:
# Return your evaluation as a Python dictionary string in the following format:
# {{'correctness': 0 or 1, 'quality': INT}}

# Do NOT include any other text or explanation — only return the dictionary string.

# Now evaluate the following:

# Question: {question}  
# Correct Answer: {correct_answer}  
# Predicted Answer: {predicted_answer}
# """

    system_correct_evaluation = """
You are an intelligent chatbot designed for evaluating the correctness of generative outputs for questionanswer pairs.
Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:
——
##INSTRUCTIONS:
- Focus on the meaningful match between the predicted answer and the correct answer.
- Consider synonyms or paraphrases as valid matches.
- Evaluate the correctness of the prediction compared to the answer
"""
    prompt_correct_evaluation = f"""
Please evaluate the following video-based question-answer pair:
Question: {question}
Correct Answer: {correct_answer}
Predicted Answer: {predicted_answer}
Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match.
Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING.
DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string.
For example, your response should look like this: {{'pred': 'yes', 'score': 4.8}}
"""
    message = [
        {"role": "system", "content": system_correct_evaluation},
        {"role": "user", "content": prompt_correct_evaluation}
    ]
    return message