evaluation_vdc/utils.py

def get_qa_generation_prompt(ground_truth_caption):
    system_qa_generation = """
You are an intelligent chatbot designed for generating 10 question-answer pairs given a detailed
description of a video or image. You are describing the video.
Here's how you can accomplish the task: INSTRUCTIONS:
- Focus on two aspects: background, and camera shots.
- The questions should be open-ended and start with What, Who, Where, When, Why, How, etc.
- The answer should be a short sentence or phrase.
- Generate exactly 5 question-answer pairs in total.
"""
    prompt_qa_generation = f"""
Please generate 5 question-answer pairs about the background, and the camera shots given a detailed description of a video or image: 
detailed description: {ground_truth_caption}

Please generate the response in the form of a Python list of tuple with the question and the corresponding answer.
DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. 
Only provide the Python list of tuple. 
For example:
[("the question 1", "the answer 1"), ("the question 2", "the answer 2"), ...]
"""
    message = [
        {"role": "system", "content": system_qa_generation},
        {"role": "user", "content": prompt_qa_generation}
    ]

    return message

# def get_qa_generation_prompt(ground_truth_caption):
#     system_qa_generation = """
# You are an intelligent chatbot designed for generating 20 question-answer pairs given a detailed
# description of a video or image. You are describing the video.
# Here's how you can accomplish the task: INSTRUCTIONS: - Cover the main objects and actions in the
# video or image.
# - The questions should be open-ended and start with What, Who, Where, When, Why, How, etc.
# - The answer should be a short sentence or phrase.
# - Generate 20 question-answer pairs.
# """
#     prompt_qa_generation = f"""
# Please generate 20 question-answer pairs given a detailed description of a video or image: detailed description: 
# {ground_truth_caption}
# Please generate the response in the form of a Python list of tuple with the question and the corresponding answer. DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python list of tuple. 
# For example, your response should look like this: [("the question 1", "the answer 1"), ("the question 2", "the answer 2"), ...].
# """
#     message = [
#         {"role": "system", "content": system_qa_generation},
#         {"role": "user", "content": prompt_qa_generation}
#     ]

#     return message


def extract_answer_from_prediction(question, predicted_caption):
    
    system_extract_answer = """
You are an intelligent chatbot designed for providing accurate answers to questions related to the
content based on a detailed description of a video or image.
Here's how you can accomplish the task:
——
##INSTRUCTIONS:
- Read the detailed description carefully.
- Answer the question only based on the detailed description.
- The answer should be a short sentence or phrase.
"""
    prompt_extract_answer = f"""
Please provide accurate answers to questions related to the content based on a detailed description of a video or image:
detailed description: {predicted_caption}
question: {question}
DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide short but accurate answer.
"""
    message = [
        {"role": "system", "content": system_extract_answer},
        {"role": "user", "content": prompt_extract_answer}
    ]
    return message


def correct_evaluation_prompt(question, correct_answer, predicted_answer):
#     prompt = f"""
# You are an intelligent chatbot designed to evaluate the accuracy and quality of predicted answers in a video-based question-answering task.

# ## INSTRUCTIONS:
# - Compare the **predicted answer** to the **correct answer** for a given question.
# - Focus on **meaningful correctness**:
#   - 1 = The predicted answer meaningfully matches the correct answer (even with paraphrasing or synonyms)
#   - 0 = The predicted answer does not match meaningfully
# - Also rate **quality (0 to 5):**
#   - Based on clarity, fluency, and coherence of the predicted answer

# ## OUTPUT:
# Return your evaluation as a Python dictionary string in the following format:
# {{'correctness': 0 or 1, 'quality': INT}}

# Do NOT include any other text or explanation — only return the dictionary string.

# Now evaluate the following:

# Question: {question}  
# Correct Answer: {correct_answer}  
# Predicted Answer: {predicted_answer}
# """

    system_correct_evaluation = """
You are an intelligent chatbot designed for evaluating the correctness of generative outputs for questionanswer pairs.
Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:
——
##INSTRUCTIONS:
- Focus on the meaningful match between the predicted answer and the correct answer.
- Consider synonyms or paraphrases as valid matches.
- Evaluate the correctness of the prediction compared to the answer
"""
    prompt_correct_evaluation = f"""
Please evaluate the following video-based question-answer pair:
Question: {question}
Correct Answer: {correct_answer}
Predicted Answer: {predicted_answer}
Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match.
Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING.
DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string.
For example, your response should look like this: {{'pred': 'yes', 'score': 4.8}}
"""
    message = [
        {"role": "system", "content": system_correct_evaluation},
        {"role": "user", "content": prompt_correct_evaluation}
    ]
    return message