quiz_generator/question_generation.py

from typing import List
from openai import OpenAI
from models import MultipleChoiceQuestion, MultipleChoiceOption, TEMPERATURE_UNAVAILABLE
from prompts.questions import (
    GENERAL_QUALITY_STANDARDS, MULTIPLE_CHOICE_STANDARDS,
    EXAMPLE_QUESTIONS, QUESTION_SPECIFIC_QUALITY_STANDARDS,
    CORRECT_ANSWER_SPECIFIC_QUALITY_STANDARDS,
    ANSWER_FEEDBACK_QUALITY_STANDARDS,
)
from prompts.incorrect_answers import (
    INCORRECT_ANSWER_EXAMPLES_WITH_EXPLANATION
)


def _get_run_manager():
    """Get run manager if available, otherwise return None."""
    try:
        from ui.run_manager import get_run_manager
        return get_run_manager()
    except:
        return None

def generate_multiple_choice_question(client: OpenAI,
    model: str,
    temperature: float,
    learning_objective: 'RankedLearningObjective',
    file_contents: List[str]) -> MultipleChoiceQuestion:
        """
        Generate a multiple choice question for a learning objective.
        
        Args:
            learning_objective: Learning objective to generate a question for
            file_contents: List of file contents with source tags
            
        Returns:
            Generated multiple choice question
        """
        run_manager = _get_run_manager()

        # Handle source references (could be string or list)
        source_references = learning_objective.source_reference
        if isinstance(source_references, str):
            source_references = [source_references]

        if run_manager:
            run_manager.log(f"Looking for content from source files: {source_references}", level="DEBUG")

        # Simply collect all content that matches any of the source references
        combined_content = ""
        for source_file in source_references:
            source_found = False
            for file_content in file_contents:
                # Look for the XML source tag with the matching filename
                if f"<source file='{source_file}'>" in file_content:
                    if run_manager:
                        run_manager.log(f"Found matching source content for {source_file}", level="DEBUG")
                    if combined_content:
                        combined_content += "\n\n"
                    combined_content += file_content
                    source_found = True
                    break

            # If no exact match found, try a more flexible match
            if not source_found:
                if run_manager:
                    run_manager.log(f"No exact match for {source_file}, looking for partial matches", level="DEBUG")
                for file_content in file_contents:
                    if source_file in file_content:
                        if run_manager:
                            run_manager.log(f"Found partial match for {source_file}", level="DEBUG")
                        if combined_content:
                            combined_content += "\n\n"
                        combined_content += file_content
                        source_found = True
                        break

        # If still no matching content, use all file contents combined
        if not combined_content:
            if run_manager:
                run_manager.log(f"No content found for any source files, using all content", level="DEBUG")
            combined_content = "\n\n".join(file_contents)
        
        # Add multi-source instruction if needed
        multi_source_instruction = ""
        if len(source_references) > 1:
            multi_source_instruction = """
        <IMPORTANT FOR MULTI-SOURCE QUESTIONS>
        This learning objective spans multiple sources. Your question should:
        1. Synthesize information across these sources
        2. Test understanding of overarching themes or connections
        3. Require knowledge from multiple sources to answer correctly
        </IMPORTANT FOR MULTI-SOURCE QUESTIONS>
        """
        
        # Create the prompt
        prompt = f"""
        Create a multiple choice question based on the following learning objective:
        
        <LEARNING OBJECTIVE>
        {learning_objective.learning_objective}
        </LEARNING OBJECTIVE>

        The correct answer to this is 
        
        <CORRECT ANSWER>
        {learning_objective.correct_answer}
        </CORRECT ANSWER>

        Follow these important instructions for writing the quiz question:
        
        <INSTRUCTIONS>
        <General Quality Standards>   
        {GENERAL_QUALITY_STANDARDS}
        </General Quality Standards>

        <Multiple Choice Specific Standards>
        {MULTIPLE_CHOICE_STANDARDS}
        </Multiple Choice Specific Standards>

        <Example Questions>
        {EXAMPLE_QUESTIONS}
        </Example Questions>

        <Question Specific Quality Standards>
        {QUESTION_SPECIFIC_QUALITY_STANDARDS}
        </Question Specific Quality Standards>

        <Correct Answer Specific Quality Standards>
        {CORRECT_ANSWER_SPECIFIC_QUALITY_STANDARDS}
        </Correct Answer Specific Quality Standards>

        These are the incorrect answer options:

        <INCORRECT_ANSWER_OPTIONS>
        {learning_objective.incorrect_answer_options}
        </INCORRECT_ANSWER_OPTIONS>

        Incorrect answers should follow the following examples with explanations:

        Here are some examples of high quality incorrect answer options for each learning objective:
        <incorrect_answer_examples>
        {INCORRECT_ANSWER_EXAMPLES_WITH_EXPLANATION}
        </incorrect_answer_examples>

        IMPORTANT:
        AVOID ABSOLUTE TERMS AND UNNECESSARY COMPARISONS
        Don't use words like "always," "never,", "mainly", "exclusively", "primarily" or "rather than". 
        These words are absolute or extreme qualifiers and comparative terms that artificially limit or overgeneralize statements, creating false dichotomies or unsubstantiated hierarchies.
        More words you should avoid are: All, every, entire, complete, none, nothing, no one, only, solely, merely, completely, totally, utterly, always, forever, constantly, never, impossible, must, mandatory, required, instead of, as opposed to, exclusively, purely



        <Answer Feedback Quality Standards>
        {ANSWER_FEEDBACK_QUALITY_STANDARDS}
        </Answer Feedback Quality Standards>

        </INSTRUCTIONS>
        
        {multi_source_instruction}
        
        Below the course content that the quiz question is based on:

        <COURSE CONTENT>
        {combined_content}
        </COURSE CONTENT>
        """
        
        # Generate question using instructor
        try:

            params = {
                "model": model,
                "messages": [
                    {"role": "system", "content": "You are an expert educational assessment creator specializing in creating high-quality multiple choice questions with detailed feedback for each option."},
                    {"role": "user", "content": prompt}
                ],
                "response_format": MultipleChoiceQuestion
            }
            if not TEMPERATURE_UNAVAILABLE.get(model, True):
                params["temperature"] = temperature

            completion = client.beta.chat.completions.parse(**params)
            response = completion.choices[0].message.parsed

            # Set learning objective ID and source reference
            response.id = learning_objective.id
            response.learning_objective_id = learning_objective.id
            response.learning_objective = learning_objective.learning_objective
            response.source_reference = learning_objective.source_reference

            # Verify all options have feedback
            for i, option in enumerate(response.options):
                if not option.feedback or option.feedback.strip() == "":
                    if option.is_correct:
                        option.feedback = "Good job! This is the correct answer."
                    else:
                        option.feedback = f"This answer is incorrect. Please review the material again."
            return response
            
        except Exception as e:
            print(f"Error generating question: {e}")
            # Create a fallback question
            options = [
                MultipleChoiceOption(
                    option_text=f"Option {chr(65+i)}", 
                    is_correct=(i==0), 
                    feedback=f"{'Correct' if i==0 else 'Incorrect'} answer."
                ) for i in range(4)
            ]
            return MultipleChoiceQuestion(
                id=learning_objective.id,
                question_text=f"Question for learning objective: {learning_objective.learning_objective}",
                options=options,
                learning_objective_id=learning_objective.id,
                learning_objective=learning_objective.learning_objective,
                source_reference=learning_objective.source_reference,
            )