agents/level_assess.py

from .base_agent import BaseAgent
import os
import openai
from dotenv import load_dotenv
import json

class LevelAssessmentAgent(BaseAgent):
    def __init__(self):
        super().__init__("LevelAssessmentAgent")
        load_dotenv()
        api_key = os.getenv("OPENAI_API_KEY")
        self.client = openai.OpenAI(api_key=api_key)

    def generate_questions(self, topic, level):
        system_prompt = (
        "You are a Master Educator and Curriculum Designer with deep expertise in pedagogical theory and assessment creation. "
        "Your primary goal is to design a short, insightful assessment to precisely gauge a student's mastery of a topic at a specific proficiency level.\n\n"
        "**Your Guiding Principles:**\n\n"
        "1. **Cognitive Depth:** The questions must go beyond rote memorization. They should probe the student's reasoning, problem-solving abilities, and their ability to connect concepts.\n"
        "2. **Level-Specific Targeting:** You must strictly adhere to the specified 'level':\n"
        "    - **Beginner:** Focus on core concepts, definitions, and simple \"how\" or \"why\" explanations. (Testing Comprehension & Application)\n"
        "    - **Intermediate:** Focus on comparing/contrasting concepts, applying knowledge to simple scenarios, and analyzing processes. (Testing Application & Analysis)\n"
        "    - **Advanced:** Demand synthesis of multiple concepts, evaluation of complex scenarios, and creation of novel solutions or arguments. (Testing Synthesis & Evaluation)\n"
        "3. **Clarity and Precision:** Each question must be unambiguous, concise, and clearly worded.\n\n"
        "**Output Format:**\n"
        "You must generate exactly 5 to 6 questions. Your response should contain ONLY the numbered list of questions. Do not include a title, introduction, conclusion, or any other text."
    )
        
        user_prompt = (
            f"Generate a assessment for 5 questions to test if a student is truly at the '{level}' level in the topic '{topic}'. "
            "The question should be open-ended and require a thoughtful answer."
        )
        response  = self.client.chat.completions.create(model = "gpt-4o-mini", 
                                                      messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]
        )
        return response.choices[0].message.content.strip()

    def evaluate_answer(self, topic, level, question, user_answer):
        system_prompt = (
    "You are a meticulous and impartial Grader AI. Your task is to evaluate a student's answer based on a provided question and the student's claimed proficiency level. You must adhere to the following STRICT rubric:\n\n"
    "1.  **Precision Requirement:** The answer must be specific, detailed, and demonstrate clear understanding. Vague, generic, or one-word answers are automatically INCORRECT.\n"
    "2.  **Correctness:** Is the answer factually accurate and relevant to the question?\n"
    "3.  **Depth:** Does the answer's depth match the expected level?\n"
    "    - **Novice/Beginner:** The answer should demonstrate basic comprehension and recall of key concepts with specific examples.\n"
    "    - **Intermediate:** The answer should show an ability to apply concepts, compare, and analyze with clear reasoning.\n"
    "    - **Advanced:** The answer must demonstrate synthesis, evaluation, and nuanced understanding with sophisticated insights.\n"
    "4.  **Completeness:** The answer must address all parts of the question comprehensively.\n\n"
    "**AUTOMATIC FAILURES:**\n"
    "- Answers under 20 words are automatically incorrect\n"
    "- Vague responses like 'yes', 'no', 'it depends', 'I think so', 'maybe', 'probably'\n"
    "- Responses that don't directly address the specific question asked\n"
    "- Copy-paste definitions without personal understanding or application\n\n"
    "You must respond ONLY with a JSON object. Do not include any other text or markdown formatting. The JSON object must have three keys:\n"
    "1.  `\"evaluation\"`: a string with a value of either `\"correct\"` or `\"incorrect\"`.\n"
    "2.  `\"reasoning\"`: a detailed explanation for your decision, including what was missing or incorrect.\n"
    "3.  `\"hint\"`: a subtle hint to guide the student toward the correct answer (only if incorrect)."
)
        user_prompt = (
    f"Evaluate the following data based on the rubric. Provide your response in the required JSON format.\n\n"
    f"**Topic:** \"{topic}\"\n"
    f"**Proficiency Level to Evaluate:** \"{level}\"\n"
    f"**Question:** \"{question}\"\n"
    f"**Student's Answer:** \"{user_answer}\""
)
        response = self.client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]
        )
        return response.choices[0].message.content.strip()

    def challenge_feedback(self, topic, level, question, user_answer, original_evaluation, original_reasoning):
        """
        Allows a student to challenge the feedback given. Re-evaluates the answer
        with a focus on being fair and reconsidering the evaluation.
        Returns a JSON string with the challenge result.
        """
        system_prompt = (
            "You are a fair and reconsidering Grader AI. A student has challenged your previous evaluation. "
            "Your task is to carefully re-examine the student's answer with fresh eyes, considering that "
            "your initial evaluation may have been too strict or may have missed valid points.\n\n"
            "**Re-evaluation Guidelines:**\n"
            "1. Be open to reconsidering your initial assessment\n"
            "2. Look for valid points in the answer that may have been overlooked\n"
            "3. Consider alternative interpretations of the question\n"
            "4. If the answer demonstrates understanding (even if not perfect), acknowledge it\n"
            "5. Be fair and balanced in your re-evaluation\n\n"
            "You must respond ONLY with a JSON object containing:\n"
            "1. `\"evaluation\"`: either `\"correct\"` or `\"incorrect\"`\n"
            "2. `\"reasoning\"`: detailed explanation of your re-evaluation, including whether you changed your mind and why\n"
            "3. `\"original_was_fair\"`: true/false indicating if the original evaluation was fair\n"
            "4. `\"hint\"`: a hint if still incorrect"
        )
        
        user_prompt = (
            f"**Topic:** {topic}\n"
            f"**Proficiency Level:** {level}\n"
            f"**Question:** {question}\n"
            f"**Student's Answer:** {user_answer}\n\n"
            f"**Original Evaluation:** {original_evaluation}\n"
            f"**Original Reasoning:** {original_reasoning}\n\n"
            "Please re-evaluate this answer fairly. The student believes the original evaluation may have been incorrect."
        )
        
        response = self.client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]
        )
        return response.choices[0].message.content.strip()

    def challenge_discussion(self, topic, level, question, user_answer, original_evaluation, original_reasoning, conversation_history, student_message):
        """
        Handles a conversational challenge discussion between student and grader.
        conversation_history: list of [user_message, assistant_message] pairs
        student_message: the current student's argument/question
        Returns: assistant response and final evaluation (if discussion is ending)
        """
        system_prompt = (
            "You are a fair and patient Grader AI engaged in a discussion with a student who is challenging your evaluation. "
            "You should be open to reconsidering your assessment, but also maintain academic standards.\n\n"
            "**Your Role:**\n"
            "1. Listen carefully to the student's arguments\n"
            "2. Be willing to reconsider if the student makes valid points\n"
            "3. Explain your reasoning clearly and respectfully\n"
            "4. If the student is right, acknowledge it and update your evaluation\n"
            "5. If the student's argument doesn't change your assessment, explain why clearly\n\n"
            "**Context:**\n"
            f"- Topic: {topic}\n"
            f"- Level: {level}\n"
            f"- Question: {question}\n"
            f"- Student's Original Answer: {user_answer}\n"
            f"- Original Evaluation: {original_evaluation}\n"
            f"- Original Reasoning: {original_reasoning}\n\n"
            "Respond naturally in a conversational manner. Be helpful and educational, not defensive."
        )
        
        messages = [{"role": "system", "content": system_prompt}]
        
        # Add conversation history
        for user_msg, assistant_msg in conversation_history:
            messages.append({"role": "user", "content": user_msg})
            messages.append({"role": "assistant", "content": assistant_msg})
        
        # Add current student message
        messages.append({"role": "user", "content": student_message})
        
        response = self.client.chat.completions.create(
            model="gpt-4o-mini",
            messages=messages
        )
        return response.choices[0].message.content.strip()

    def process(self, topic, level):
        print(f"\n[Assessment] Let's test your knowledge for the '{level}' level in '{topic}'.")
        print("I will ask you 5 questions.")
        
        NUM_QUESTIONS = 5
        # New scoring thresholds based on your requirements
        ADVANCED_PASS = 0.70  # 70% for advanced level
        INTERMEDIATE_PASS = 0.65  # 65% for intermediate level
        MINIMUM_THRESHOLD = 0.30  # 30% minimum to avoid novice

        all_questions_str = self.generate_questions(topic, level)
        questions = [q.strip() for q in all_questions_str.split('\n') if q.strip()]
        questions = questions[:NUM_QUESTIONS]

        correct_answers = 0
        per_question_feedback = []  # collect feedback for summary
        
        for i, question_text in enumerate(questions):
            current_question = ". ".join(question_text.split('. ')[1:])
            
            print(f"\n----------\n[Question {i+1}/{len(questions)}] {current_question}")
            user_answer = input("Your answer: ")
            try:
                evaluation_json_str = self.evaluate_answer(topic, level, current_question, user_answer)
                evaluation_data = json.loads(evaluation_json_str)

                evaluation = str(evaluation_data.get("evaluation", "incorrect")).lower()
                reasoning = evaluation_data.get("reasoning", "No explanation provided.")
                hint = evaluation_data.get("hint", "")

                is_correct = evaluation == "correct"
                if is_correct:
                    correct_answers += 1
                    print(f"[Feedback] {reasoning}")
                    print("✅ Correct!")
                else:
                    print(f"[Feedback] {reasoning}")
                    print("❌ Incorrect.")
                    if hint:
                        print(f"💡 Hint: {hint}")

                per_question_feedback.append({
                    "question": current_question,
                    "correct": is_correct,
                    "reason": reasoning,
                    "hint": hint,
                })

            except (json.JSONDecodeError, AttributeError) as e:
                print(f"[System Error] Could not parse the evaluation. Let's skip this one. Error: {e}")
                continue

        score_percentage = (correct_answers / len(questions)) * 100
        # Print detailed feedback summary
        print("\n----------")
        print("ASSESSMENT FEEDBACK SUMMARY")
        for i, fb in enumerate(per_question_feedback, 1):
            tag = "✅" if fb["correct"] else "❌"
            print(f"{tag} Q{i}: {fb['question']}")
            if fb.get("reason"):
                print(f"   Reason: {fb['reason']}")
            if (not fb["correct"]) and fb.get("hint"):
                print(f"   Hint: {fb['hint']}")

        print(f"\n[Assessment Complete] You scored {score_percentage:.1f}% ({correct_answers}/{len(questions)} questions correct).")

        # Level determination based on claimed level thresholds
        if level == "advanced":
            if score_percentage >= 70:
                print("🎯 Assigned Level: advanced")
                return "advanced"
            elif score_percentage >= 65:
                print("🎯 Assigned Level: intermediate")
                return "intermediate"
            else:
                print("🎯 Assigned Level: novice")
                return "novice"
        elif level == "intermediate":
            if score_percentage >= 65:
                print("🎯 Assigned Level: intermediate")
                return "intermediate"
            else:
                print("🎯 Assigned Level: novice")
                return "novice"
        else:
            print("🎯 Assigned Level: novice")
            return "novice"