ComputerAssignments/CA6_systematic_generalization/integrations/systematic_generalization_gemini.py

"""
پیاده‌سازی Systematic Generalization با Gemini API
Systematic Generalization Implementation with Gemini API
"""

import os
import logging
import json
import random
from typing import List, Dict, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from enum import Enum
import numpy as np
from datetime import datetime
import asyncio
import aiohttp

# Local imports
from gemini_config import GeminiConfig, GeminiSystematicGeneralization, GeminiRAGSystem
from advanced_rag_system import AdvancedRAGSystem, RAGConfig

logger = logging.getLogger(__name__)


class TaskType(Enum):
    """انواع وظایف Systematic Generalization"""

    ARITHMETIC_COMPOSITION = "arithmetic_composition"
    LANGUAGE_COMPOSITION = "language_composition"
    VISUAL_REASONING = "visual_reasoning"
    PROGRAM_SYNTHESIS = "program_synthesis"
    LOGICAL_REASONING = "logical_reasoning"
    SPATIAL_REASONING = "spatial_reasoning"


class GeneralizationType(Enum):
    """انواع تعمیم سیستماتیک"""

    COMPOSITIONAL = "compositional"  # ترکیب اجزای جدید
    LENGTH = "length"  # تعمیم به طول‌های بیشتر
    SYSTEMATIC = "systematic"  # تعمیم سیستماتیک
    STRUCTURAL = "structural"  # تعمیم ساختاری


@dataclass
class SystematicTask:
    """وظیفه Systematic Generalization"""

    task_id: str
    task_type: TaskType
    generalization_type: GeneralizationType
    training_examples: List[Dict[str, Any]]
    test_examples: List[Dict[str, Any]]
    description: str
    difficulty_level: int = 1  # 1-5
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class EvaluationResult:
    """نتیجه ارزیابی"""

    task_id: str
    accuracy: float
    systematic_score: float
    compositional_score: float
    length_generalization_score: float
    detailed_results: List[Dict[str, Any]]
    error_analysis: List[str]
    timestamp: str = field(default_factory=lambda: datetime.now().isoformat())


class SystematicGeneralizationEvaluator:
    """ارزیاب Systematic Generalization"""

    def __init__(self, gemini_config: GeminiConfig):
        self.gemini_config = gemini_config
        self.gemini_system = GeminiSystematicGeneralization(gemini_config)
        self.evaluation_history = []

    def create_arithmetic_composition_task(
        self, number_range: Tuple[int, int] = (1, 20), complexity_level: int = 2
    ) -> SystematicTask:
        """ایجاد وظیفه ترکیب ریاضی"""

        training_examples = []
        test_examples = []

        # تولید مثال‌های آموزشی
        for i in range(number_range[0], min(number_range[1], 11)):
            for j in range(number_range[0], min(number_range[1], 11)):
                for op in ["+", "-", "*"]:
                    if op == "+" and i + j <= 30:
                        expr = f"{i} {op} {j}"
                        result = i + j
                        training_examples.append(
                            {"input": expr, "output": str(result), "type": "arithmetic"}
                        )
                    elif op == "-" and i >= j:
                        expr = f"{i} {op} {j}"
                        result = i - j
                        training_examples.append(
                            {"input": expr, "output": str(result), "type": "arithmetic"}
                        )
                    elif op == "*" and i * j <= 50:
                        expr = f"{i} {op} {j}"
                        result = i * j
                        training_examples.append(
                            {"input": expr, "output": str(result), "type": "arithmetic"}
                        )

        # تولید مثال‌های تست (ترکیبات جدید)
        for i in range(11, min(number_range[1] + 1, 21)):
            for j in range(11, min(number_range[1] + 1, 21)):
                for op in ["+", "-", "*"]:
                    if op == "+" and i + j <= 50:
                        expr = f"{i} {op} {j}"
                        result = i + j
                        test_examples.append(
                            {
                                "input": expr,
                                "output": str(result),
                                "type": "arithmetic",
                                "expected": True,
                            }
                        )

        return SystematicTask(
            task_id=f"arithmetic_comp_{complexity_level}",
            task_type=TaskType.ARITHMETIC_COMPOSITION,
            generalization_type=GeneralizationType.COMPOSITIONAL,
            training_examples=training_examples,
            test_examples=test_examples,
            description="ترکیب ریاضی: یادگیری عملیات ریاضی و تعمیم به اعداد جدید",
            difficulty_level=complexity_level,
        )

    def create_language_composition_task(self) -> SystematicTask:
        """ایجاد وظیفه ترکیب زبانی"""

        # مثال‌های آموزشی
        training_examples = [
            {"input": "walk left", "output": "TURN_LEFT WALK", "type": "navigation"},
            {"input": "run right", "output": "TURN_RIGHT RUN", "type": "navigation"},
            {"input": "jump twice", "output": "JUMP JUMP", "type": "repetition"},
            {
                "input": "look around",
                "output": "TURN_LEFT TURN_RIGHT LOOK",
                "type": "direction",
            },
            {
                "input": "walk left twice",
                "output": "TURN_LEFT WALK TURN_LEFT WALK",
                "type": "combined",
            },
        ]

        # مثال‌های تست (ترکیبات جدید)
        test_examples = [
            {
                "input": "run left thrice",
                "output": "TURN_LEFT RUN TURN_LEFT RUN TURN_LEFT RUN",
                "type": "combined",
            },
            {
                "input": "jump around",
                "output": "TURN_LEFT JUMP TURN_RIGHT JUMP",
                "type": "combined",
            },
            {
                "input": "look right twice",
                "output": "TURN_RIGHT LOOK TURN_RIGHT LOOK",
                "type": "combined",
            },
            {
                "input": "walk around twice",
                "output": "TURN_LEFT WALK TURN_RIGHT WALK TURN_LEFT WALK TURN_RIGHT WALK",
                "type": "combined",
            },
        ]

        return SystematicTask(
            task_id="language_comp_1",
            task_type=TaskType.LANGUAGE_COMPOSITION,
            generalization_type=GeneralizationType.COMPOSITIONAL,
            training_examples=training_examples,
            test_examples=test_examples,
            description="ترکیب زبانی: یادگیری دستورات ناوبری و تعمیم به ترکیبات جدید",
            difficulty_level=2,
        )

    def create_visual_reasoning_task(self) -> SystematicTask:
        """ایجاد وظیفه استدلال بصری"""

        # مثال‌های آموزشی
        training_examples = [
            {
                "input": "red circle left of blue square",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "green triangle above yellow circle",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "blue square right of red circle",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "yellow circle below green triangle",
                "output": "true",
                "type": "spatial",
            },
        ]

        # مثال‌های تست (ترکیبات جدید)
        test_examples = [
            {
                "input": "purple diamond left of orange rectangle",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "cyan star above magenta hexagon",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "brown oval right of pink triangle",
                "output": "true",
                "type": "spatial",
            },
            {
                "input": "gray pentagon below lime circle",
                "output": "true",
                "type": "spatial",
            },
        ]

        return SystematicTask(
            task_id="visual_reasoning_1",
            task_type=TaskType.VISUAL_REASONING,
            generalization_type=GeneralizationType.COMPOSITIONAL,
            training_examples=training_examples,
            test_examples=test_examples,
            description="استدلال بصری: یادگیری روابط فضایی و تعمیم به اشکال و رنگ‌های جدید",
            difficulty_level=3,
        )

    def create_program_synthesis_task(self) -> SystematicTask:
        """ایجاد وظیفه سنتز برنامه"""

        # مثال‌های آموزشی
        training_examples = [
            {"input": "double(3)", "output": "6", "type": "function"},
            {"input": "square(4)", "output": "16", "type": "function"},
            {"input": "add(2, 3)", "output": "5", "type": "function"},
            {"input": "multiply(3, 4)", "output": "12", "type": "function"},
        ]

        # مثال‌های تست (ترکیبات جدید)
        test_examples = [
            {"input": "double(square(2))", "output": "8", "type": "composition"},
            {
                "input": "add(square(3), multiply(2, 4))",
                "output": "17",
                "type": "composition",
            },
            {"input": "square(add(2, 3))", "output": "25", "type": "composition"},
            {
                "input": "multiply(double(3), square(2))",
                "output": "24",
                "type": "composition",
            },
        ]

        return SystematicTask(
            task_id="program_synthesis_1",
            task_type=TaskType.PROGRAM_SYNTHESIS,
            generalization_type=GeneralizationType.COMPOSITIONAL,
            training_examples=training_examples,
            test_examples=test_examples,
            description="سنتز برنامه: یادگیری توابع و تعمیم به ترکیبات پیچیده",
            difficulty_level=4,
        )

    def evaluate_task(self, task: SystematicTask) -> EvaluationResult:
        """ارزیابی یک وظیفه Systematic Generalization"""

        logger.info(f"🔍 شروع ارزیابی وظیفه: {task.task_id}")

        detailed_results = []
        correct_predictions = 0
        systematic_scores = []
        compositional_scores = []
        length_scores = []
        error_analysis = []

        # ارزیابی هر مثال تست
        for i, test_example in enumerate(task.test_examples):
            try:
                # تولید پاسخ با Gemini
                response = self.gemini_system.generate_systematic_response(
                    prompt=test_example["input"], examples=task.training_examples
                )

                predicted_output = response["response"].strip()
                expected_output = test_example["output"]

                # ارزیابی صحت پاسخ
                is_correct = self._evaluate_prediction(
                    predicted_output, expected_output, task.task_type
                )

                if is_correct:
                    correct_predictions += 1

                # محاسبه امتیازات مختلف
                systematic_score = self._calculate_systematic_score(
                    predicted_output, expected_output, task.task_type
                )
                systematic_scores.append(systematic_score)

                compositional_score = self._calculate_compositional_score(
                    predicted_output, expected_output, task.task_type
                )
                compositional_scores.append(compositional_score)

                length_score = self._calculate_length_score(
                    predicted_output, expected_output, task.task_type
                )
                length_scores.append(length_score)

                detailed_results.append(
                    {
                        "test_index": i,
                        "input": test_example["input"],
                        "expected_output": expected_output,
                        "predicted_output": predicted_output,
                        "is_correct": is_correct,
                        "systematic_score": systematic_score,
                        "compositional_score": compositional_score,
                        "length_score": length_score,
                        "response_success": response["success"],
                    }
                )

                # تحلیل خطاها
                if not is_correct:
                    error_analysis.append(
                        f"خطا در تست {i}: ورودی '{test_example['input']}' - "
                        f"انتظار '{expected_output}' - دریافت '{predicted_output}'"
                    )

            except Exception as e:
                logger.error(f"❌ خطا در ارزیابی تست {i}: {e}")
                error_analysis.append(f"خطای سیستم در تست {i}: {str(e)}")
                detailed_results.append(
                    {
                        "test_index": i,
                        "input": test_example["input"],
                        "error": str(e),
                        "is_correct": False,
                    }
                )

        # محاسبه امتیازات کلی
        accuracy = (
            correct_predictions / len(task.test_examples) if task.test_examples else 0
        )
        systematic_score_avg = np.mean(systematic_scores) if systematic_scores else 0
        compositional_score_avg = (
            np.mean(compositional_scores) if compositional_scores else 0
        )
        length_score_avg = np.mean(length_scores) if length_scores else 0

        result = EvaluationResult(
            task_id=task.task_id,
            accuracy=accuracy,
            systematic_score=systematic_score_avg,
            compositional_score=compositional_score_avg,
            length_generalization_score=length_score_avg,
            detailed_results=detailed_results,
            error_analysis=error_analysis,
        )

        self.evaluation_history.append(result)
        logger.info(f"✅ ارزیابی تکمیل شد - دقت: {accuracy:.2%}")

        return result

    def _evaluate_prediction(
        self, predicted: str, expected: str, task_type: TaskType
    ) -> bool:
        """ارزیابی پیش‌بینی بر اساس نوع وظیفه"""

        predicted_clean = predicted.lower().strip()
        expected_clean = expected.lower().strip()

        if task_type == TaskType.ARITHMETIC_COMPOSITION:
            # برای ریاضی، بررسی عدد صحیح
            try:
                pred_num = float(predicted_clean)
                exp_num = float(expected_clean)
                return abs(pred_num - exp_num) < 0.01
            except:
                return expected_clean in predicted_clean

        elif task_type == TaskType.LANGUAGE_COMPOSITION:
            # برای زبان، بررسی وجود کلمات کلیدی
            expected_words = expected_clean.split()
            predicted_words = predicted_clean.split()
            return all(word in predicted_words for word in expected_words)

        elif task_type == TaskType.VISUAL_REASONING:
            # برای بصری، بررسی پاسخ بولی
            return expected_clean in predicted_clean

        else:
            # پیش‌فرض: بررسی تطبیق جزئی
            return (
                expected_clean in predicted_clean or predicted_clean in expected_clean
            )

    def _calculate_systematic_score(
        self, predicted: str, expected: str, task_type: TaskType
    ) -> float:
        """محاسبه امتیاز سیستماتیک بودن"""
        # امتیاز بر اساس تطبیق ساختاری
        if self._evaluate_prediction(predicted, expected, task_type):
            return 1.0

        # امتیاز جزئی بر اساس شباهت ساختاری
        predicted_words = predicted.lower().split()
        expected_words = expected.lower().split()

        if not predicted_words or not expected_words:
            return 0.0

        # محاسبه شباهت کلمات
        common_words = set(predicted_words) & set(expected_words)
        similarity = len(common_words) / max(len(predicted_words), len(expected_words))

        return similarity

    def _calculate_compositional_score(
        self, predicted: str, expected: str, task_type: TaskType
    ) -> float:
        """محاسبه امتیاز ترکیب‌پذیری"""
        # بررسی وجود اجزای ترکیبی در پاسخ
        predicted_clean = predicted.lower().strip()
        expected_clean = expected.lower().strip()

        # شمارش اجزای مشترک
        predicted_parts = predicted_clean.split()
        expected_parts = expected_clean.split()

        if not predicted_parts or not expected_parts:
            return 0.0

        common_parts = set(predicted_parts) & set(expected_parts)
        compositional_score = len(common_parts) / len(expected_parts)

        return compositional_score

    def _calculate_length_score(
        self, predicted: str, expected: str, task_type: TaskType
    ) -> float:
        """محاسبه امتیاز تعمیم طول"""
        predicted_length = len(predicted.split())
        expected_length = len(expected.split())

        if expected_length == 0:
            return 0.0

        # امتیاز بر اساس نسبت طول
        length_ratio = min(predicted_length, expected_length) / max(
            predicted_length, expected_length
        )

        return length_ratio

    def run_comprehensive_evaluation(self) -> Dict[str, Any]:
        """اجرای ارزیابی جامع"""

        logger.info("🚀 شروع ارزیابی جامع Systematic Generalization")

        # ایجاد وظایف مختلف
        tasks = [
            self.create_arithmetic_composition_task(),
            self.create_language_composition_task(),
            self.create_visual_reasoning_task(),
            self.create_program_synthesis_task(),
        ]

        results = {
            "evaluation_timestamp": datetime.now().isoformat(),
            "total_tasks": len(tasks),
            "task_results": {},
            "overall_scores": {},
            "summary": {},
        }

        # ارزیابی هر وظیفه
        for task in tasks:
            logger.info(f"📋 ارزیابی وظیفه: {task.task_id}")
            task_result = self.evaluate_task(task)
            results["task_results"][task.task_id] = {
                "task_type": task.task_type.value,
                "generalization_type": task.generalization_type.value,
                "difficulty_level": task.difficulty_level,
                "accuracy": task_result.accuracy,
                "systematic_score": task_result.systematic_score,
                "compositional_score": task_result.compositional_score,
                "length_generalization_score": task_result.length_generalization_score,
                "total_test_cases": len(task.test_examples),
                "correct_predictions": sum(
                    1
                    for r in task_result.detailed_results
                    if r.get("is_correct", False)
                ),
            }

        # محاسبه امتیازات کلی
        all_accuracies = [r["accuracy"] for r in results["task_results"].values()]
        all_systematic_scores = [
            r["systematic_score"] for r in results["task_results"].values()
        ]
        all_compositional_scores = [
            r["compositional_score"] for r in results["task_results"].values()
        ]
        all_length_scores = [
            r["length_generalization_score"] for r in results["task_results"].values()
        ]

        results["overall_scores"] = {
            "average_accuracy": np.mean(all_accuracies),
            "average_systematic_score": np.mean(all_systematic_scores),
            "average_compositional_score": np.mean(all_compositional_scores),
            "average_length_generalization_score": np.mean(all_length_scores),
            "overall_systematic_generalization_score": np.mean(
                [
                    np.mean(all_accuracies),
                    np.mean(all_systematic_scores),
                    np.mean(all_compositional_scores),
                    np.mean(all_length_scores),
                ]
            ),
        }

        # خلاصه نتایج
        results["summary"] = {
            "best_performing_task": max(
                results["task_results"].items(), key=lambda x: x[1]["accuracy"]
            ),
            "worst_performing_task": min(
                results["task_results"].items(), key=lambda x: x[1]["accuracy"]
            ),
            "most_systematic_task": max(
                results["task_results"].items(), key=lambda x: x[1]["systematic_score"]
            ),
            "most_compositional_task": max(
                results["task_results"].items(),
                key=lambda x: x[1]["compositional_score"],
            ),
            "total_evaluations": len(self.evaluation_history),
        }

        logger.info("✅ ارزیابی جامع تکمیل شد")
        return results

    def save_evaluation_results(self, results: Dict[str, Any], filepath: str):
        """ذخیره نتایج ارزیابی"""
        try:
            with open(filepath, "w", encoding="utf-8") as f:
                json.dump(results, f, ensure_ascii=False, indent=2)
            logger.info(f"✅ نتایج ارزیابی ذخیره شد: {filepath}")
        except Exception as e:
            logger.error(f"❌ خطا در ذخیره نتایج: {e}")

    def load_evaluation_results(self, filepath: str) -> Dict[str, Any]:
        """بارگذاری نتایج ارزیابی"""
        try:
            with open(filepath, "r", encoding="utf-8") as f:
                results = json.load(f)
            logger.info(f"✅ نتایج ارزیابی بارگذاری شد: {filepath}")
            return results
        except Exception as e:
            logger.error(f"❌ خطا در بارگذاری نتایج: {e}")
            return {}


class SystematicGeneralizationBenchmark:
    """بنچمارک Systematic Generalization"""

    def __init__(self, gemini_config: GeminiConfig):
        self.gemini_config = gemini_config
        self.evaluator = SystematicGeneralizationEvaluator(gemini_config)
        self.benchmark_results = {}

    def run_benchmark_suite(self) -> Dict[str, Any]:
        """اجرای مجموعه بنچمارک"""

        logger.info("🏁 شروع بنچمارک Systematic Generalization")

        benchmark_results = {
            "benchmark_timestamp": datetime.now().isoformat(),
            "gemini_config": {
                "model_name": self.gemini_config.model_name,
                "temperature": self.gemini_config.temperature,
            },
            "evaluation_results": self.evaluator.run_comprehensive_evaluation(),
            "benchmark_metadata": {
                "total_tasks": 4,
                "evaluation_method": "comprehensive_systematic_generalization",
                "scoring_methods": [
                    "accuracy",
                    "systematic_score",
                    "compositional_score",
                    "length_score",
                ],
            },
        }

        self.benchmark_results = benchmark_results
        return benchmark_results

    def compare_with_baseline(self, baseline_results: Dict[str, Any]) -> Dict[str, Any]:
        """مقایسه با نتایج پایه"""

        current_results = self.benchmark_results.get("evaluation_results", {})
        current_overall = current_results.get("overall_scores", {})
        baseline_overall = baseline_results.get("overall_scores", {})

        comparison = {
            "accuracy_improvement": current_overall.get("average_accuracy", 0)
            - baseline_overall.get("average_accuracy", 0),
            "systematic_score_improvement": current_overall.get(
                "average_systematic_score", 0
            )
            - baseline_overall.get("average_systematic_score", 0),
            "compositional_score_improvement": current_overall.get(
                "average_compositional_score", 0
            )
            - baseline_overall.get("average_compositional_score", 0),
            "overall_improvement": current_overall.get(
                "overall_systematic_generalization_score", 0
            )
            - baseline_overall.get("overall_systematic_generalization_score", 0),
        }

        return comparison


# مثال استفاده
if __name__ == "__main__":
    # تنظیم پیکربندی Gemini
    gemini_config = GeminiConfig(
        api_key="your-gemini-api-key-here", model_name="gemini-1.5-pro", temperature=0.7
    )

    # ایجاد ارزیاب
    evaluator = SystematicGeneralizationEvaluator(gemini_config)

    # اجرای ارزیابی جامع
    results = evaluator.run_comprehensive_evaluation()

    # نمایش نتایج
    print("📊 نتایج ارزیابی Systematic Generalization:")
    print(f"دقت کلی: {results['overall_scores']['average_accuracy']:.2%}")
    print(
        f"امتیاز سیستماتیک: {results['overall_scores']['average_systematic_score']:.2%}"
    )
    print(
        f"امتیاز ترکیب‌پذیری: {results['overall_scores']['average_compositional_score']:.2%}"
    )
    print(
        f"امتیاز تعمیم طول: {results['overall_scores']['average_length_generalization_score']:.2%}"
    )

    # ذخیره نتایج
    evaluator.save_evaluation_results(results, "systematic_generalization_results.json")