init

notebooks/llm_benchmark.ipynb

@@ -53,7 +53,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Libraries loaded successfully\n"
+      "\u2705 Libraries loaded successfully\n"
      ]
     }
    ],
@@ -78,19 +78,59 @@
     "sns.set_style('whitegrid')\n",
     "plt.rcParams['figure.figsize'] = (14, 8)\n",
     "\n",
-    "print(\"✅ Libraries loaded successfully\")"
+    "print(\"\u2705 Libraries loaded successfully\")"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": 3,
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u2705 Project root: /Users/ismatsamadov/SOCAR_Hackathon\n",
+      "\u2705 Docs directory: /Users/ismatsamadov/SOCAR_Hackathon/docs\n",
+      "\u2705 Output directory: /Users/ismatsamadov/SOCAR_Hackathon/output\n"
+     ]
+    }
+   ],
    "source": [
-    "## 1. Load Test Questions and Expected Answers"
+    "# Auto-detect project root (works from any directory)\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if Path('data').exists() and Path('docs').exists():\n",
+    "    # Already in project root\n",
+    "    PROJECT_ROOT = Path.cwd()\n",
+    "elif Path('../data').exists() and Path('../docs').exists():\n",
+    "    # In notebooks/ subdirectory\n",
+    "    PROJECT_ROOT = Path.cwd().parent\n",
+    "else:\n",
+    "    # Fallback: try to find project root\n",
+    "    current = Path.cwd()\n",
+    "    while current != current.parent:\n",
+    "        if (current / 'data').exists() and (current / 'docs').exists():\n",
+    "            PROJECT_ROOT = current\n",
+    "            break\n",
+    "        current = current.parent\n",
+    "    else:\n",
+    "        PROJECT_ROOT = Path.cwd()\n",
+    "\n",
+    "# Define all paths relative to project root\n",
+    "DATA_DIR = PROJECT_ROOT / 'data'\n",
+    "DOCS_DIR = PROJECT_ROOT / 'docs'\n",
+    "OUTPUT_DIR = PROJECT_ROOT / 'output'\n",
+    "\n",
+    "print(f\"\u2705 Project root: {PROJECT_ROOT}\")\n",
+    "print(f\"\u2705 Docs directory: {DOCS_DIR}\")\n",
+    "print(f\"\u2705 Output directory: {OUTPUT_DIR}\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -100,21 +140,21 @@
       "Loaded 5 test cases\n",
       "\n",
       "Test Questions:\n",
-      "1. Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasiyası tələb olunur?...\n",
-      "2. Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq edilmişdir və bunun m...\n",
-      "3. Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO oksidləri arasında ha...\n",
-      "4. Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrinə əsasən neft və qaz...\n",
-      "5. Example5: Bu zonada hansı proseslər baş verir?...\n"
+      "1. Example1: Daha az quyu il\u0259 daha \u00e7ox hasilat \u0259ld\u0259 etm\u0259k \u00fc\u00e7\u00fcn hans\u0131 \u0259sas amill\u0259rin inteqrasiyas\u0131 t\u0259l\u0259b olunur?...\n",
+      "2. Example2: Q\u0259rbi Ab\u015feron yata\u011f\u0131nda suvurma t\u0259dbirl\u0259ri hans\u0131 tarixd\u0259 v\u0259 hans\u0131 layda t\u0259tbiq edilmi\u015fdir v\u0259 bunun m...\n",
+      "3. Example3: Pirallah\u0131 strukturunda 1253 n\u00f6mr\u0259li quyudan g\u00f6t\u00fcr\u00fclm\u00fc\u015f n\u00fcmun\u0259l\u0259rd\u0259 SiO2 v\u0259 CaO oksidl\u0259ri aras\u0131nda ha...\n",
+      "4. Example4: Bak\u0131 arxipelaq\u0131 (BA) v\u0259 A\u015fa\u011f\u0131 K\u00fcr \u00e7\u00f6k\u0259kliyi (AK\u00c7) \u00fc\u00e7\u00fcn geotemperatur x\u0259rit\u0259l\u0259rin\u0259 \u0259sas\u0259n neft v\u0259 qaz...\n",
+      "5. Example5: Bu zonada hans\u0131 prosesl\u0259r ba\u015f verir?...\n"
      ]
     }
    ],
    "source": [
-    "# Load sample questions\n",
-    "with open('docs/sample_questions.json', 'r', encoding='utf-8') as f:\n",
+    "# Load sample questions - using dynamic paths\n",
+    "with open(DOCS_DIR / 'sample_questions.json', 'r', encoding='utf-8') as f:\n",
     "    questions = json.load(f)\n",
     "\n",
-    "# Load expected answers\n",
-    "with open('docs/sample_answers.json', 'r', encoding='utf-8') as f:\n",
+    "# Load expected answers - using dynamic paths\n",
+    "with open(DOCS_DIR / 'sample_answers.json', 'r', encoding='utf-8') as f:\n",
     "    expected_answers = json.load(f)\n",
     "\n",
     "print(f\"Loaded {len(questions)} test cases\")\n",
@@ -133,38 +173,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Vector DB connected: {'_response_info': {'raw_headers': {'connection': 'keep-alive',\n",
-      "                                    'content-length': '188',\n",
-      "                                    'content-type': 'application/json',\n",
-      "                                    'date': 'Sun, 14 Dec 2025 03:21:33 GMT',\n",
-      "                                    'grpc-status': '0',\n",
-      "                                    'server': 'envoy',\n",
-      "                                    'x-envoy-upstream-service-time': '4',\n",
-      "                                    'x-pinecone-request-id': '3979707437017514155',\n",
-      "                                    'x-pinecone-request-latency-ms': '4'}},\n",
-      " 'dimension': 1024,\n",
-      " 'index_fullness': 0.0,\n",
-      " 'memoryFullness': 0.0,\n",
-      " 'metric': 'cosine',\n",
-      " 'namespaces': {'__default__': {'vector_count': 1300}},\n",
-      " 'storageFullness': 0.0,\n",
-      " 'total_vector_count': 1300,\n",
-      " 'vector_type': 'dense'}\n",
-      "✅ Embedding model loaded: SentenceTransformer(\n",
-      "  (0): Transformer({'max_seq_length': 512, 'do_lower_case': True, 'architecture': 'BertModel'})\n",
-      "  (1): Pooling({'word_embedding_dimension': 1024, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})\n",
-      "  (2): Normalize()\n",
-      ")\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Initialize Pinecone\n",
     "pc = Pinecone(api_key=os.getenv('PINECONE_API_KEY'))\n",
@@ -173,8 +184,8 @@
     "# Initialize embedding model (same as used for ingestion)\n",
     "embed_model = SentenceTransformer('BAAI/bge-large-en-v1.5')\n",
     "\n",
-    "print(f\"✅ Vector DB connected: {index.describe_index_stats()}\")\n",
-    "print(f\"✅ Embedding model loaded: {embed_model}\")"
+    "print(f\"\u2705 Vector DB connected: {index.describe_index_stats()}\")\n",
+    "print(f\"\u2705 Embedding model loaded: {embed_model}\")"
    ]
   },
   {
@@ -186,19 +197,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "✅ Retrieved 3 documents for test query\n",
-      "Top result: document_10.pdf, page 8 (score: 0.767)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "def retrieve_documents(query: str, top_k: int = 3) -> List[Dict]:\n",
     "    \"\"\"\n",
@@ -227,9 +228,9 @@
     "    return documents\n",
     "\n",
     "# Test retrieval\n",
-    "test_query = \"Palçıq vulkanlarının təsir radiusu nə qədərdir?\"\n",
+    "test_query = \"Pal\u00e7\u0131q vulkanlar\u0131n\u0131n t\u0259sir radiusu n\u0259 q\u0259d\u0259rdir?\"\n",
     "test_docs = retrieve_documents(test_query)\n",
-    "print(f\"\\n✅ Retrieved {len(test_docs)} documents for test query\")\n",
+    "print(f\"\\n\u2705 Retrieved {len(test_docs)} documents for test query\")\n",
     "print(f\"Top result: {test_docs[0]['pdf_name']}, page {test_docs[0]['page_number']} (score: {test_docs[0]['score']:.3f})\")"
    ]
   },
@@ -245,7 +246,83 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "# Initialize Azure OpenAI\nazure_client = AzureOpenAI(\n    api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n    api_version=os.getenv('AZURE_OPENAI_API_VERSION', '2024-08-01-preview'),\n    azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT')\n)\n\nLLM_MODELS = {\n    'Llama-4-Maverick': 'Llama-4-Maverick-17B-128E-Instruct-FP8',\n    'DeepSeek-R1': 'DeepSeek-R1',\n    'GPT-4.1': 'gpt-4.1',\n    'GPT-5-mini': 'gpt-5-mini',\n    'Claude-Sonnet-4.5': 'claude-sonnet-4-5',\n}\n\ndef generate_answer(model_name: str, query: str, documents: List[Dict], \n                   temperature: float = 0.2, max_tokens: int = 1000) -> Tuple[str, float]:\n    \"\"\"\n    Generate answer using specified LLM model.\n    Returns: (answer, response_time)\n    \"\"\"\n    # Build context from retrieved documents\n    context_parts = []\n    for i, doc in enumerate(documents, 1):\n        context_parts.append(\n            f\"Document {i} (Source: {doc['pdf_name']}, Page {doc['page_number']}):\\n{doc['content']}\"\n        )\n    context = \"\\n\\n\".join(context_parts)\n    \n    # Create prompt\n    prompt = f\"\"\"Siz SOCAR-ın tarixi neft və qaz sənədləri üzrə mütəxəssis köməkçisisiniz.\n\nKontekst (əlaqəli sənədlər):\n{context}\n\nSual: {query}\n\nƏtraflı cavab verin və mütləq sənəd mənbələrinə istinad edin (PDF adı və səhifə nömrəsi ilə).\nCavabınız dəqiq, faktlara əsaslanan və kontekst məlumatlarından istifadə edən olmalıdır.\"\"\"\n    \n    # Get model deployment\n    deployment = MODELS[model_name]['deployment']\n    \n    try:\n        start_time = time.time()\n        \n        # GPT-5 models use max_completion_tokens, others use max_tokens\n        if deployment.startswith('gpt-5'):\n            response = azure_client.chat.completions.create(\n                model=deployment,\n                messages=[\n                    {\"role\": \"user\", \"content\": prompt}\n                ],\n                temperature=temperature,\n                max_completion_tokens=max_tokens\n            )\n        else:\n            response = azure_client.chat.completions.create(\n                model=deployment,\n                messages=[\n                    {\"role\": \"user\", \"content\": prompt}\n                ],\n                temperature=temperature,\n                max_tokens=max_tokens\n            )\n        \n        response_time = time.time() - start_time\n        answer = response.choices[0].message.content\n        \n        return answer, response_time\n    \n    except Exception as e:\n        return f\"ERROR: {str(e)}\", 0.0\n\nprint(f\"\\n✅ Configured {len(LLM_MODELS)} LLM models for testing\")"
+   "source": [
+    "# Initialize Azure OpenAI\n",
+    "azure_client = AzureOpenAI(\n",
+    "    api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
+    "    api_version=os.getenv('AZURE_OPENAI_API_VERSION', '2024-08-01-preview'),\n",
+    "    azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT')\n",
+    ")\n",
+    "\n",
+    "LLM_MODELS = {\n",
+    "    'Llama-4-Maverick': 'Llama-4-Maverick-17B-128E-Instruct-FP8',\n",
+    "    'DeepSeek-R1': 'DeepSeek-R1',\n",
+    "    'GPT-4.1': 'gpt-4.1',\n",
+    "    'GPT-5-mini': 'gpt-5-mini',\n",
+    "    'Claude-Sonnet-4.5': 'claude-sonnet-4-5',\n",
+    "}\n",
+    "\n",
+    "def generate_answer(model_name: str, query: str, documents: List[Dict], \n",
+    "                   temperature: float = 0.2, max_tokens: int = 1000) -> Tuple[str, float]:\n",
+    "    \"\"\"\n",
+    "    Generate answer using specified LLM model.\n",
+    "    Returns: (answer, response_time)\n",
+    "    \"\"\"\n",
+    "    # Build context from retrieved documents\n",
+    "    context_parts = []\n",
+    "    for i, doc in enumerate(documents, 1):\n",
+    "        context_parts.append(\n",
+    "            f\"Document {i} (Source: {doc['pdf_name']}, Page {doc['page_number']}):\\n{doc['content']}\"\n",
+    "        )\n",
+    "    context = \"\\n\\n\".join(context_parts)\n",
+    "    \n",
+    "    # Create prompt\n",
+    "    prompt = f\"\"\"Siz SOCAR-\u0131n tarixi neft v\u0259 qaz s\u0259n\u0259dl\u0259ri \u00fczr\u0259 m\u00fct\u0259x\u0259ssis k\u00f6m\u0259k\u00e7isisiniz.\n",
+    "\n",
+    "Kontekst (\u0259laq\u0259li s\u0259n\u0259dl\u0259r):\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "\u018ftrafl\u0131 cavab verin v\u0259 m\u00fctl\u0259q s\u0259n\u0259d m\u0259nb\u0259l\u0259rin\u0259 istinad edin (PDF ad\u0131 v\u0259 s\u0259hif\u0259 n\u00f6mr\u0259si il\u0259).\n",
+    "Cavab\u0131n\u0131z d\u0259qiq, faktlara \u0259saslanan v\u0259 kontekst m\u0259lumatlar\u0131ndan istifad\u0259 ed\u0259n olmal\u0131d\u0131r.\"\"\"\n",
+    "    \n",
+    "    # Get model deployment\n",
+    "    deployment = MODELS[model_name]['deployment']\n",
+    "    \n",
+    "    try:\n",
+    "        start_time = time.time()\n",
+    "        \n",
+    "        # GPT-5 models use max_completion_tokens, others use max_tokens\n",
+    "        if deployment.startswith('gpt-5'):\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[\n",
+    "                    {\"role\": \"user\", \"content\": prompt}\n",
+    "                ],\n",
+    "                temperature=temperature,\n",
+    "                max_completion_tokens=max_tokens\n",
+    "            )\n",
+    "        else:\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[\n",
+    "                    {\"role\": \"user\", \"content\": prompt}\n",
+    "                ],\n",
+    "                temperature=temperature,\n",
+    "                max_tokens=max_tokens\n",
+    "            )\n",
+    "        \n",
+    "        response_time = time.time() - start_time\n",
+    "        answer = response.choices[0].message.content\n",
+    "        \n",
+    "        return answer, response_time\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        return f\"ERROR: {str(e)}\", 0.0\n",
+    "\n",
+    "print(f\"\\n\u2705 Configured {len(LLM_MODELS)} LLM models for testing\")"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -256,17 +333,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Evaluation functions ready\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "def normalize_text(text: str) -> str:\n",
     "    \"\"\"Normalize text for comparison.\"\"\"\n",
@@ -313,7 +382,7 @@
     "    cited_pages = sum(1 for page in page_numbers if page in answer)\n",
     "    \n",
     "    # Check for source keywords\n",
-    "    source_keywords = ['mənbə', 'sənəd', 'səhifə', 'pdf', 'document', 'page', 'source']\n",
+    "    source_keywords = ['m\u0259nb\u0259', 's\u0259n\u0259d', 's\u0259hif\u0259', 'pdf', 'document', 'page', 'source']\n",
     "    has_source_ref = any(kw in answer.lower() for kw in source_keywords)\n",
     "    \n",
     "    citation_score = (\n",
@@ -350,7 +419,7 @@
     "        'Char_Count': char_count\n",
     "    }\n",
     "\n",
-    "print(\"✅ Evaluation functions ready\")"
+    "print(\"\u2705 Evaluation functions ready\")"
    ]
   },
   {
@@ -362,20 +431,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Testing 5 models on 5 questions...\n",
-      "\n",
-      "This may take several minutes...\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Select models to test (you can comment out models to skip)\n",
     "MODELS_TO_TEST = [\n",
@@ -395,133 +453,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "================================================================================\n",
-      "Testing: Llama-4-Maverick-17B\n",
-      "================================================================================\n",
-      "\n",
-      "  Question Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasi...\n",
-      "  ✅ Response time: 4.39s\n",
-      "\n",
-      "  Question Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq e...\n",
-      "  ✅ Response time: 3.74s\n",
-      "\n",
-      "  Question Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO o...\n",
-      "  ✅ Response time: 4.07s\n",
-      "\n",
-      "  Question Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrin...\n",
-      "  ✅ Response time: 4.20s\n",
-      "\n",
-      "  Question Example5: Bu zonada hansı proseslər baş verir?...\n",
-      "  ✅ Response time: 3.50s\n",
-      "\n",
-      "  📊 Llama-4-Maverick-17B Summary:\n",
-      "     Avg Response Time: 3.98s\n",
-      "     Avg Similarity: 0.0%\n",
-      "     Avg Citation Score: 84.0%\n",
-      "     Avg Completeness: 100.0%\n",
-      "\n",
-      "================================================================================\n",
-      "Testing: DeepSeek-R1\n",
-      "================================================================================\n",
-      "\n",
-      "  Question Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasi...\n",
-      "  ✅ Response time: 10.00s\n",
-      "\n",
-      "  Question Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq e...\n",
-      "  ✅ Response time: 10.39s\n",
-      "\n",
-      "  Question Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO o...\n",
-      "  ✅ Response time: 10.73s\n",
-      "\n",
-      "  Question Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrin...\n",
-      "  ✅ Response time: 12.17s\n",
-      "\n",
-      "  Question Example5: Bu zonada hansı proseslər baş verir?...\n",
-      "  ✅ Response time: 10.72s\n",
-      "\n",
-      "  📊 DeepSeek-R1 Summary:\n",
-      "     Avg Response Time: 10.80s\n",
-      "     Avg Similarity: 0.0%\n",
-      "     Avg Citation Score: 80.0%\n",
-      "     Avg Completeness: 67.7%\n",
-      "\n",
-      "================================================================================\n",
-      "Testing: GPT-4.1\n",
-      "================================================================================\n",
-      "\n",
-      "  Question Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasi...\n",
-      "  ✅ Response time: 6.66s\n",
-      "\n",
-      "  Question Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq e...\n",
-      "  ✅ Response time: 5.05s\n",
-      "\n",
-      "  Question Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO o...\n",
-      "  ✅ Response time: 7.65s\n",
-      "\n",
-      "  Question Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrin...\n",
-      "  ✅ Response time: 6.68s\n",
-      "\n",
-      "  Question Example5: Bu zonada hansı proseslər baş verir?...\n",
-      "  ✅ Response time: 3.69s\n",
-      "\n",
-      "  📊 GPT-4.1 Summary:\n",
-      "     Avg Response Time: 5.95s\n",
-      "     Avg Similarity: 0.0%\n",
-      "     Avg Citation Score: 84.0%\n",
-      "     Avg Completeness: 93.5%\n",
-      "\n",
-      "================================================================================\n",
-      "Testing: GPT-5-mini\n",
-      "================================================================================\n",
-      "\n",
-      "  Question Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasi...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.\", 'type': 'invalid_request_error', 'param': 'max_tokens', 'code': 'unsupported_parameter'}}\n",
-      "\n",
-      "  Question Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq e...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.\", 'type': 'invalid_request_error', 'param': 'max_tokens', 'code': 'unsupported_parameter'}}\n",
-      "\n",
-      "  Question Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO o...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.\", 'type': 'invalid_request_error', 'param': 'max_tokens', 'code': 'unsupported_parameter'}}\n",
-      "\n",
-      "  Question Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrin...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.\", 'type': 'invalid_request_error', 'param': 'max_tokens', 'code': 'unsupported_parameter'}}\n",
-      "\n",
-      "  Question Example5: Bu zonada hansı proseslər baş verir?...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.\", 'type': 'invalid_request_error', 'param': 'max_tokens', 'code': 'unsupported_parameter'}}\n",
-      "\n",
-      "================================================================================\n",
-      "Testing: Claude-Sonnet-4.5\n",
-      "================================================================================\n",
-      "\n",
-      "  Question Example1: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasi...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
-      "\n",
-      "  Question Example2: Qərbi Abşeron yatağında suvurma tədbirləri hansı tarixdə və hansı layda tətbiq e...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
-      "\n",
-      "  Question Example3: Pirallahı strukturunda 1253 nömrəli quyudan götürülmüş nümunələrdə SiO2 və CaO o...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
-      "\n",
-      "  Question Example4: Bakı arxipelaqı (BA) və Aşağı Kür çökəkliyi (AKÇ) üçün geotemperatur xəritələrin...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
-      "\n",
-      "  Question Example5: Bu zonada hansı proseslər baş verir?...\n",
-      "  ❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
-      "\n",
-      "================================================================================\n",
-      "✅ Benchmarking complete!\n",
-      "================================================================================\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Run benchmark\n",
     "results = []\n",
@@ -547,10 +481,10 @@
     "        answer, response_time = generate_answer(model_name, query, documents)\n",
     "        \n",
     "        if answer.startswith('ERROR'):\n",
-    "            print(f\"  ❌ Failed: {answer}\")\n",
+    "            print(f\"  \u274c Failed: {answer}\")\n",
     "            continue\n",
     "        \n",
-    "        print(f\"  ✅ Response time: {response_time:.2f}s\")\n",
+    "        print(f\"  \u2705 Response time: {response_time:.2f}s\")\n",
     "        \n",
     "        # Get expected answer\n",
     "        expected = expected_answers.get(example_key, {}).get('Answer', '')\n",
@@ -584,14 +518,14 @@
     "        avg_citation = sum(r['Citation_Score'] for r in model_results) / len(model_results)\n",
     "        avg_completeness = sum(r['Completeness_Score'] for r in model_results) / len(model_results)\n",
     "        \n",
-    "        print(f\"\\n  📊 {model_name} Summary:\")\n",
+    "        print(f\"\\n  \ud83d\udcca {model_name} Summary:\")\n",
     "        print(f\"     Avg Response Time: {avg_response_time:.2f}s\")\n",
     "        print(f\"     Avg Similarity: {avg_similarity:.1f}%\")\n",
     "        print(f\"     Avg Citation Score: {avg_citation:.1f}%\")\n",
     "        print(f\"     Avg Completeness: {avg_completeness:.1f}%\")\n",
     "\n",
     "print(f\"\\n{'='*80}\")\n",
-    "print(\"✅ Benchmarking complete!\")\n",
+    "print(\"\u2705 Benchmarking complete!\")\n",
     "print(f\"{'='*80}\")"
    ]
   },
@@ -604,26 +538,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "====================================================================================================\n",
-      "📊 LLM BENCHMARKING RESULTS - MODEL SUMMARY\n",
-      "====================================================================================================\n",
-      "                      Response_Time  Similarity  Citation_Score  Completeness_Score     CER     WER  Open_Source Architecture_Score  Quality_Score\n",
-      "Model                                                                                                                                             \n",
-      "Llama-4-Maverick-17B           3.98         0.0            84.0              100.00  330.97  378.42         True               High          59.40\n",
-      "GPT-4.1                        5.95         0.0            84.0               93.54  755.19  780.64        False             Medium          57.46\n",
-      "DeepSeek-R1                   10.80         0.0            80.0               67.73  855.43  992.02         True               High          48.32\n",
-      "====================================================================================================\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Create DataFrame\n",
     "df = pd.DataFrame(results)\n",
@@ -652,17 +569,77 @@
     "\n",
     "# Display summary table\n",
     "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"📊 LLM BENCHMARKING RESULTS - MODEL SUMMARY\")\n",
+    "print(\"\ud83d\udcca LLM BENCHMARKING RESULTS - MODEL SUMMARY\")\n",
     "print(\"=\"*100)\n",
     "print(model_summary.to_string())\n",
     "print(\"=\"*100)"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
    "metadata": {},
    "source": [
-    "## 8. Visualizations"
+    "# Create comprehensive visualization\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Create output directory - using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'llm_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "fig, axes = plt.subplots(2, 2, figsize=(16, 10))\n",
+    "\n",
+    "models = df['Model'].tolist()\n",
+    "colors = sns.color_palette('viridis', len(models))\n",
+    "\n",
+    "# 1. CSR - Character Success Rate (MAIN METRIC)\n",
+    "ax1 = axes[0, 0]\n",
+    "bars1 = ax1.barh(models, df['CSR'], color=colors)\n",
+    "ax1.set_xlabel('CSR (%) - Higher is Better', fontsize=12, fontweight='bold')\n",
+    "ax1.set_title('Character Success Rate (CSR)\\n\ud83c\udfc6 HACKATHON PRIMARY METRIC', \n",
+    "              fontsize=14, fontweight='bold')\n",
+    "ax1.set_xlim(0, 100)\n",
+    "for i, (model, csr) in enumerate(zip(models, df['CSR'])):\n",
+    "    ax1.text(csr + 1, i, f'{csr:.2f}%', va='center', fontsize=11, fontweight='bold')\n",
+    "ax1.axvline(x=90, color='green', linestyle='--', alpha=0.3, label='Excellent (>90%)')\n",
+    "ax1.axvline(x=80, color='orange', linestyle='--', alpha=0.3, label='Good (>80%)')\n",
+    "ax1.legend(fontsize=9)\n",
+    "\n",
+    "# 2. WSR - Word Success Rate\n",
+    "ax2 = axes[0, 1]\n",
+    "bars2 = ax2.barh(models, df['WSR'], color=colors)\n",
+    "ax2.set_xlabel('WSR (%) - Higher is Better', fontsize=12, fontweight='bold')\n",
+    "ax2.set_title('Word Success Rate (WSR)', fontsize=14, fontweight='bold')\n",
+    "ax2.set_xlim(0, 100)\n",
+    "for i, (model, wsr) in enumerate(zip(models, df['WSR'])):\n",
+    "    ax2.text(wsr + 1, i, f'{wsr:.2f}%', va='center', fontsize=11, fontweight='bold')\n",
+    "\n",
+    "# 3. Response Time\n",
+    "ax3 = axes[1, 0]\n",
+    "bars3 = ax3.barh(models, df['Response_Time'], color=colors)\n",
+    "ax3.set_xlabel('Total Time (seconds) - Lower is Better', fontsize=12, fontweight='bold')\n",
+    "ax3.set_title('Processing Speed', fontsize=14, fontweight='bold')\n",
+    "for i, (model, time_val) in enumerate(zip(models, df['Response_Time'])):\n",
+    "    ax3.text(time_val + 0.5, i, f'{time_val:.1f}s', va='center', fontsize=11)\n",
+    "\n",
+    "# 4. Error Rates Comparison\n",
+    "ax4 = axes[1, 1]\n",
+    "x = range(len(models))\n",
+    "width = 0.35\n",
+    "ax4.bar([i - width/2 for i in x], df['CER'], width, label='CER', color='coral', alpha=0.8)\n",
+    "ax4.bar([i + width/2 for i in x], df['WER'], width, label='WER', color='skyblue', alpha=0.8)\n",
+    "ax4.set_ylabel('Error Rate (%) - Lower is Better', fontsize=12, fontweight='bold')\n",
+    "ax4.set_title('Error Rates', fontsize=14, fontweight='bold')\n",
+    "ax4.set_xticks(x)\n",
+    "ax4.set_xticklabels(models, rotation=45, ha='right')\n",
+    "ax4.legend(fontsize=11)\n",
+    "ax4.grid(axis='y', alpha=0.3)\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(output_dir / 'results.png', dpi=300, bbox_inches='tight')\n",
+    "plt.show()\n",
+    "\n",
+    "print(f\"\\n\u2705 Visualization saved to '{output_dir}/results.png'\")"
    ]
   },
   {
@@ -670,71 +647,6 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "# Create comprehensive visualization\nimport os\nfrom pathlib import Path\n\n# Create output directory\noutput_dir = Path('output/llm_benchmark')\noutput_dir.mkdir(parents=True, exist_ok=True)\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\n\nmodels = model_summary.index.tolist()\ncolors = sns.color_palette('husl', len(models))\n\n# 1. Overall Quality Score\nax1 = axes[0, 0]\nbars1 = ax1.barh(models, model_summary['Quality_Score'], color=colors)\nax1.set_xlabel('Quality Score (Higher is Better)', fontsize=11)\nax1.set_title('Overall Quality Score\\n(Similarity 35% + Citation 35% + Completeness 30%)', \n              fontsize=12, fontweight='bold')\nax1.set_xlim(0, 100)\nfor i, (model, score) in enumerate(zip(models, model_summary['Quality_Score'])):\n    ax1.text(score + 1, i, f'{score:.1f}', va='center', fontsize=10, fontweight='bold')\n\n# 2. Answer Similarity (Accuracy)\nax2 = axes[0, 1]\nax2.barh(models, model_summary['Similarity'], color=colors)\nax2.set_xlabel('Similarity to Expected Answer (%)', fontsize=11)\nax2.set_title('Answer Accuracy', fontsize=12, fontweight='bold')\nax2.set_xlim(0, 100)\nfor i, (model, score) in enumerate(zip(models, model_summary['Similarity'])):\n    ax2.text(score + 1, i, f'{score:.1f}%', va='center', fontsize=9)\n\n# 3. Citation Quality\nax3 = axes[0, 2]\nax3.barh(models, model_summary['Citation_Score'], color=colors)\nax3.set_xlabel('Citation Score (%)', fontsize=11)\nax3.set_title('Citation Quality\\n(PDF names + Page numbers)', fontsize=12, fontweight='bold')\nax3.set_xlim(0, 100)\nfor i, (model, score) in enumerate(zip(models, model_summary['Citation_Score'])):\n    ax3.text(score + 1, i, f'{score:.1f}%', va='center', fontsize=9)\n\n# 4. Response Time\nax4 = axes[1, 0]\nax4.barh(models, model_summary['Response_Time'], color=colors)\nax4.set_xlabel('Response Time (seconds - Lower is Better)', fontsize=11)\nax4.set_title('Speed Performance', fontsize=12, fontweight='bold')\nfor i, (model, time) in enumerate(zip(models, model_summary['Response_Time'])):\n    ax4.text(time + 0.1, i, f'{time:.2f}s', va='center', fontsize=9)\n\n# 5. Completeness\nax5 = axes[1, 1]\nax5.barh(models, model_summary['Completeness_Score'], color=colors)\nax5.set_xlabel('Completeness Score (%)', fontsize=11)\nax5.set_title('Answer Completeness', fontsize=12, fontweight='bold')\nax5.set_xlim(0, 100)\nfor i, (model, score) in enumerate(zip(models, model_summary['Completeness_Score'])):\n    ax5.text(score + 1, i, f'{score:.1f}%', va='center', fontsize=9)\n\n# 6. Error Rates (CER vs WER)\nax6 = axes[1, 2]\nx = range(len(models))\nwidth = 0.35\nax6.bar([i - width/2 for i in x], model_summary['CER'], width, label='CER', alpha=0.8)\nax6.bar([i + width/2 for i in x], model_summary['WER'], width, label='WER', alpha=0.8)\nax6.set_ylabel('Error Rate (% - Lower is Better)', fontsize=11)\nax6.set_title('Error Rates', fontsize=12, fontweight='bold')\nax6.set_xticks(x)\nax6.set_xticklabels(models, rotation=45, ha='right')\nax6.legend()\nax6.grid(axis='y', alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(output_dir / 'results.png', dpi=300, bbox_inches='tight')\nplt.show()\n\nprint(f\"\\n✅ Visualization saved to '{output_dir}/results.png'\")"
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 9. Final Rankings and Recommendations"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "====================================================================================================\n",
-      "🏆 FINAL RANKINGS\n",
-      "====================================================================================================\n",
-      "                      Rank  Quality_Score  Similarity  Citation_Score  Completeness_Score  Response_Time  Open_Source Architecture_Score\n",
-      "Model                                                                                                                                   \n",
-      "Llama-4-Maverick-17B     1          59.40         0.0            84.0              100.00           3.98         True               High\n",
-      "GPT-4.1                  2          57.46         0.0            84.0               93.54           5.95        False             Medium\n",
-      "DeepSeek-R1              3          48.32         0.0            80.0               67.73          10.80         True               High\n",
-      "====================================================================================================\n",
-      "\n",
-      "====================================================================================================\n",
-      "💡 RECOMMENDATIONS FOR HACKATHON\n",
-      "====================================================================================================\n",
-      "\n",
-      "🥇 Best Overall Quality: Llama-4-Maverick-17B\n",
-      "   Quality Score: 59.4%\n",
-      "   Similarity: 0.0%\n",
-      "   Citation Score: 84.0%\n",
-      "   Response Time: 3.98s\n",
-      "   Open Source: True\n",
-      "   Architecture Score: High\n",
-      "\n",
-      "🔓 Best Open-Source Model: Llama-4-Maverick-17B\n",
-      "   Quality Score: 59.4%\n",
-      "   Architecture Score: High (Better for hackathon!)\n",
-      "   Response Time: 3.98s\n",
-      "\n",
-      "⚡ Fastest Model: Llama-4-Maverick-17B\n",
-      "   Response Time: 3.98s\n",
-      "   Quality Score: 59.4%\n",
-      "\n",
-      "====================================================================================================\n",
-      "📝 FINAL RECOMMENDATION\n",
-      "====================================================================================================\n",
-      "\n",
-      "Scoring Breakdown:\n",
-      "  - LLM Quality: 30% of total hackathon score\n",
-      "  - Architecture: 20% of total hackathon score (open-source preferred!)\n",
-      "\n",
-      "Best Choice:\n",
-      "  ✅ Llama-4-Maverick-17B - Best balance of quality and architecture score\n",
-      "     Only 0.0% quality drop for higher architecture score!\n",
-      "====================================================================================================\n"
-     ]
-    }
-   ],
    "source": [
     "# Create rankings table\n",
     "rankings = model_summary[[\n",
@@ -745,7 +657,7 @@
     "rankings.insert(0, 'Rank', range(1, len(rankings) + 1))\n",
     "\n",
     "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"🏆 FINAL RANKINGS\")\n",
+    "print(\"\ud83c\udfc6 FINAL RANKINGS\")\n",
     "print(\"=\"*100)\n",
     "print(rankings.to_string())\n",
     "print(\"=\"*100)\n",
@@ -756,10 +668,10 @@
     "fastest = model_summary['Response_Time'].idxmin()\n",
     "\n",
     "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"💡 RECOMMENDATIONS FOR HACKATHON\")\n",
+    "print(\"\ud83d\udca1 RECOMMENDATIONS FOR HACKATHON\")\n",
     "print(\"=\"*100)\n",
     "\n",
-    "print(f\"\\n🥇 Best Overall Quality: {best_overall}\")\n",
+    "print(f\"\\n\ud83e\udd47 Best Overall Quality: {best_overall}\")\n",
     "print(f\"   Quality Score: {model_summary.loc[best_overall, 'Quality_Score']:.1f}%\")\n",
     "print(f\"   Similarity: {model_summary.loc[best_overall, 'Similarity']:.1f}%\")\n",
     "print(f\"   Citation Score: {model_summary.loc[best_overall, 'Citation_Score']:.1f}%\")\n",
@@ -768,113 +680,61 @@
     "print(f\"   Architecture Score: {model_summary.loc[best_overall, 'Architecture_Score']}\")\n",
     "\n",
     "if best_open_source:\n",
-    "    print(f\"\\n🔓 Best Open-Source Model: {best_open_source}\")\n",
+    "    print(f\"\\n\ud83d\udd13 Best Open-Source Model: {best_open_source}\")\n",
     "    print(f\"   Quality Score: {model_summary.loc[best_open_source, 'Quality_Score']:.1f}%\")\n",
     "    print(f\"   Architecture Score: {model_summary.loc[best_open_source, 'Architecture_Score']} (Better for hackathon!)\")\n",
     "    print(f\"   Response Time: {model_summary.loc[best_open_source, 'Response_Time']:.2f}s\")\n",
     "\n",
-    "print(f\"\\n⚡ Fastest Model: {fastest}\")\n",
+    "print(f\"\\n\u26a1 Fastest Model: {fastest}\")\n",
     "print(f\"   Response Time: {model_summary.loc[fastest, 'Response_Time']:.2f}s\")\n",
     "print(f\"   Quality Score: {model_summary.loc[fastest, 'Quality_Score']:.1f}%\")\n",
     "\n",
     "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"📝 FINAL RECOMMENDATION\")\n",
+    "print(\"\ud83d\udcdd FINAL RECOMMENDATION\")\n",
     "print(\"=\"*100)\n",
     "print(\"\\nScoring Breakdown:\")\n",
     "print(\"  - LLM Quality: 30% of total hackathon score\")\n",
     "print(\"  - Architecture: 20% of total hackathon score (open-source preferred!)\")\n",
     "print(\"\\nBest Choice:\")\n",
     "if best_open_source and model_summary.loc[best_open_source, 'Quality_Score'] >= model_summary.loc[best_overall, 'Quality_Score'] * 0.9:\n",
-    "    print(f\"  ✅ {best_open_source} - Best balance of quality and architecture score\")\n",
+    "    print(f\"  \u2705 {best_open_source} - Best balance of quality and architecture score\")\n",
     "    print(f\"     Only {model_summary.loc[best_overall, 'Quality_Score'] - model_summary.loc[best_open_source, 'Quality_Score']:.1f}% quality drop for higher architecture score!\")\n",
     "else:\n",
-    "    print(f\"  ✅ {best_overall} - Highest quality, use if quality gap is significant\")\n",
+    "    print(f\"  \u2705 {best_overall} - Highest quality, use if quality gap is significant\")\n",
     "    if best_open_source:\n",
-    "        print(f\"  ⚠️  Consider {best_open_source} for higher architecture score (trade-off: {model_summary.loc[best_overall, 'Quality_Score'] - model_summary.loc[best_open_source, 'Quality_Score']:.1f}% quality)\")\n",
+    "        print(f\"  \u26a0\ufe0f  Consider {best_open_source} for higher architecture score (trade-off: {model_summary.loc[best_overall, 'Quality_Score'] - model_summary.loc[best_open_source, 'Quality_Score']:.1f}% quality)\")\n",
     "\n",
     "print(\"=\"*100)"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
    "metadata": {},
    "source": [
-    "## 10. Export Results"
+    "# Save results\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'llm_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "df.to_csv(output_dir / 'detailed_results.csv', index=False, encoding='utf-8')\n",
+    "model_summary.to_csv(output_dir / 'summary.csv', encoding='utf-8')\n",
+    "rankings.to_csv(output_dir / 'rankings.csv', index=False, encoding='utf-8')\n",
+    "\n",
+    "print(\"\\n\u2705 Results exported to output/llm_benchmark/:\")\n",
+    "print(\"   - detailed_results.csv (all questions and answers)\")\n",
+    "print(\"   - summary.csv (model averages)\")\n",
+    "print(\"   - rankings.csv (final rankings)\")\n",
+    "print(\"   - results.png (visualizations)\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": "# Save results\nfrom pathlib import Path\n\noutput_dir = Path('output/llm_benchmark')\noutput_dir.mkdir(parents=True, exist_ok=True)\n\ndf.to_csv(output_dir / 'detailed_results.csv', index=False, encoding='utf-8')\nmodel_summary.to_csv(output_dir / 'summary.csv', encoding='utf-8')\nrankings.to_csv(output_dir / 'rankings.csv', index=False, encoding='utf-8')\n\nprint(\"\\n✅ Results exported to output/llm_benchmark/:\")\nprint(\"   - detailed_results.csv (all questions and answers)\")\nprint(\"   - summary.csv (model averages)\")\nprint(\"   - rankings.csv (final rankings)\")\nprint(\"   - results.png (visualizations)\")"
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## 11. Sample Answer Comparison"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "====================================================================================================\n",
-      "📝 SAMPLE ANSWER COMPARISON - Example1\n",
-      "====================================================================================================\n",
-      "\n",
-      "❓ Question: Daha az quyu ilə daha çox hasilat əldə etmək üçün hansı əsas amillərin inteqrasiyası tələb olunur?\n",
-      "\n",
-      "✅ Expected Answer:\n",
-      "Daha az quyu ilə daha çox hasilat əldə etmək üçün düzgün seçilmiş texnoloji inteqrasiya (horizontal və çoxtərəfli qazma texnikaları) və qazma məhlullarının səmərəli idarə edilməsi tələb olunur. Bu yanaşma həm iqtisadi, həm də ekoloji baxımdan üstünlük yaradır.\n",
-      "\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "\n",
-      "🤖 Llama-4-Maverick-17B (Quality: 59.4%, Time: 4.39s):\n",
-      "Daha az quyu ilə daha çox hasilat əldə etmək üçün düzgün seçilmiş texnoloji inteqrasiya və qazma məhlullarının səmərəli idarəsi əsas amillərdir. Bu, Document 1 (document_11.pdf, Səhifə 3)-də qeyd olun...\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "\n",
-      "🤖 DeepSeek-R1 (Quality: 48.3%, Time: 10.00s):\n",
-      "<think>\n",
-      "Okay, let's tackle this question. The user is asking about the main factors that need to be integrated to achieve more production with fewer wells. They provided three documents, so I need to ...\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "\n",
-      "🤖 GPT-4.1 (Quality: 57.5%, Time: 6.66s):\n",
-      "Daha az quyu ilə daha çox hasilat əldə etmək üçün bir neçə əsas amilin inteqrasiyası tələb olunur. Bu amillər aşağıdakı kimi sistematik şəkildə sənəd mənbələrinə istinadla izah olunur:\n",
-      "\n",
-      "1. **Düzgün se...\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "====================================================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Show sample answers for first question\n",
-    "sample_question = 'Example1'\n",
-    "sample_results = df[df['Question'] == sample_question]\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(f\"📝 SAMPLE ANSWER COMPARISON - {sample_question}\")\n",
-    "print(\"=\"*100)\n",
-    "\n",
-    "print(f\"\\n❓ Question: {questions[sample_question][0]['content']}\")\n",
-    "print(f\"\\n✅ Expected Answer:\\n{expected_answers[sample_question]['Answer']}\")\n",
-    "print(\"\\n\" + \"-\"*100)\n",
-    "\n",
-    "for _, row in sample_results.iterrows():\n",
-    "    print(f\"\\n🤖 {row['Model']} (Quality: {model_summary.loc[row['Model'], 'Quality_Score']:.1f}%, Time: {row['Response_Time']:.2f}s):\")\n",
-    "    print(f\"{row['Answer']}\")\n",
-    "    print(\"-\"*100)\n",
-    "\n",
-    "print(\"=\"*100)"
-   ]
   }
  ],
  "metadata": {

notebooks/llm_benchmark.ipynb.backup

@@ -0,0 +1,761 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LLM Benchmarking for SOCAR Hackathon RAG Chatbot\n",
+    "\n",
+    "This notebook tests different LLM models for the `/llm` endpoint to find the best performer.\n",
+    "\n",
+    "## Evaluation Criteria (LLM Judge Metrics):\n",
+    "- **Accuracy**: Is the answer correct?\n",
+    "- **Relevance**: Are retrieved citations relevant?\n",
+    "- **Completeness**: Does it fully answer the question?\n",
+    "- **Citation Quality**: Proper sources with page numbers?\n",
+    "- **Response Time**: Speed of generation\n",
+    "\n",
+    "## Available LLM Models:\n",
+    "1. **Llama-4-Maverick-17B-128E-Instruct-FP8** (Current choice, open-source)\n",
+    "2. **DeepSeek-R1** (Open-source reasoning model)\n",
+    "3. **GPT-4.1** (Strong general performance)\n",
+    "4. **GPT-5, GPT-5-mini**\n",
+    "5. **Claude Sonnet 4.5** (Best quality)\n",
+    "6. **Claude Opus 4.1**\n",
+    "7. **Phi-4-multimodal-instruct**\n",
+    "8. **gpt-oss-120b**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install required packages\n",
+    "# !pip install openai pinecone-client sentence-transformers python-dotenv pandas matplotlib seaborn jiwer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/ismatsamadov/SOCAR_Hackathon/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u2705 Libraries loaded successfully\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import time\n",
+    "from typing import Dict, List, Tuple\n",
+    "from dotenv import load_dotenv\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from openai import AzureOpenAI\n",
+    "from pinecone import Pinecone\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "from jiwer import wer, cer\n",
+    "\n",
+    "# Load environment variables\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Set style\n",
+    "sns.set_style('whitegrid')\n",
+    "plt.rcParams['figure.figsize'] = (14, 8)\n",
+    "\n",
+    "print(\"\u2705 Libraries loaded successfully\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u2705 Project root: /Users/ismatsamadov/SOCAR_Hackathon\n",
+      "\u2705 Docs directory: /Users/ismatsamadov/SOCAR_Hackathon/docs\n",
+      "\u2705 Output directory: /Users/ismatsamadov/SOCAR_Hackathon/output\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Auto-detect project root (works from any directory)\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if Path('data').exists() and Path('docs').exists():\n",
+    "    # Already in project root\n",
+    "    PROJECT_ROOT = Path.cwd()\n",
+    "elif Path('../data').exists() and Path('../docs').exists():\n",
+    "    # In notebooks/ subdirectory\n",
+    "    PROJECT_ROOT = Path.cwd().parent\n",
+    "else:\n",
+    "    # Fallback: try to find project root\n",
+    "    current = Path.cwd()\n",
+    "    while current != current.parent:\n",
+    "        if (current / 'data').exists() and (current / 'docs').exists():\n",
+    "            PROJECT_ROOT = current\n",
+    "            break\n",
+    "        current = current.parent\n",
+    "    else:\n",
+    "        PROJECT_ROOT = Path.cwd()\n",
+    "\n",
+    "# Define all paths relative to project root\n",
+    "DATA_DIR = PROJECT_ROOT / 'data'\n",
+    "DOCS_DIR = PROJECT_ROOT / 'docs'\n",
+    "OUTPUT_DIR = PROJECT_ROOT / 'output'\n",
+    "\n",
+    "print(f\"\u2705 Project root: {PROJECT_ROOT}\")\n",
+    "print(f\"\u2705 Docs directory: {DOCS_DIR}\")\n",
+    "print(f\"\u2705 Output directory: {OUTPUT_DIR}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded 5 test cases\n",
+      "\n",
+      "Test Questions:\n",
+      "1. Example1: Daha az quyu il\u0259 daha \u00e7ox hasilat \u0259ld\u0259 etm\u0259k \u00fc\u00e7\u00fcn hans\u0131 \u0259sas amill\u0259rin inteqrasiyas\u0131 t\u0259l\u0259b olunur?...\n",
+      "2. Example2: Q\u0259rbi Ab\u015feron yata\u011f\u0131nda suvurma t\u0259dbirl\u0259ri hans\u0131 tarixd\u0259 v\u0259 hans\u0131 layda t\u0259tbiq edilmi\u015fdir v\u0259 bunun m...\n",
+      "3. Example3: Pirallah\u0131 strukturunda 1253 n\u00f6mr\u0259li quyudan g\u00f6t\u00fcr\u00fclm\u00fc\u015f n\u00fcmun\u0259l\u0259rd\u0259 SiO2 v\u0259 CaO oksidl\u0259ri aras\u0131nda ha...\n",
+      "4. Example4: Bak\u0131 arxipelaq\u0131 (BA) v\u0259 A\u015fa\u011f\u0131 K\u00fcr \u00e7\u00f6k\u0259kliyi (AK\u00c7) \u00fc\u00e7\u00fcn geotemperatur x\u0259rit\u0259l\u0259rin\u0259 \u0259sas\u0259n neft v\u0259 qaz...\n",
+      "5. Example5: Bu zonada hans\u0131 prosesl\u0259r ba\u015f verir?...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load sample questions - using dynamic paths\n",
+    "with open(DOCS_DIR / 'sample_questions.json', 'r', encoding='utf-8') as f:\n",
+    "    questions = json.load(f)\n",
+    "\n",
+    "# Load expected answers - using dynamic paths\n",
+    "with open(DOCS_DIR / 'sample_answers.json', 'r', encoding='utf-8') as f:\n",
+    "    expected_answers = json.load(f)\n",
+    "\n",
+    "print(f\"Loaded {len(questions)} test cases\")\n",
+    "print(\"\\nTest Questions:\")\n",
+    "for i, (key, msgs) in enumerate(questions.items(), 1):\n",
+    "    user_msg = [m for m in msgs if m['role'] == 'user'][-1]\n",
+    "    print(f\"{i}. {key}: {user_msg['content'][:100]}...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Initialize Vector Database and Embedding Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize Pinecone\n",
+    "pc = Pinecone(api_key=os.getenv('PINECONE_API_KEY'))\n",
+    "index = pc.Index(os.getenv('PINECONE_INDEX_NAME', 'hackathon'))\n",
+    "\n",
+    "# Initialize embedding model (same as used for ingestion)\n",
+    "embed_model = SentenceTransformer('BAAI/bge-large-en-v1.5')\n",
+    "\n",
+    "print(f\"\u2705 Vector DB connected: {index.describe_index_stats()}\")\n",
+    "print(f\"\u2705 Embedding model loaded: {embed_model}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. RAG Retrieval Function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def retrieve_documents(query: str, top_k: int = 3) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    Retrieve relevant documents from vector database.\n",
+    "    \"\"\"\n",
+    "    # Generate query embedding\n",
+    "    query_embedding = embed_model.encode(query).tolist()\n",
+    "    \n",
+    "    # Search vector DB\n",
+    "    results = index.query(\n",
+    "        vector=query_embedding,\n",
+    "        top_k=top_k,\n",
+    "        include_metadata=True\n",
+    "    )\n",
+    "    \n",
+    "    # Extract documents\n",
+    "    documents = []\n",
+    "    for match in results['matches']:\n",
+    "        documents.append({\n",
+    "            'pdf_name': match['metadata'].get('pdf_name', 'unknown.pdf'),\n",
+    "            'page_number': match['metadata'].get('page_number', 0),\n",
+    "            'content': match['metadata'].get('text', ''),\n",
+    "            'score': match.get('score', 0.0)\n",
+    "        })\n",
+    "    \n",
+    "    return documents\n",
+    "\n",
+    "# Test retrieval\n",
+    "test_query = \"Pal\u00e7\u0131q vulkanlar\u0131n\u0131n t\u0259sir radiusu n\u0259 q\u0259d\u0259rdir?\"\n",
+    "test_docs = retrieve_documents(test_query)\n",
+    "print(f\"\\n\u2705 Retrieved {len(test_docs)} documents for test query\")\n",
+    "print(f\"Top result: {test_docs[0]['pdf_name']}, page {test_docs[0]['page_number']} (score: {test_docs[0]['score']:.3f})\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. LLM Client Functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize Azure OpenAI\n",
+    "azure_client = AzureOpenAI(\n",
+    "    api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
+    "    api_version=os.getenv('AZURE_OPENAI_API_VERSION', '2024-08-01-preview'),\n",
+    "    azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT')\n",
+    ")\n",
+    "\n",
+    "LLM_MODELS = {\n",
+    "    'Llama-4-Maverick': 'Llama-4-Maverick-17B-128E-Instruct-FP8',\n",
+    "    'DeepSeek-R1': 'DeepSeek-R1',\n",
+    "    'GPT-4.1': 'gpt-4.1',\n",
+    "    'GPT-5-mini': 'gpt-5-mini',\n",
+    "    'Claude-Sonnet-4.5': 'claude-sonnet-4-5',\n",
+    "}\n",
+    "\n",
+    "def generate_answer(model_name: str, query: str, documents: List[Dict], \n",
+    "                   temperature: float = 0.2, max_tokens: int = 1000) -> Tuple[str, float]:\n",
+    "    \"\"\"\n",
+    "    Generate answer using specified LLM model.\n",
+    "    Returns: (answer, response_time)\n",
+    "    \"\"\"\n",
+    "    # Build context from retrieved documents\n",
+    "    context_parts = []\n",
+    "    for i, doc in enumerate(documents, 1):\n",
+    "        context_parts.append(\n",
+    "            f\"Document {i} (Source: {doc['pdf_name']}, Page {doc['page_number']}):\\n{doc['content']}\"\n",
+    "        )\n",
+    "    context = \"\\n\\n\".join(context_parts)\n",
+    "    \n",
+    "    # Create prompt\n",
+    "    prompt = f\"\"\"Siz SOCAR-\u0131n tarixi neft v\u0259 qaz s\u0259n\u0259dl\u0259ri \u00fczr\u0259 m\u00fct\u0259x\u0259ssis k\u00f6m\u0259k\u00e7isisiniz.\n",
+    "\n",
+    "Kontekst (\u0259laq\u0259li s\u0259n\u0259dl\u0259r):\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "\u018ftrafl\u0131 cavab verin v\u0259 m\u00fctl\u0259q s\u0259n\u0259d m\u0259nb\u0259l\u0259rin\u0259 istinad edin (PDF ad\u0131 v\u0259 s\u0259hif\u0259 n\u00f6mr\u0259si il\u0259).\n",
+    "Cavab\u0131n\u0131z d\u0259qiq, faktlara \u0259saslanan v\u0259 kontekst m\u0259lumatlar\u0131ndan istifad\u0259 ed\u0259n olmal\u0131d\u0131r.\"\"\"\n",
+    "    \n",
+    "    # Get model deployment\n",
+    "    deployment = MODELS[model_name]['deployment']\n",
+    "    \n",
+    "    try:\n",
+    "        start_time = time.time()\n",
+    "        \n",
+    "        # GPT-5 models use max_completion_tokens, others use max_tokens\n",
+    "        if deployment.startswith('gpt-5'):\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[\n",
+    "                    {\"role\": \"user\", \"content\": prompt}\n",
+    "                ],\n",
+    "                temperature=temperature,\n",
+    "                max_completion_tokens=max_tokens\n",
+    "            )\n",
+    "        else:\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[\n",
+    "                    {\"role\": \"user\", \"content\": prompt}\n",
+    "                ],\n",
+    "                temperature=temperature,\n",
+    "                max_tokens=max_tokens\n",
+    "            )\n",
+    "        \n",
+    "        response_time = time.time() - start_time\n",
+    "        answer = response.choices[0].message.content\n",
+    "        \n",
+    "        return answer, response_time\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        return f\"ERROR: {str(e)}\", 0.0\n",
+    "\n",
+    "print(f\"\\n\u2705 Configured {len(LLM_MODELS)} LLM models for testing\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Evaluation Metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize_text(text: str) -> str:\n",
+    "    \"\"\"Normalize text for comparison.\"\"\"\n",
+    "    import re\n",
+    "    text = text.lower().strip()\n",
+    "    text = re.sub(r'\\s+', ' ', text)\n",
+    "    return text\n",
+    "\n",
+    "def calculate_answer_similarity(reference: str, hypothesis: str) -> Dict[str, float]:\n",
+    "    \"\"\"\n",
+    "    Calculate similarity between generated and expected answer.\n",
+    "    Lower is better for error rates.\n",
+    "    \"\"\"\n",
+    "    ref_norm = normalize_text(reference)\n",
+    "    hyp_norm = normalize_text(hypothesis)\n",
+    "    \n",
+    "    # Character Error Rate\n",
+    "    cer_score = cer(ref_norm, hyp_norm) * 100\n",
+    "    \n",
+    "    # Word Error Rate  \n",
+    "    wer_score = wer(ref_norm, hyp_norm) * 100\n",
+    "    \n",
+    "    # Similarity scores (higher is better)\n",
+    "    similarity = max(0, 100 - wer_score)\n",
+    "    \n",
+    "    return {\n",
+    "        'CER': round(cer_score, 2),\n",
+    "        'WER': round(wer_score, 2),\n",
+    "        'Similarity': round(similarity, 2)\n",
+    "    }\n",
+    "\n",
+    "def check_citations(answer: str, documents: List[Dict]) -> Dict[str, any]:\n",
+    "    \"\"\"\n",
+    "    Check if answer includes proper citations.\n",
+    "    \"\"\"\n",
+    "    import re\n",
+    "    \n",
+    "    # Check for PDF names\n",
+    "    pdf_names = [doc['pdf_name'] for doc in documents]\n",
+    "    cited_pdfs = sum(1 for pdf in pdf_names if pdf.replace('.pdf', '') in answer)\n",
+    "    \n",
+    "    # Check for page numbers\n",
+    "    page_numbers = [str(doc['page_number']) for doc in documents]\n",
+    "    cited_pages = sum(1 for page in page_numbers if page in answer)\n",
+    "    \n",
+    "    # Check for source keywords\n",
+    "    source_keywords = ['m\u0259nb\u0259', 's\u0259n\u0259d', 's\u0259hif\u0259', 'pdf', 'document', 'page', 'source']\n",
+    "    has_source_ref = any(kw in answer.lower() for kw in source_keywords)\n",
+    "    \n",
+    "    citation_score = (\n",
+    "        (cited_pdfs / len(pdf_names) * 40) +  # 40% for PDF citation\n",
+    "        (cited_pages / len(page_numbers) * 40) +  # 40% for page citation\n",
+    "        (20 if has_source_ref else 0)  # 20% for having source keywords\n",
+    "    )\n",
+    "    \n",
+    "    return {\n",
+    "        'Citation_Score': round(citation_score, 2),\n",
+    "        'Cited_PDFs': cited_pdfs,\n",
+    "        'Cited_Pages': cited_pages,\n",
+    "        'Has_Source_Reference': has_source_ref\n",
+    "    }\n",
+    "\n",
+    "def evaluate_completeness(answer: str, min_length: int = 100) -> Dict[str, any]:\n",
+    "    \"\"\"\n",
+    "    Evaluate answer completeness.\n",
+    "    \"\"\"\n",
+    "    word_count = len(answer.split())\n",
+    "    char_count = len(answer)\n",
+    "    \n",
+    "    # Penalize very short or very long answers\n",
+    "    if char_count < min_length:\n",
+    "        completeness_score = (char_count / min_length) * 100\n",
+    "    elif char_count > 2000:\n",
+    "        completeness_score = 100 - ((char_count - 2000) / 2000 * 20)  # Penalty for verbosity\n",
+    "    else:\n",
+    "        completeness_score = 100\n",
+    "    \n",
+    "    return {\n",
+    "        'Completeness_Score': round(max(0, completeness_score), 2),\n",
+    "        'Word_Count': word_count,\n",
+    "        'Char_Count': char_count\n",
+    "    }\n",
+    "\n",
+    "print(\"\u2705 Evaluation functions ready\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Run Benchmark on All Models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Select models to test (you can comment out models to skip)\n",
+    "MODELS_TO_TEST = [\n",
+    "    'Llama-4-Maverick-17B',\n",
+    "    'DeepSeek-R1',\n",
+    "    'GPT-4.1',\n",
+    "    'GPT-5-mini',\n",
+    "    'Claude-Sonnet-4.5',\n",
+    "    # 'Claude-Opus-4.1',  # Uncomment to test\n",
+    "    # 'Phi-4-multimodal',  # Uncomment to test\n",
+    "    # 'GPT-OSS-120B',  # Uncomment to test\n",
+    "]\n",
+    "\n",
+    "print(f\"Testing {len(MODELS_TO_TEST)} models on {len(questions)} questions...\\n\")\n",
+    "print(\"This may take several minutes...\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run benchmark\n",
+    "results = []\n",
+    "\n",
+    "for model_name in MODELS_TO_TEST:\n",
+    "    print(f\"\\n{'='*80}\")\n",
+    "    print(f\"Testing: {model_name}\")\n",
+    "    print(f\"{'='*80}\")\n",
+    "    \n",
+    "    model_results = []\n",
+    "    \n",
+    "    for example_key, messages in questions.items():\n",
+    "        # Get the last user message (the actual question)\n",
+    "        user_msg = [m for m in messages if m['role'] == 'user'][-1]\n",
+    "        query = user_msg['content']\n",
+    "        \n",
+    "        print(f\"\\n  Question {example_key}: {query[:80]}...\")\n",
+    "        \n",
+    "        # Retrieve documents\n",
+    "        documents = retrieve_documents(query, top_k=3)\n",
+    "        \n",
+    "        # Generate answer\n",
+    "        answer, response_time = generate_answer(model_name, query, documents)\n",
+    "        \n",
+    "        if answer.startswith('ERROR'):\n",
+    "            print(f\"  \u274c Failed: {answer}\")\n",
+    "            continue\n",
+    "        \n",
+    "        print(f\"  \u2705 Response time: {response_time:.2f}s\")\n",
+    "        \n",
+    "        # Get expected answer\n",
+    "        expected = expected_answers.get(example_key, {}).get('Answer', '')\n",
+    "        \n",
+    "        # Calculate metrics\n",
+    "        similarity_metrics = calculate_answer_similarity(expected, answer) if expected else {'CER': 0, 'WER': 0, 'Similarity': 0}\n",
+    "        citation_metrics = check_citations(answer, documents)\n",
+    "        completeness_metrics = evaluate_completeness(answer)\n",
+    "        \n",
+    "        # Store result\n",
+    "        result = {\n",
+    "            'Model': model_name,\n",
+    "            'Question': example_key,\n",
+    "            'Query': query[:100],\n",
+    "            'Answer': answer[:200] + '...',\n",
+    "            'Response_Time': round(response_time, 2),\n",
+    "            **similarity_metrics,\n",
+    "            **citation_metrics,\n",
+    "            **completeness_metrics,\n",
+    "            'Open_Source': MODELS[model_name]['open_source'],\n",
+    "            'Architecture_Score': MODELS[model_name]['architecture_score']\n",
+    "        }\n",
+    "        \n",
+    "        model_results.append(result)\n",
+    "        results.append(result)\n",
+    "    \n",
+    "    # Show summary for this model\n",
+    "    if model_results:\n",
+    "        avg_response_time = sum(r['Response_Time'] for r in model_results) / len(model_results)\n",
+    "        avg_similarity = sum(r['Similarity'] for r in model_results) / len(model_results)\n",
+    "        avg_citation = sum(r['Citation_Score'] for r in model_results) / len(model_results)\n",
+    "        avg_completeness = sum(r['Completeness_Score'] for r in model_results) / len(model_results)\n",
+    "        \n",
+    "        print(f\"\\n  \ud83d\udcca {model_name} Summary:\")\n",
+    "        print(f\"     Avg Response Time: {avg_response_time:.2f}s\")\n",
+    "        print(f\"     Avg Similarity: {avg_similarity:.1f}%\")\n",
+    "        print(f\"     Avg Citation Score: {avg_citation:.1f}%\")\n",
+    "        print(f\"     Avg Completeness: {avg_completeness:.1f}%\")\n",
+    "\n",
+    "print(f\"\\n{'='*80}\")\n",
+    "print(\"\u2705 Benchmarking complete!\")\n",
+    "print(f\"{'='*80}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 7. Aggregate Results and Rankings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create DataFrame\n",
+    "df = pd.DataFrame(results)\n",
+    "\n",
+    "# Calculate aggregate scores per model\n",
+    "model_summary = df.groupby('Model').agg({\n",
+    "    'Response_Time': 'mean',\n",
+    "    'Similarity': 'mean',\n",
+    "    'Citation_Score': 'mean',\n",
+    "    'Completeness_Score': 'mean',\n",
+    "    'CER': 'mean',\n",
+    "    'WER': 'mean',\n",
+    "    'Open_Source': 'first',\n",
+    "    'Architecture_Score': 'first'\n",
+    "}).round(2)\n",
+    "\n",
+    "# Calculate overall quality score (weighted average)\n",
+    "model_summary['Quality_Score'] = (\n",
+    "    model_summary['Similarity'] * 0.35 +  # 35% answer accuracy\n",
+    "    model_summary['Citation_Score'] * 0.35 +  # 35% citation quality\n",
+    "    model_summary['Completeness_Score'] * 0.30  # 30% completeness\n",
+    ").round(2)\n",
+    "\n",
+    "# Sort by Quality Score\n",
+    "model_summary = model_summary.sort_values('Quality_Score', ascending=False)\n",
+    "\n",
+    "# Display summary table\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"\ud83d\udcca LLM BENCHMARKING RESULTS - MODEL SUMMARY\")\n",
+    "print(\"=\"*100)\n",
+    "print(model_summary.to_string())\n",
+    "print(\"=\"*100)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create comprehensive visualization\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Create output directory - using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'llm_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "fig, axes = plt.subplots(2, 2, figsize=(16, 10))\n",
+    "\n",
+    "models = df['Model'].tolist()\n",
+    "colors = sns.color_palette('viridis', len(models))\n",
+    "\n",
+    "# 1. CSR - Character Success Rate (MAIN METRIC)\n",
+    "ax1 = axes[0, 0]\n",
+    "bars1 = ax1.barh(models, df['CSR'], color=colors)\n",
+    "ax1.set_xlabel('CSR (%) - Higher is Better', fontsize=12, fontweight='bold')\n",
+    "ax1.set_title('Character Success Rate (CSR)\\n\ud83c\udfc6 HACKATHON PRIMARY METRIC', \n",
+    "              fontsize=14, fontweight='bold')\n",
+    "ax1.set_xlim(0, 100)\n",
+    "for i, (model, csr) in enumerate(zip(models, df['CSR'])):\n",
+    "    ax1.text(csr + 1, i, f'{csr:.2f}%', va='center', fontsize=11, fontweight='bold')\n",
+    "ax1.axvline(x=90, color='green', linestyle='--', alpha=0.3, label='Excellent (>90%)')\n",
+    "ax1.axvline(x=80, color='orange', linestyle='--', alpha=0.3, label='Good (>80%)')\n",
+    "ax1.legend(fontsize=9)\n",
+    "\n",
+    "# 2. WSR - Word Success Rate\n",
+    "ax2 = axes[0, 1]\n",
+    "bars2 = ax2.barh(models, df['WSR'], color=colors)\n",
+    "ax2.set_xlabel('WSR (%) - Higher is Better', fontsize=12, fontweight='bold')\n",
+    "ax2.set_title('Word Success Rate (WSR)', fontsize=14, fontweight='bold')\n",
+    "ax2.set_xlim(0, 100)\n",
+    "for i, (model, wsr) in enumerate(zip(models, df['WSR'])):\n",
+    "    ax2.text(wsr + 1, i, f'{wsr:.2f}%', va='center', fontsize=11, fontweight='bold')\n",
+    "\n",
+    "# 3. Response Time\n",
+    "ax3 = axes[1, 0]\n",
+    "bars3 = ax3.barh(models, df['Response_Time'], color=colors)\n",
+    "ax3.set_xlabel('Total Time (seconds) - Lower is Better', fontsize=12, fontweight='bold')\n",
+    "ax3.set_title('Processing Speed', fontsize=14, fontweight='bold')\n",
+    "for i, (model, time_val) in enumerate(zip(models, df['Response_Time'])):\n",
+    "    ax3.text(time_val + 0.5, i, f'{time_val:.1f}s', va='center', fontsize=11)\n",
+    "\n",
+    "# 4. Error Rates Comparison\n",
+    "ax4 = axes[1, 1]\n",
+    "x = range(len(models))\n",
+    "width = 0.35\n",
+    "ax4.bar([i - width/2 for i in x], df['CER'], width, label='CER', color='coral', alpha=0.8)\n",
+    "ax4.bar([i + width/2 for i in x], df['WER'], width, label='WER', color='skyblue', alpha=0.8)\n",
+    "ax4.set_ylabel('Error Rate (%) - Lower is Better', fontsize=12, fontweight='bold')\n",
+    "ax4.set_title('Error Rates', fontsize=14, fontweight='bold')\n",
+    "ax4.set_xticks(x)\n",
+    "ax4.set_xticklabels(models, rotation=45, ha='right')\n",
+    "ax4.legend(fontsize=11)\n",
+    "ax4.grid(axis='y', alpha=0.3)\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(output_dir / 'results.png', dpi=300, bbox_inches='tight')\n",
+    "plt.show()\n",
+    "\n",
+    "print(f\"\\n\u2705 Visualization saved to '{output_dir}/results.png'\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create rankings table\n",
+    "rankings = model_summary[[\n",
+    "    'Quality_Score', 'Similarity', 'Citation_Score', 'Completeness_Score', \n",
+    "    'Response_Time', 'Open_Source', 'Architecture_Score'\n",
+    "]].copy()\n",
+    "\n",
+    "rankings.insert(0, 'Rank', range(1, len(rankings) + 1))\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"\ud83c\udfc6 FINAL RANKINGS\")\n",
+    "print(\"=\"*100)\n",
+    "print(rankings.to_string())\n",
+    "print(\"=\"*100)\n",
+    "\n",
+    "# Winner analysis\n",
+    "best_overall = rankings.index[0]\n",
+    "best_open_source = rankings[rankings['Open_Source'] == True].index[0] if any(rankings['Open_Source']) else None\n",
+    "fastest = model_summary['Response_Time'].idxmin()\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"\ud83d\udca1 RECOMMENDATIONS FOR HACKATHON\")\n",
+    "print(\"=\"*100)\n",
+    "\n",
+    "print(f\"\\n\ud83e\udd47 Best Overall Quality: {best_overall}\")\n",
+    "print(f\"   Quality Score: {model_summary.loc[best_overall, 'Quality_Score']:.1f}%\")\n",
+    "print(f\"   Similarity: {model_summary.loc[best_overall, 'Similarity']:.1f}%\")\n",
+    "print(f\"   Citation Score: {model_summary.loc[best_overall, 'Citation_Score']:.1f}%\")\n",
+    "print(f\"   Response Time: {model_summary.loc[best_overall, 'Response_Time']:.2f}s\")\n",
+    "print(f\"   Open Source: {model_summary.loc[best_overall, 'Open_Source']}\")\n",
+    "print(f\"   Architecture Score: {model_summary.loc[best_overall, 'Architecture_Score']}\")\n",
+    "\n",
+    "if best_open_source:\n",
+    "    print(f\"\\n\ud83d\udd13 Best Open-Source Model: {best_open_source}\")\n",
+    "    print(f\"   Quality Score: {model_summary.loc[best_open_source, 'Quality_Score']:.1f}%\")\n",
+    "    print(f\"   Architecture Score: {model_summary.loc[best_open_source, 'Architecture_Score']} (Better for hackathon!)\")\n",
+    "    print(f\"   Response Time: {model_summary.loc[best_open_source, 'Response_Time']:.2f}s\")\n",
+    "\n",
+    "print(f\"\\n\u26a1 Fastest Model: {fastest}\")\n",
+    "print(f\"   Response Time: {model_summary.loc[fastest, 'Response_Time']:.2f}s\")\n",
+    "print(f\"   Quality Score: {model_summary.loc[fastest, 'Quality_Score']:.1f}%\")\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"\ud83d\udcdd FINAL RECOMMENDATION\")\n",
+    "print(\"=\"*100)\n",
+    "print(\"\\nScoring Breakdown:\")\n",
+    "print(\"  - LLM Quality: 30% of total hackathon score\")\n",
+    "print(\"  - Architecture: 20% of total hackathon score (open-source preferred!)\")\n",
+    "print(\"\\nBest Choice:\")\n",
+    "if best_open_source and model_summary.loc[best_open_source, 'Quality_Score'] >= model_summary.loc[best_overall, 'Quality_Score'] * 0.9:\n",
+    "    print(f\"  \u2705 {best_open_source} - Best balance of quality and architecture score\")\n",
+    "    print(f\"     Only {model_summary.loc[best_overall, 'Quality_Score'] - model_summary.loc[best_open_source, 'Quality_Score']:.1f}% quality drop for higher architecture score!\")\n",
+    "else:\n",
+    "    print(f\"  \u2705 {best_overall} - Highest quality, use if quality gap is significant\")\n",
+    "    if best_open_source:\n",
+    "        print(f\"  \u26a0\ufe0f  Consider {best_open_source} for higher architecture score (trade-off: {model_summary.loc[best_overall, 'Quality_Score'] - model_summary.loc[best_open_source, 'Quality_Score']:.1f}% quality)\")\n",
+    "\n",
+    "print(\"=\"*100)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Save results\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'llm_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "df.to_csv(output_dir / 'detailed_results.csv', index=False, encoding='utf-8')\n",
+    "model_summary.to_csv(output_dir / 'summary.csv', encoding='utf-8')\n",
+    "rankings.to_csv(output_dir / 'rankings.csv', index=False, encoding='utf-8')\n",
+    "\n",
+    "print(\"\\n\u2705 Results exported to output/llm_benchmark/:\")\n",
+    "print(\"   - detailed_results.csv (all questions and answers)\")\n",
+    "print(\"   - summary.csv (model averages)\")\n",
+    "print(\"   - rankings.csv (final rankings)\")\n",
+    "print(\"   - results.png (visualizations)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 11. Sample Answer Comparison"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/rag_optimization_benchmark.ipynb.backup

@@ -0,0 +1,1072 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# RAG Pipeline Optimization Benchmark\n",
+    "\n",
+    "**Comprehensive testing of ALL RAG components to maximize LLM Judge score**\n",
+    "\n",
+    "## What We're Testing:\n",
+    "\n",
+    "### 1. Embedding Models (Vector Representations)\n",
+    "- `BAAI/bge-large-en-v1.5` (Current - 1024 dim, best quality)\n",
+    "- `BAAI/bge-base-en-v1.5` (768 dim, faster)\n",
+    "- `intfloat/multilingual-e5-large` (1024 dim, multi-language)\n",
+    "- `sentence-transformers/paraphrase-multilingual-mpnet-base-v2` (768 dim, multilingual)\n",
+    "- `sentence-transformers/all-MiniLM-L6-v2` (384 dim, very fast)\n",
+    "\n",
+    "### 2. Retrieval Strategies\n",
+    "- **Top-K**: Test 1, 3, 5, 10 documents\n",
+    "- **MMR** (Maximal Marginal Relevance): Diversity vs relevance trade-off\n",
+    "- **Similarity Threshold**: Filter low-relevance docs\n",
+    "- **Reranking**: Use cross-encoder to rerank results\n",
+    "\n",
+    "### 3. Chunking Strategies (Already in Vector DB, but we'll compare)\n",
+    "- Chunk size: 256, 512, 600 (current), 1000 tokens\n",
+    "- Overlap: 0, 50, 100 (current), 200 chars\n",
+    "\n",
+    "### 4. LLM Models\n",
+    "- Llama-4-Maverick-17B (open-source)\n",
+    "- DeepSeek-R1 (reasoning)\n",
+    "- GPT-4.1, GPT-5, GPT-5-mini\n",
+    "- Claude-Sonnet-4.5\n",
+    "\n",
+    "### 5. Prompting Techniques\n",
+    "- **Baseline**: Simple context + question\n",
+    "- **Citation-focused**: Emphasize source references\n",
+    "- **Step-by-step**: Chain-of-thought reasoning\n",
+    "- **Few-shot**: Include example Q&A\n",
+    "\n",
+    "## LLM Judge Evaluation Criteria:\n",
+    "- **Accuracy** (35%): Answer correctness\n",
+    "- **Relevance** (35%): Citation quality and relevance\n",
+    "- **Completeness** (30%): Thorough answers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !pip install openai pinecone-client sentence-transformers rank-bm25 python-dotenv pandas matplotlib seaborn jiwer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/ismatsamadov/SOCAR_Hackathon/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Libraries loaded\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from typing import Dict, List, Tuple, Any\n",
+    "from collections import defaultdict\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from openai import AzureOpenAI\n",
+    "from pinecone import Pinecone\n",
+    "from sentence_transformers import SentenceTransformer, CrossEncoder\n",
+    "from jiwer import wer, cer\n",
+    "import numpy as np\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "sns.set_style('whitegrid')\n",
+    "plt.rcParams['figure.figsize'] = (16, 10)\n",
+    "\n",
+    "print(\"✅ Libraries loaded\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Project root: /Users/ismatsamadov/SOCAR_Hackathon\n",
+      "✅ Docs directory: /Users/ismatsamadov/SOCAR_Hackathon/docs\n",
+      "✅ Output directory: /Users/ismatsamadov/SOCAR_Hackathon/output\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Auto-detect project root (works from any directory)\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if Path('data').exists() and Path('docs').exists():\n",
+    "    # Already in project root\n",
+    "    PROJECT_ROOT = Path.cwd()\n",
+    "elif Path('../data').exists() and Path('../docs').exists():\n",
+    "    # In notebooks/ subdirectory\n",
+    "    PROJECT_ROOT = Path.cwd().parent\n",
+    "else:\n",
+    "    # Fallback: try to find project root\n",
+    "    current = Path.cwd()\n",
+    "    while current != current.parent:\n",
+    "        if (current / 'data').exists() and (current / 'docs').exists():\n",
+    "            PROJECT_ROOT = current\n",
+    "            break\n",
+    "        current = current.parent\n",
+    "    else:\n",
+    "        PROJECT_ROOT = Path.cwd()\n",
+    "\n",
+    "# Define all paths relative to project root\n",
+    "DATA_DIR = PROJECT_ROOT / 'data'\n",
+    "DOCS_DIR = PROJECT_ROOT / 'docs'\n",
+    "OUTPUT_DIR = PROJECT_ROOT / 'output'\n",
+    "\n",
+    "print(f\"✅ Project root: {PROJECT_ROOT}\")\n",
+    "print(f\"✅ Docs directory: {DOCS_DIR}\")\n",
+    "print(f\"✅ Output directory: {OUTPUT_DIR}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Loaded 5 test questions\n",
+      "  - Example1\n",
+      "  - Example2\n",
+      "  - Example3\n",
+      "  - Example4\n",
+      "  - Example5\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load test cases - using dynamic paths\n",
+    "with open(DOCS_DIR / 'sample_questions.json', 'r', encoding='utf-8') as f:\n",
+    "    questions = json.load(f)\n",
+    "\n",
+    "with open(DOCS_DIR / 'sample_answers.json', 'r', encoding='utf-8') as f:\n",
+    "    expected_answers = json.load(f)\n",
+    "\n",
+    "print(f\"✅ Loaded {len(questions)} test questions\")\n",
+    "for key in questions.keys():\n",
+    "    print(f\"  - {key}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Initialize Vector Database"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Connect to Pinecone\n",
+    "pc = Pinecone(api_key=os.getenv('PINECONE_API_KEY'))\n",
+    "index = pc.Index(os.getenv('PINECONE_INDEX_NAME', 'hackathon'))\n",
+    "\n",
+    "stats = index.describe_index_stats()\n",
+    "print(f\"✅ Vector DB connected\")\n",
+    "print(f\"   Total vectors: {stats['total_vector_count']}\")\n",
+    "print(f\"   Dimensions: {stats['dimension']}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Embedding Models Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "EMBEDDING_MODELS = {\n",
+    "    'bge-large-en': {\n",
+    "        'name': 'BAAI/bge-large-en-v1.5',\n",
+    "        'dimensions': 1024,\n",
+    "        'notes': 'Current model - best quality'\n",
+    "    },\n",
+    "    'bge-base-en': {\n",
+    "        'name': 'BAAI/bge-base-en-v1.5',\n",
+    "        'dimensions': 768,\n",
+    "        'notes': 'Faster, slightly lower quality'\n",
+    "    },\n",
+    "    'multilingual-e5-large': {\n",
+    "        'name': 'intfloat/multilingual-e5-large',\n",
+    "        'dimensions': 1024,\n",
+    "        'notes': 'Multi-language optimized'\n",
+    "    },\n",
+    "    'paraphrase-multilingual': {\n",
+    "        'name': 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2',\n",
+    "        'dimensions': 768,\n",
+    "        'notes': 'Good for Azerbaijani/Russian'\n",
+    "    },\n",
+    "    'all-MiniLM-L6': {\n",
+    "        'name': 'sentence-transformers/all-MiniLM-L6-v2',\n",
+    "        'dimensions': 384,\n",
+    "        'notes': 'Very fast, lower quality'\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "# Load embedding models (only test 1024-dim models for existing Pinecone index)\n",
+    "EMBEDDING_MODELS_TO_TEST = [\n",
+    "    'bge-large-en',  # Current\n",
+    "    'multilingual-e5-large',  # Alternative with same dims\n",
+    "]\n",
+    "\n",
+    "embedding_cache = {}\n",
+    "\n",
+    "for model_key in EMBEDDING_MODELS_TO_TEST:\n",
+    "    model_name = EMBEDDING_MODELS[model_key]['name']\n",
+    "    print(f\"Loading {model_key}...\")\n",
+    "    embedding_cache[model_key] = SentenceTransformer(model_name)\n",
+    "    print(f\"  ✅ {model_name}\")\n",
+    "\n",
+    "print(f\"\\n✅ Loaded {len(embedding_cache)} embedding models\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Retrieval Strategies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def retrieve_vanilla(query: str, embed_model: SentenceTransformer, top_k: int = 3) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    Vanilla retrieval: Simple top-k vector search.\n",
+    "    \"\"\"\n",
+    "    query_embedding = embed_model.encode(query).tolist()\n",
+    "    results = index.query(vector=query_embedding, top_k=top_k, include_metadata=True)\n",
+    "    \n",
+    "    documents = []\n",
+    "    for match in results['matches']:\n",
+    "        documents.append({\n",
+    "            'pdf_name': match['metadata'].get('pdf_name', 'unknown.pdf'),\n",
+    "            'page_number': match['metadata'].get('page_number', 0),\n",
+    "            'content': match['metadata'].get('text', ''),\n",
+    "            'score': match.get('score', 0.0)\n",
+    "        })\n",
+    "    \n",
+    "    return documents\n",
+    "\n",
+    "\n",
+    "def retrieve_with_threshold(query: str, embed_model: SentenceTransformer, \n",
+    "                           top_k: int = 10, threshold: float = 0.7) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    Retrieve with similarity threshold filtering.\n",
+    "    \"\"\"\n",
+    "    docs = retrieve_vanilla(query, embed_model, top_k=top_k)\n",
+    "    return [doc for doc in docs if doc['score'] >= threshold]\n",
+    "\n",
+    "\n",
+    "def retrieve_with_mmr(query: str, embed_model: SentenceTransformer, \n",
+    "                     top_k: int = 3, lambda_param: float = 0.5, fetch_k: int = 20) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    MMR (Maximal Marginal Relevance) for diversity.\n",
+    "    lambda=1 → pure relevance, lambda=0 → pure diversity\n",
+    "    \"\"\"\n",
+    "    # Fetch more candidates\n",
+    "    candidates = retrieve_vanilla(query, embed_model, top_k=fetch_k)\n",
+    "    \n",
+    "    if len(candidates) <= top_k:\n",
+    "        return candidates[:top_k]\n",
+    "    \n",
+    "    # Query embedding\n",
+    "    query_emb = embed_model.encode(query)\n",
+    "    \n",
+    "    # Get embeddings for candidates\n",
+    "    candidate_texts = [doc['content'] for doc in candidates]\n",
+    "    candidate_embs = embed_model.encode(candidate_texts)\n",
+    "    \n",
+    "    # MMR algorithm\n",
+    "    selected = []\n",
+    "    selected_embs = []\n",
+    "    \n",
+    "    for _ in range(min(top_k, len(candidates))):\n",
+    "        mmr_scores = []\n",
+    "        \n",
+    "        for i, (doc, emb) in enumerate(zip(candidates, candidate_embs)):\n",
+    "            if i in [candidates.index(s) for s in selected]:\n",
+    "                mmr_scores.append(-float('inf'))\n",
+    "                continue\n",
+    "            \n",
+    "            # Relevance to query\n",
+    "            relevance = np.dot(query_emb, emb) / (np.linalg.norm(query_emb) * np.linalg.norm(emb))\n",
+    "            \n",
+    "            # Max similarity to already selected\n",
+    "            if selected_embs:\n",
+    "                similarities = [np.dot(emb, s_emb) / (np.linalg.norm(emb) * np.linalg.norm(s_emb)) \n",
+    "                              for s_emb in selected_embs]\n",
+    "                max_sim = max(similarities)\n",
+    "            else:\n",
+    "                max_sim = 0\n",
+    "            \n",
+    "            # MMR score\n",
+    "            mmr = lambda_param * relevance - (1 - lambda_param) * max_sim\n",
+    "            mmr_scores.append(mmr)\n",
+    "        \n",
+    "        # Select best MMR score\n",
+    "        best_idx = np.argmax(mmr_scores)\n",
+    "        selected.append(candidates[best_idx])\n",
+    "        selected_embs.append(candidate_embs[best_idx])\n",
+    "    \n",
+    "    return selected\n",
+    "\n",
+    "\n",
+    "def retrieve_with_reranking(query: str, embed_model: SentenceTransformer, \n",
+    "                           top_k: int = 3, fetch_k: int = 20) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    Two-stage: retrieve with embeddings, rerank with cross-encoder.\n",
+    "    \"\"\"\n",
+    "    # Stage 1: Retrieve candidates\n",
+    "    candidates = retrieve_vanilla(query, embed_model, top_k=fetch_k)\n",
+    "    \n",
+    "    if len(candidates) <= top_k:\n",
+    "        return candidates[:top_k]\n",
+    "    \n",
+    "    # Stage 2: Rerank with cross-encoder\n",
+    "    reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')\n",
+    "    \n",
+    "    pairs = [[query, doc['content']] for doc in candidates]\n",
+    "    scores = reranker.predict(pairs)\n",
+    "    \n",
+    "    # Sort by reranker score\n",
+    "    scored_docs = [(doc, score) for doc, score in zip(candidates, scores)]\n",
+    "    scored_docs.sort(key=lambda x: x[1], reverse=True)\n",
+    "    \n",
+    "    # Update scores and return top-k\n",
+    "    reranked = []\n",
+    "    for doc, score in scored_docs[:top_k]:\n",
+    "        doc['rerank_score'] = float(score)\n",
+    "        reranked.append(doc)\n",
+    "    \n",
+    "    return reranked\n",
+    "\n",
+    "\n",
+    "RETRIEVAL_STRATEGIES = {\n",
+    "    'vanilla_k3': {'func': retrieve_vanilla, 'params': {'top_k': 3}, 'notes': 'Current setup'},\n",
+    "    'vanilla_k5': {'func': retrieve_vanilla, 'params': {'top_k': 5}, 'notes': 'More context'},\n",
+    "    'vanilla_k10': {'func': retrieve_vanilla, 'params': {'top_k': 10}, 'notes': 'Maximum context'},\n",
+    "    'threshold_0.7': {'func': retrieve_with_threshold, 'params': {'top_k': 10, 'threshold': 0.7}, 'notes': 'Quality filter'},\n",
+    "    'mmr_balanced': {'func': retrieve_with_mmr, 'params': {'top_k': 3, 'lambda_param': 0.5}, 'notes': 'Balance diversity'},\n",
+    "    'mmr_diverse': {'func': retrieve_with_mmr, 'params': {'top_k': 3, 'lambda_param': 0.3}, 'notes': 'More diversity'},\n",
+    "    'reranked_k3': {'func': retrieve_with_reranking, 'params': {'top_k': 3, 'fetch_k': 20}, 'notes': 'Two-stage rerank'},\n",
+    "}\n",
+    "\n",
+    "print(f\"✅ Configured {len(RETRIEVAL_STRATEGIES)} retrieval strategies\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. LLM Models and Prompting Strategies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize Azure OpenAI\n",
+    "azure_client = AzureOpenAI(\n",
+    "    api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
+    "    api_version=os.getenv('AZURE_OPENAI_API_VERSION', '2024-08-01-preview'),\n",
+    "    azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT')\n",
+    ")\n",
+    "\n",
+    "LLM_MODELS = {\n",
+    "    'Llama-4-Maverick': 'Llama-4-Maverick-17B-128E-Instruct-FP8',\n",
+    "    'DeepSeek-R1': 'DeepSeek-R1',\n",
+    "    'GPT-4.1': 'gpt-4.1',\n",
+    "    'GPT-5-mini': 'gpt-5-mini',\n",
+    "    'Claude-Sonnet-4.5': 'claude-sonnet-4-5',\n",
+    "}\n",
+    "\n",
+    "# Prompting strategies\n",
+    "PROMPTING_STRATEGIES = {\n",
+    "    'baseline': \"\"\"\n",
+    "Siz SOCAR-ın tarixi neft və qaz sənədləri üzrə köməkçisiniz.\n",
+    "\n",
+    "Kontekst:\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "Kontekstə əsaslanaraq cavab verin.\n",
+    "\"\"\",\n",
+    "    \n",
+    "    'citation_focused': \"\"\"\n",
+    "Siz SOCAR-ın tarixi sənədlər üzrə mütəxəssis köməkçisisiniz.\n",
+    "\n",
+    "ÖNƏMLİ: Hər bir faktı mütləq mənbə ilə təsdiqləyin (PDF adı və səhifə nömrəsi).\n",
+    "\n",
+    "Kontekst:\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "Cavab verərkən:\n",
+    "1. Dəqiq faktlar yazın\n",
+    "2. Hər faktı mənbə ilə göstərin: (PDF: fayl_adı.pdf, Səhifə: X)\n",
+    "3. Kontekstdə olmayan məlumat əlavə etməyin\n",
+    "\"\"\",\n",
+    "    \n",
+    "    'step_by_step': \"\"\"\n",
+    "Siz SOCAR-ın tarixi sənədlər üzrə analitik köməkçisisiniz.\n",
+    "\n",
+    "Kontekst:\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "Addım-addım cavab verin:\n",
+    "1. Əvvəlcə kontekstdən əlaqəli məlumatları müəyyənləşdirin\n",
+    "2. Bu məlumatları təhlil edin\n",
+    "3. Nəticəni mənbələr ilə birlikdə təqdim edin\n",
+    "\"\"\",\n",
+    "    \n",
+    "    'few_shot': \"\"\"\n",
+    "Siz SOCAR-ın tarixi sənədlər üzrə mütəxəssis köməkçisisiniz.\n",
+    "\n",
+    "Nümunə:\n",
+    "Sual: \"Palçıq vulkanlarının təsir radiusu nə qədərdir?\"\n",
+    "Cavab: \"Sahə müşahidələri və modelləşdirmə göstərir ki, palçıq vulkanlarının təsir radiusu təqribən 10 km-dir (PDF: document_06.pdf, Səhifə: 5).\"\n",
+    "\n",
+    "Kontekst:\n",
+    "{context}\n",
+    "\n",
+    "Sual: {query}\n",
+    "\n",
+    "Yuxarıdakı nümunə kimi cavab verin - dəqiq, qısa, mənbə ilə.\n",
+    "\"\"\"\n",
+    "}\n",
+    "\n",
+    "print(f\"✅ Configured {len(LLM_MODELS)} LLM models\")\n",
+    "print(f\"✅ Configured {len(PROMPTING_STRATEGIES)} prompting strategies\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_answer(llm_model: str, query: str, documents: List[Dict], \n",
+    "                   prompt_strategy: str = 'baseline',\n",
+    "                   temperature: float = 0.2) -> Tuple[str, float]:\n",
+    "    \"\"\"\n",
+    "    Generate answer using LLM with specified prompting strategy.\n",
+    "    \"\"\"\n",
+    "    # Build context\n",
+    "    context_parts = []\n",
+    "    for i, doc in enumerate(documents, 1):\n",
+    "        context_parts.append(\n",
+    "            f\"Sənəd {i} (Mənbə: {doc['pdf_name']}, Səhifə {doc['page_number']}):\\n{doc['content']}\"\n",
+    "        )\n",
+    "    context = \"\\n\\n\".join(context_parts)\n",
+    "    \n",
+    "    # Get prompt template\n",
+    "    prompt_template = PROMPTING_STRATEGIES[prompt_strategy]\n",
+    "    prompt = prompt_template.format(context=context, query=query)\n",
+    "    \n",
+    "    try:\n",
+    "        start_time = time.time()\n",
+    "        \n",
+    "        deployment = LLM_MODELS[llm_model]\n",
+    "        \n",
+    "        # GPT-5 models use max_completion_tokens, others use max_tokens\n",
+    "        if deployment.startswith('gpt-5'):\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[{\"role\": \"user\", \"content\": prompt}],\n",
+    "                temperature=temperature,\n",
+    "                max_completion_tokens=1000\n",
+    "            )\n",
+    "        else:\n",
+    "            response = azure_client.chat.completions.create(\n",
+    "                model=deployment,\n",
+    "                messages=[{\"role\": \"user\", \"content\": prompt}],\n",
+    "                temperature=temperature,\n",
+    "                max_tokens=1000\n",
+    "            )\n",
+    "        \n",
+    "        elapsed = time.time() - start_time\n",
+    "        answer = response.choices[0].message.content\n",
+    "        \n",
+    "        return answer, elapsed\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        return f\"ERROR: {str(e)}\", 0.0\n",
+    "\n",
+    "print(\"✅ LLM generation function ready\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Evaluation Metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize_text(text: str) -> str:\n",
+    "    text = text.lower().strip()\n",
+    "    text = re.sub(r'\\s+', ' ', text)\n",
+    "    return text\n",
+    "\n",
+    "def calculate_answer_quality(reference: str, hypothesis: str) -> Dict[str, float]:\n",
+    "    \"\"\"Accuracy metrics.\"\"\"\n",
+    "    ref_norm = normalize_text(reference)\n",
+    "    hyp_norm = normalize_text(hypothesis)\n",
+    "    \n",
+    "    cer_score = cer(ref_norm, hyp_norm) * 100\n",
+    "    wer_score = wer(ref_norm, hyp_norm) * 100\n",
+    "    similarity = max(0, 100 - wer_score)\n",
+    "    \n",
+    "    return {\n",
+    "        'Accuracy_Score': round(similarity, 2)\n",
+    "    }\n",
+    "\n",
+    "def evaluate_citation_quality(answer: str, documents: List[Dict]) -> Dict[str, float]:\n",
+    "    \"\"\"Relevance - citation quality.\"\"\"\n",
+    "    pdf_names = [doc['pdf_name'].replace('.pdf', '') for doc in documents]\n",
+    "    page_numbers = [str(doc['page_number']) for doc in documents]\n",
+    "    \n",
+    "    cited_pdfs = sum(1 for pdf in pdf_names if pdf in answer)\n",
+    "    cited_pages = sum(1 for page in page_numbers if page in answer)\n",
+    "    \n",
+    "    citation_keywords = ['mənbə', 'sənəd', 'səhifə', 'pdf', 'document', 'page']\n",
+    "    has_citation_format = any(kw in answer.lower() for kw in citation_keywords)\n",
+    "    \n",
+    "    citation_score = (\n",
+    "        (cited_pdfs / len(pdf_names) * 40) +\n",
+    "        (cited_pages / len(page_numbers) * 40) +\n",
+    "        (20 if has_citation_format else 0)\n",
+    "    )\n",
+    "    \n",
+    "    return {\n",
+    "        'Citation_Score': round(citation_score, 2),\n",
+    "        'Cited_PDFs': cited_pdfs,\n",
+    "        'Cited_Pages': cited_pages\n",
+    "    }\n",
+    "\n",
+    "def evaluate_retrieval_quality(query: str, documents: List[Dict], expected_answer: str) -> Dict[str, float]:\n",
+    "    \"\"\"Measure if retrieved docs are relevant to answer.\"\"\"\n",
+    "    if not documents or not expected_answer:\n",
+    "        return {'Retrieval_Relevance': 0.0}\n",
+    "    \n",
+    "    # Simple heuristic: check if expected answer words appear in retrieved docs\n",
+    "    expected_words = set(normalize_text(expected_answer).split())\n",
+    "    retrieved_text = ' '.join([doc['content'] for doc in documents])\n",
+    "    retrieved_words = set(normalize_text(retrieved_text).split())\n",
+    "    \n",
+    "    overlap = len(expected_words & retrieved_words) / len(expected_words) if expected_words else 0\n",
+    "    \n",
+    "    return {\n",
+    "        'Retrieval_Relevance': round(overlap * 100, 2)\n",
+    "    }\n",
+    "\n",
+    "def evaluate_completeness(answer: str) -> Dict[str, float]:\n",
+    "    \"\"\"Completeness metrics.\"\"\"\n",
+    "    word_count = len(answer.split())\n",
+    "    \n",
+    "    if word_count < 20:\n",
+    "        completeness = (word_count / 20) * 100\n",
+    "    elif word_count > 200:\n",
+    "        completeness = 100 - ((word_count - 200) / 200 * 20)\n",
+    "    else:\n",
+    "        completeness = 100\n",
+    "    \n",
+    "    return {\n",
+    "        'Completeness_Score': round(max(0, completeness), 2),\n",
+    "        'Word_Count': word_count\n",
+    "    }\n",
+    "\n",
+    "def calculate_llm_judge_score(accuracy: float, citation: float, completeness: float) -> float:\n",
+    "    \"\"\"Overall LLM Judge score (weighted).\"\"\"\n",
+    "    return round(\n",
+    "        accuracy * 0.35 +\n",
+    "        citation * 0.35 +\n",
+    "        completeness * 0.30,\n",
+    "        2\n",
+    "    )\n",
+    "\n",
+    "print(\"✅ Evaluation metrics ready\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 7. Run Comprehensive Benchmark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configuration: Select what to test\n",
+    "CONFIGS_TO_TEST = [\n",
+    "    # Format: (embed_model, retrieval_strategy, llm_model, prompt_strategy)\n",
+    "    \n",
+    "    # Baseline (current setup)\n",
+    "    ('bge-large-en', 'vanilla_k3', 'Llama-4-Maverick', 'baseline'),\n",
+    "    \n",
+    "    # Test different embedding models\n",
+    "    ('multilingual-e5-large', 'vanilla_k3', 'Llama-4-Maverick', 'baseline'),\n",
+    "    \n",
+    "    # Test different retrieval strategies\n",
+    "    ('bge-large-en', 'vanilla_k5', 'Llama-4-Maverick', 'baseline'),\n",
+    "    ('bge-large-en', 'mmr_balanced', 'Llama-4-Maverick', 'baseline'),\n",
+    "    ('bge-large-en', 'reranked_k3', 'Llama-4-Maverick', 'baseline'),\n",
+    "    \n",
+    "    # Test different LLM models\n",
+    "    ('bge-large-en', 'vanilla_k3', 'GPT-5-mini', 'baseline'),\n",
+    "    ('bge-large-en', 'vanilla_k3', 'Claude-Sonnet-4.5', 'baseline'),\n",
+    "    \n",
+    "    # Test different prompting strategies\n",
+    "    ('bge-large-en', 'vanilla_k3', 'Llama-4-Maverick', 'citation_focused'),\n",
+    "    ('bge-large-en', 'vanilla_k3', 'Llama-4-Maverick', 'few_shot'),\n",
+    "    \n",
+    "    # Best combinations\n",
+    "    ('bge-large-en', 'reranked_k3', 'GPT-5-mini', 'citation_focused'),\n",
+    "    ('bge-large-en', 'mmr_balanced', 'Claude-Sonnet-4.5', 'citation_focused'),\n",
+    "]\n",
+    "\n",
+    "print(f\"Testing {len(CONFIGS_TO_TEST)} configurations on {len(questions)} questions\")\n",
+    "print(f\"Total API calls: ~{len(CONFIGS_TO_TEST) * len(questions)}\")\n",
+    "print(\"This will take 15-30 minutes...\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run benchmark\n",
+    "results = []\n",
+    "\n",
+    "for config_idx, (embed_key, retrieval_key, llm_key, prompt_key) in enumerate(CONFIGS_TO_TEST, 1):\n",
+    "    config_name = f\"{embed_key}_{retrieval_key}_{llm_key}_{prompt_key}\"\n",
+    "    \n",
+    "    print(f\"\\n{'='*100}\")\n",
+    "    print(f\"Config {config_idx}/{len(CONFIGS_TO_TEST)}: {config_name}\")\n",
+    "    print(f\"{'='*100}\")\n",
+    "    \n",
+    "    # Get components\n",
+    "    embed_model = embedding_cache[embed_key]\n",
+    "    retrieval_func = RETRIEVAL_STRATEGIES[retrieval_key]['func']\n",
+    "    retrieval_params = RETRIEVAL_STRATEGIES[retrieval_key]['params']\n",
+    "    \n",
+    "    config_results = []\n",
+    "    \n",
+    "    for example_key, messages in questions.items():\n",
+    "        user_msg = [m for m in messages if m['role'] == 'user'][-1]\n",
+    "        query = user_msg['content']\n",
+    "        \n",
+    "        print(f\"\\n  {example_key}: {query[:60]}...\")\n",
+    "        \n",
+    "        # Retrieve documents\n",
+    "        documents = retrieval_func(query, embed_model, **retrieval_params)\n",
+    "        print(f\"    Retrieved {len(documents)} docs\")\n",
+    "        \n",
+    "        # Generate answer\n",
+    "        answer, response_time = generate_answer(llm_key, query, documents, prompt_key)\n",
+    "        \n",
+    "        if answer.startswith('ERROR'):\n",
+    "            print(f\"    ❌ {answer}\")\n",
+    "            continue\n",
+    "        \n",
+    "        print(f\"    ✅ Generated in {response_time:.2f}s\")\n",
+    "        \n",
+    "        # Evaluate\n",
+    "        expected = expected_answers.get(example_key, {}).get('Answer', '')\n",
+    "        \n",
+    "        accuracy_metrics = calculate_answer_quality(expected, answer) if expected else {'Accuracy_Score': 0}\n",
+    "        citation_metrics = evaluate_citation_quality(answer, documents)\n",
+    "        retrieval_metrics = evaluate_retrieval_quality(query, documents, expected)\n",
+    "        completeness_metrics = evaluate_completeness(answer)\n",
+    "        \n",
+    "        # Calculate overall score\n",
+    "        llm_judge_score = calculate_llm_judge_score(\n",
+    "            accuracy_metrics['Accuracy_Score'],\n",
+    "            citation_metrics['Citation_Score'],\n",
+    "            completeness_metrics['Completeness_Score']\n",
+    "        )\n",
+    "        \n",
+    "        result = {\n",
+    "            'Config': config_name,\n",
+    "            'Embedding_Model': embed_key,\n",
+    "            'Retrieval_Strategy': retrieval_key,\n",
+    "            'LLM_Model': llm_key,\n",
+    "            'Prompt_Strategy': prompt_key,\n",
+    "            'Question': example_key,\n",
+    "            'Query': query[:80],\n",
+    "            'Num_Docs_Retrieved': len(documents),\n",
+    "            'Response_Time': round(response_time, 2),\n",
+    "            'LLM_Judge_Score': llm_judge_score,\n",
+    "            **accuracy_metrics,\n",
+    "            **citation_metrics,\n",
+    "            **retrieval_metrics,\n",
+    "            **completeness_metrics,\n",
+    "            'Answer_Preview': answer[:150]\n",
+    "        }\n",
+    "        \n",
+    "        results.append(result)\n",
+    "        config_results.append(result)\n",
+    "    \n",
+    "    # Show config summary\n",
+    "    if config_results:\n",
+    "        avg_score = sum(r['LLM_Judge_Score'] for r in config_results) / len(config_results)\n",
+    "        avg_time = sum(r['Response_Time'] for r in config_results) / len(config_results)\n",
+    "        print(f\"\\n  📊 Config Summary:\")\n",
+    "        print(f\"     Avg LLM Judge Score: {avg_score:.2f}%\")\n",
+    "        print(f\"     Avg Response Time: {avg_time:.2f}s\")\n",
+    "\n",
+    "print(f\"\\n{'='*100}\")\n",
+    "print(\"✅ Comprehensive benchmark complete!\")\n",
+    "print(f\"{'='*100}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 8. Analyze Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create DataFrame\n",
+    "df = pd.DataFrame(results)\n",
+    "\n",
+    "# Aggregate by configuration\n",
+    "config_summary = df.groupby('Config').agg({\n",
+    "    'LLM_Judge_Score': 'mean',\n",
+    "    'Accuracy_Score': 'mean',\n",
+    "    'Citation_Score': 'mean',\n",
+    "    'Retrieval_Relevance': 'mean',\n",
+    "    'Completeness_Score': 'mean',\n",
+    "    'Response_Time': 'mean',\n",
+    "    'Embedding_Model': 'first',\n",
+    "    'Retrieval_Strategy': 'first',\n",
+    "    'LLM_Model': 'first',\n",
+    "    'Prompt_Strategy': 'first'\n",
+    "}).round(2)\n",
+    "\n",
+    "# Sort by LLM Judge Score\n",
+    "config_summary = config_summary.sort_values('LLM_Judge_Score', ascending=False)\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*120)\n",
+    "print(\"📊 CONFIGURATION RANKINGS (By LLM Judge Score)\")\n",
+    "print(\"=\"*120)\n",
+    "display_cols = ['Embedding_Model', 'Retrieval_Strategy', 'LLM_Model', 'Prompt_Strategy', \n",
+    "                'LLM_Judge_Score', 'Accuracy_Score', 'Citation_Score', 'Response_Time']\n",
+    "print(config_summary[display_cols].to_string())\n",
+    "print(\"=\"*120)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 9. Component Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Analyze impact of each component\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"🔍 COMPONENT IMPACT ANALYSIS\")\n",
+    "print(\"=\"*100)\n",
+    "\n",
+    "# 1. Embedding Models\n",
+    "print(\"\\n📚 EMBEDDING MODELS:\")\n",
+    "embed_impact = df.groupby('Embedding_Model')['LLM_Judge_Score'].mean().sort_values(ascending=False)\n",
+    "for model, score in embed_impact.items():\n",
+    "    print(f\"   {model}: {score:.2f}%\")\n",
+    "\n",
+    "# 2. Retrieval Strategies\n",
+    "print(\"\\n🔎 RETRIEVAL STRATEGIES:\")\n",
+    "retrieval_impact = df.groupby('Retrieval_Strategy')['LLM_Judge_Score'].mean().sort_values(ascending=False)\n",
+    "for strategy, score in retrieval_impact.items():\n",
+    "    notes = RETRIEVAL_STRATEGIES[strategy]['notes']\n",
+    "    print(f\"   {strategy}: {score:.2f}% ({notes})\")\n",
+    "\n",
+    "# 3. LLM Models\n",
+    "print(\"\\n🤖 LLM MODELS:\")\n",
+    "llm_impact = df.groupby('LLM_Model')['LLM_Judge_Score'].mean().sort_values(ascending=False)\n",
+    "for model, score in llm_impact.items():\n",
+    "    print(f\"   {model}: {score:.2f}%\")\n",
+    "\n",
+    "# 4. Prompting Strategies\n",
+    "print(\"\\n💬 PROMPTING STRATEGIES:\")\n",
+    "prompt_impact = df.groupby('Prompt_Strategy')['LLM_Judge_Score'].mean().sort_values(ascending=False)\n",
+    "for strategy, score in prompt_impact.items():\n",
+    "    print(f\"   {strategy}: {score:.2f}%\")\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Create output directory - using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'rag_optimization_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "fig, axes = plt.subplots(2, 3, figsize=(20, 12))\n",
+    "\n",
+    "# 1. Top Configurations\n",
+    "ax1 = axes[0, 0]\n",
+    "top_configs = config_summary.head(10)\n",
+    "config_labels = [c.split('_')[-2] + '+' + c.split('_')[-1] for c in top_configs.index]\n",
+    "ax1.barh(config_labels, top_configs['LLM_Judge_Score'], color=sns.color_palette('viridis', len(top_configs)))\n",
+    "ax1.set_xlabel('LLM Judge Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax1.set_title('Top 10 Configurations', fontsize=13, fontweight='bold')\n",
+    "ax1.set_xlim(0, 100)\n",
+    "for i, score in enumerate(top_configs['LLM_Judge_Score']):\n",
+    "    ax1.text(score + 1, i, f'{score:.1f}', va='center', fontsize=10)\n",
+    "\n",
+    "# 2. Embedding Model Impact\n",
+    "ax2 = axes[0, 1]\n",
+    "ax2.bar(embed_impact.index, embed_impact.values, color='skyblue', alpha=0.8)\n",
+    "ax2.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax2.set_title('Embedding Model Impact', fontsize=13, fontweight='bold')\n",
+    "ax2.set_ylim(0, 100)\n",
+    "ax2.tick_params(axis='x', rotation=45)\n",
+    "for i, (model, score) in enumerate(embed_impact.items()):\n",
+    "    ax2.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n",
+    "\n",
+    "# 3. Retrieval Strategy Impact\n",
+    "ax3 = axes[0, 2]\n",
+    "ax3.bar(retrieval_impact.index, retrieval_impact.values, color='coral', alpha=0.8)\n",
+    "ax3.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax3.set_title('Retrieval Strategy Impact', fontsize=13, fontweight='bold')\n",
+    "ax3.set_ylim(0, 100)\n",
+    "ax3.tick_params(axis='x', rotation=45)\n",
+    "for i, (strategy, score) in enumerate(retrieval_impact.items()):\n",
+    "    ax3.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=9)\n",
+    "\n",
+    "# 4. LLM Model Impact\n",
+    "ax4 = axes[1, 0]\n",
+    "ax4.bar(llm_impact.index, llm_impact.values, color='mediumseagreen', alpha=0.8)\n",
+    "ax4.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax4.set_title('LLM Model Impact', fontsize=13, fontweight='bold')\n",
+    "ax4.set_ylim(0, 100)\n",
+    "ax4.tick_params(axis='x', rotation=45)\n",
+    "for i, (model, score) in enumerate(llm_impact.items()):\n",
+    "    ax4.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n",
+    "\n",
+    "# 5. Prompting Strategy Impact\n",
+    "ax5 = axes[1, 1]\n",
+    "ax5.bar(prompt_impact.index, prompt_impact.values, color='mediumpurple', alpha=0.8)\n",
+    "ax5.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax5.set_title('Prompting Strategy Impact', fontsize=13, fontweight='bold')\n",
+    "ax5.set_ylim(0, 100)\n",
+    "ax5.tick_params(axis='x', rotation=45)\n",
+    "for i, (strategy, score) in enumerate(prompt_impact.items()):\n",
+    "    ax5.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n",
+    "\n",
+    "# 6. Score Components (best config)\n",
+    "ax6 = axes[1, 2]\n",
+    "best_config = config_summary.iloc[0]\n",
+    "components = ['Accuracy', 'Citation', 'Completeness']\n",
+    "scores = [best_config['Accuracy_Score'], best_config['Citation_Score'], best_config['Completeness_Score']]\n",
+    "colors_comp = ['#FF6B6B', '#4ECDC4', '#45B7D1']\n",
+    "bars = ax6.bar(components, scores, color=colors_comp, alpha=0.8)\n",
+    "ax6.set_ylabel('Score (%)', fontsize=11, fontweight='bold')\n",
+    "ax6.set_title(f'Best Config Components\\n{best_config.name.split(\"_\")[2]}', fontsize=13, fontweight='bold')\n",
+    "ax6.set_ylim(0, 100)\n",
+    "for i, score in enumerate(scores):\n",
+    "    ax6.text(i, score + 2, f'{score:.1f}%', ha='center', fontsize=10, fontweight='bold')\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(output_dir / 'results.png', dpi=300, bbox_inches='tight')\n",
+    "plt.show()\n",
+    "\n",
+    "print(f\"\\n✅ Visualization saved to '{output_dir}/results.png'\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": "import os\nfrom pathlib import Path\n\n# Create output directory - using dynamic path\noutput_dir = OUTPUT_DIR / 'rag_optimization_benchmark'\noutput_dir.mkdir(parents=True, exist_ok=True)\n\nfig, axes = plt.subplots(2, 3, figsize=(20, 12))\n\n# 1. Top Configurations\nax1 = axes[0, 0]\ntop_configs = config_summary.head(10)\nconfig_labels = [c.split('_')[-2] + '+' + c.split('_')[-1] for c in top_configs.index]\nax1.barh(config_labels, top_configs['LLM_Judge_Score'], color=sns.color_palette('viridis', len(top_configs)))\nax1.set_xlabel('LLM Judge Score (%)', fontsize=11, fontweight='bold')\nax1.set_title('Top 10 Configurations', fontsize=13, fontweight='bold')\nax1.set_xlim(0, 100)\nfor i, score in enumerate(top_configs['LLM_Judge_Score']):\n    ax1.text(score + 1, i, f'{score:.1f}', va='center', fontsize=10)\n\n# 2. Embedding Model Impact\nax2 = axes[0, 1]\nax2.bar(embed_impact.index, embed_impact.values, color='skyblue', alpha=0.8)\nax2.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\nax2.set_title('Embedding Model Impact', fontsize=13, fontweight='bold')\nax2.set_ylim(0, 100)\nax2.tick_params(axis='x', rotation=45)\nfor i, (model, score) in enumerate(embed_impact.items()):\n    ax2.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n\n# 3. Retrieval Strategy Impact\nax3 = axes[0, 2]\nax3.bar(retrieval_impact.index, retrieval_impact.values, color='coral', alpha=0.8)\nax3.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\nax3.set_title('Retrieval Strategy Impact', fontsize=13, fontweight='bold')\nax3.set_ylim(0, 100)\nax3.tick_params(axis='x', rotation=45)\nfor i, (strategy, score) in enumerate(retrieval_impact.items()):\n    ax3.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=9)\n\n# 4. LLM Model Impact\nax4 = axes[1, 0]\nax4.bar(llm_impact.index, llm_impact.values, color='mediumseagreen', alpha=0.8)\nax4.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\nax4.set_title('LLM Model Impact', fontsize=13, fontweight='bold')\nax4.set_ylim(0, 100)\nax4.tick_params(axis='x', rotation=45)\nfor i, (model, score) in enumerate(llm_impact.items()):\n    ax4.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n\n# 5. Prompting Strategy Impact\nax5 = axes[1, 1]\nax5.bar(prompt_impact.index, prompt_impact.values, color='mediumpurple', alpha=0.8)\nax5.set_ylabel('Avg LLM Judge Score (%)', fontsize=11, fontweight='bold')\nax5.set_title('Prompting Strategy Impact', fontsize=13, fontweight='bold')\nax5.set_ylim(0, 100)\nax5.tick_params(axis='x', rotation=45)\nfor i, (strategy, score) in enumerate(prompt_impact.items()):\n    ax5.text(i, score + 2, f'{score:.1f}', ha='center', fontsize=10)\n\n# 6. Score Components (best config)\nax6 = axes[1, 2]\nbest_config = config_summary.iloc[0]\ncomponents = ['Accuracy', 'Citation', 'Completeness']\nscores = [best_config['Accuracy_Score'], best_config['Citation_Score'], best_config['Completeness_Score']]\ncolors_comp = ['#FF6B6B', '#4ECDC4', '#45B7D1']\nbars = ax6.bar(components, scores, color=colors_comp, alpha=0.8)\nax6.set_ylabel('Score (%)', fontsize=11, fontweight='bold')\nax6.set_title(f'Best Config Components\\n{best_config.name.split(\"_\")[2]}', fontsize=13, fontweight='bold')\nax6.set_ylim(0, 100)\nfor i, score in enumerate(scores):\n    ax6.text(i, score + 2, f'{score:.1f}%', ha='center', fontsize=10, fontweight='bold')\n\nplt.tight_layout()\nplt.savefig(output_dir / 'results.png', dpi=300, bbox_inches='tight')\nplt.show()\n\nprint(f\"\\n✅ Visualization saved to '{output_dir}/results.png'\")"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "best_config = config_summary.iloc[0]\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"🏆 OPTIMAL RAG CONFIGURATION\")\n",
+    "print(\"=\"*100)\n",
+    "\n",
+    "print(f\"\\n✅ Best Configuration: {best_config.name}\")\n",
+    "print(f\"\\n📊 Performance:\")\n",
+    "print(f\"   LLM Judge Score: {best_config['LLM_Judge_Score']:.2f}%\")\n",
+    "print(f\"   Accuracy: {best_config['Accuracy_Score']:.2f}%\")\n",
+    "print(f\"   Citation Quality: {best_config['Citation_Score']:.2f}%\")\n",
+    "print(f\"   Completeness: {best_config['Completeness_Score']:.2f}%\")\n",
+    "print(f\"   Avg Response Time: {best_config['Response_Time']:.2f}s\")\n",
+    "\n",
+    "print(f\"\\n⚙️ Components:\")\n",
+    "print(f\"   Embedding Model: {best_config['Embedding_Model']}\")\n",
+    "print(f\"      → {EMBEDDING_MODELS[best_config['Embedding_Model']]['name']}\")\n",
+    "print(f\"   Retrieval Strategy: {best_config['Retrieval_Strategy']}\")\n",
+    "print(f\"      → {RETRIEVAL_STRATEGIES[best_config['Retrieval_Strategy']]['notes']}\")\n",
+    "print(f\"   LLM Model: {best_config['LLM_Model']}\")\n",
+    "print(f\"   Prompting Strategy: {best_config['Prompt_Strategy']}\")\n",
+    "\n",
+    "print(f\"\\n💡 Key Findings:\")\n",
+    "print(f\"   1. Best Embedding: {embed_impact.index[0]} ({embed_impact.values[0]:.2f}%)\")\n",
+    "print(f\"   2. Best Retrieval: {retrieval_impact.index[0]} ({retrieval_impact.values[0]:.2f}%)\")\n",
+    "print(f\"   3. Best LLM: {llm_impact.index[0]} ({llm_impact.values[0]:.2f}%)\")\n",
+    "print(f\"   4. Best Prompt: {prompt_impact.index[0]} ({prompt_impact.values[0]:.2f}%)\")\n",
+    "\n",
+    "print(f\"\\n🎯 Hackathon Impact:\")\n",
+    "print(f\"   LLM Quality = 30% of total score\")\n",
+    "print(f\"   Your score: {best_config['LLM_Judge_Score']:.2f}% × 30% = {best_config['LLM_Judge_Score'] * 0.3:.2f} points\")\n",
+    "\n",
+    "baseline = df[df['Config'].str.contains('baseline')].iloc[0] if len(df[df['Config'].str.contains('baseline')]) > 0 else None\n",
+    "if baseline is not None:\n",
+    "    improvement = best_config['LLM_Judge_Score'] - baseline['LLM_Judge_Score']\n",
+    "    print(f\"\\n📈 Improvement vs Baseline:\")\n",
+    "    print(f\"   +{improvement:.2f}% quality improvement\")\n",
+    "    print(f\"   = +{improvement * 0.3:.2f} hackathon points\")\n",
+    "\n",
+    "print(\"\\n\" + \"=\"*100)\n",
+    "print(\"📝 IMPLEMENTATION CHECKLIST\")\n",
+    "print(\"=\"*100)\n",
+    "print(f\"\\n1. Use embedding model: {EMBEDDING_MODELS[best_config['Embedding_Model']]['name']}\")\n",
+    "print(f\"2. Implement retrieval: {best_config['Retrieval_Strategy']}\")\n",
+    "print(f\"3. Use LLM model: {best_config['LLM_Model']}\")\n",
+    "print(f\"4. Apply prompt: {best_config['Prompt_Strategy']}\")\n",
+    "print(f\"\\n5. Expected performance:\")\n",
+    "print(f\"   - LLM Judge Score: {best_config['LLM_Judge_Score']:.2f}%\")\n",
+    "print(f\"   - Response time: ~{best_config['Response_Time']:.1f}s\")\n",
+    "print(\"=\"*100)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Save results\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'rag_optimization_benchmark'\n",
+    "output_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "df.to_csv(output_dir / 'detailed_results.csv', index=False, encoding='utf-8')\n",
+    "config_summary.to_csv(output_dir / 'summary.csv', encoding='utf-8')\n",
+    "\n",
+    "# Save component impacts\n",
+    "impacts = pd.DataFrame({\n",
+    "    'Embedding_Impact': embed_impact,\n",
+    "    'Retrieval_Impact': retrieval_impact.reindex(embed_impact.index, fill_value=0),\n",
+    "    'LLM_Impact': llm_impact.reindex(embed_impact.index, fill_value=0),\n",
+    "    'Prompt_Impact': prompt_impact.reindex(embed_impact.index, fill_value=0)\n",
+    "}).fillna(0)\n",
+    "impacts.to_csv(output_dir / 'component_impacts.csv', encoding='utf-8')\n",
+    "\n",
+    "print(\"\\n✅ Results exported to output/rag_optimization_benchmark/:\")\n",
+    "print(\"   - detailed_results.csv (all tests)\")\n",
+    "print(\"   - summary.csv (config rankings)\")\n",
+    "print(\"   - component_impacts.csv (component analysis)\")\n",
+    "print(\"   - results.png (visualizations)\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/vlm_ocr_benchmark.ipynb

@@ -31,7 +31,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -81,15 +81,58 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": 3,
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Project root: /Users/ismatsamadov/SOCAR_Hackathon\n",
+      "✅ Data directory: /Users/ismatsamadov/SOCAR_Hackathon/data\n",
+      "✅ PDFs directory: /Users/ismatsamadov/SOCAR_Hackathon/data/pdfs\n",
+      "✅ Output directory: /Users/ismatsamadov/SOCAR_Hackathon/output\n"
+     ]
+    }
+   ],
    "source": [
-    "## 1. Load Ground Truth"
+    "# Auto-detect project root (works from any directory)\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if Path('data').exists() and Path('docs').exists():\n",
+    "    # Already in project root\n",
+    "    PROJECT_ROOT = Path.cwd()\n",
+    "elif Path('../data').exists() and Path('../docs').exists():\n",
+    "    # In notebooks/ subdirectory\n",
+    "    PROJECT_ROOT = Path.cwd().parent\n",
+    "else:\n",
+    "    # Fallback: try to find project root\n",
+    "    current = Path.cwd()\n",
+    "    while current != current.parent:\n",
+    "        if (current / 'data').exists() and (current / 'docs').exists():\n",
+    "            PROJECT_ROOT = current\n",
+    "            break\n",
+    "        current = current.parent\n",
+    "    else:\n",
+    "        PROJECT_ROOT = Path.cwd()\n",
+    "\n",
+    "# Define all paths relative to project root\n",
+    "DATA_DIR = PROJECT_ROOT / 'data'\n",
+    "DOCS_DIR = PROJECT_ROOT / 'docs'\n",
+    "OUTPUT_DIR = PROJECT_ROOT / 'output'\n",
+    "PDFS_DIR = DATA_DIR / 'pdfs'\n",
+    "\n",
+    "print(f\"✅ Project root: {PROJECT_ROOT}\")\n",
+    "print(f\"✅ Data directory: {DATA_DIR}\")\n",
+    "print(f\"✅ PDFs directory: {PDFS_DIR}\")\n",
+    "print(f\"✅ Output directory: {OUTPUT_DIR}\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -122,22 +165,15 @@
     "    \n",
     "    return text.strip()\n",
     "\n",
-    "# Load ground truth\n",
-    "ground_truth = load_ground_truth('data/document_00.md')\n",
+    "# Load ground truth - using dynamic path\n",
+    "ground_truth = load_ground_truth(str(DATA_DIR / 'document_00.md'))\n",
     "print(f\"✅ Ground truth loaded: {len(ground_truth)} characters\")\n",
     "print(f\"Preview:\\n{ground_truth[:300]}...\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 2. PDF to Image Conversion"
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -181,23 +217,16 @@
     "    image.save(buffered, format=format)\n",
     "    return base64.b64encode(buffered.getvalue()).decode('utf-8')\n",
     "\n",
-    "# Test conversion\n",
-    "pdf_path = 'data/pdfs/document_00.pdf'\n",
+    "# Test conversion - using dynamic path\n",
+    "pdf_path = str(PDFS_DIR / 'document_00.pdf')\n",
     "test_images = pdf_to_images(pdf_path)\n",
     "print(f\"\\n✅ Converted PDF to {len(test_images)} images\")\n",
     "print(f\"First image size: {test_images[0].size}\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 3. Vision-Language Model Client"
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -261,7 +290,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -370,68 +399,6 @@
     "## 4. Metrics Calculation"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Metrics functions ready\n"
-     ]
-    }
-   ],
-   "source": [
-    "def normalize_text(text: str) -> str:\n",
-    "    \"\"\"Normalize text for comparison.\"\"\"\n",
-    "    text = text.lower().strip()\n",
-    "    text = re.sub(r'\\s+', ' ', text)\n",
-    "    return text\n",
-    "\n",
-    "def calculate_ocr_metrics(reference: str, hypothesis: str) -> Dict[str, float]:\n",
-    "    \"\"\"\n",
-    "    Calculate comprehensive OCR metrics.\n",
-    "    \"\"\"\n",
-    "    ref_norm = normalize_text(reference)\n",
-    "    hyp_norm = normalize_text(hypothesis)\n",
-    "    \n",
-    "    # Character Error Rate\n",
-    "    cer_score = cer(ref_norm, hyp_norm) * 100\n",
-    "    \n",
-    "    # Word Error Rate\n",
-    "    wer_score = wer(ref_norm, hyp_norm) * 100\n",
-    "    \n",
-    "    # Success rates\n",
-    "    csr_score = max(0, 100 - cer_score)\n",
-    "    wsr_score = max(0, 100 - wer_score)\n",
-    "    \n",
-    "    # Length metrics\n",
-    "    ref_chars = len(ref_norm)\n",
-    "    hyp_chars = len(hyp_norm)\n",
-    "    ref_words = len(ref_norm.split())\n",
-    "    hyp_words = len(hyp_norm.split())\n",
-    "    \n",
-    "    char_length_acc = (min(ref_chars, hyp_chars) / max(ref_chars, hyp_chars) * 100) if max(ref_chars, hyp_chars) > 0 else 0\n",
-    "    word_length_acc = (min(ref_words, hyp_words) / max(ref_words, hyp_words) * 100) if max(ref_words, hyp_words) > 0 else 0\n",
-    "    \n",
-    "    return {\n",
-    "        'CER': round(cer_score, 2),\n",
-    "        'WER': round(wer_score, 2),\n",
-    "        'CSR': round(csr_score, 2),\n",
-    "        'WSR': round(wsr_score, 2),\n",
-    "        'Char_Count_Ref': ref_chars,\n",
-    "        'Char_Count_Hyp': hyp_chars,\n",
-    "        'Word_Count_Ref': ref_words,\n",
-    "        'Word_Count_Hyp': hyp_words,\n",
-    "        'Char_Length_Accuracy': round(char_length_acc, 2),\n",
-    "        'Word_Length_Accuracy': round(word_length_acc, 2)\n",
-    "    }\n",
-    "\n",
-    "print(\"✅ Metrics functions ready\")"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -441,7 +408,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -471,7 +438,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -486,9 +453,68 @@
       "Notes: Excellent OCR\n",
       "================================================================================\n",
       "  Page 1/12: 9.2s\n",
-      "  Page 2/12: 9.5s\n",
-      "  Page 3/12: 10.9s\n",
-      "  Page 4/12: 10.7s\n"
+      "  Page 2/12: 8.8s\n",
+      "  Page 3/12: 9.7s\n",
+      "  Page 4/12: 10.7s\n",
+      "  Page 5/12: 12.4s\n",
+      "  Page 6/12: 9.9s\n",
+      "  Page 7/12: 12.1s\n",
+      "  Page 8/12: 10.7s\n",
+      "  Page 9/12: 12.0s\n",
+      "  Page 10/12: 10.9s\n",
+      "  Page 11/12: 7.7s\n",
+      "  Page 12/12: 6.6s\n",
+      "\n",
+      "✅ Total time: 120.73s\n",
+      "✅ Extracted: 22018 characters\n",
+      "\n",
+      "📊 Metrics:\n",
+      "   CSR (Character Success): 86.17%\n",
+      "   WSR (Word Success): 68.19%\n",
+      "   CER (Character Error): 13.83%\n",
+      "   WER (Word Error): 31.81%\n",
+      "\n",
+      "================================================================================\n",
+      "Testing: GPT-5 ⭐⭐⭐⭐⭐\n",
+      "Notes: Latest model\n",
+      "================================================================================\n",
+      "❌ Failed: ERROR: Completions.create() got an unexpected keyword argument 'max_completion_tokens'\n",
+      "\n",
+      "================================================================================\n",
+      "Testing: GPT-5-mini ⭐⭐⭐⭐⭐\n",
+      "Notes: Fast + excellent\n",
+      "================================================================================\n",
+      "❌ Failed: ERROR: Completions.create() got an unexpected keyword argument 'max_completion_tokens'\n",
+      "\n",
+      "================================================================================\n",
+      "Testing: Claude-Sonnet-4.5 ⭐⭐⭐⭐⭐\n",
+      "Notes: Very good OCR\n",
+      "================================================================================\n",
+      "❌ Failed: ERROR: Error code: 400 - {'error': {'code': 'unknown_model', 'message': 'Unknown model: claude-sonnet-4-5', 'details': 'Unknown model: claude-sonnet-4-5'}}\n",
+      "\n",
+      "================================================================================\n",
+      "Testing: Phi-4-multimodal ⭐⭐⭐⭐\n",
+      "Notes: Explicitly multimodal\n",
+      "================================================================================\n",
+      "  Page 1/12: 32.6s\n",
+      "  Page 2/12: 3.9s\n",
+      "  Page 3/12: 76.4s\n",
+      "  Page 4/12: 6.0s\n",
+      "❌ Failed: ERROR: Error code: 500 - {'statusCode': 500, 'message': 'Internal server error', 'activityId': '22657b68-c10c-4f10-b8d7-4f5d7e1d963b'}\n",
+      "\n",
+      "================================================================================\n",
+      "Testing: Llama-4-Maverick-17B ⭐⭐⭐⭐\n",
+      "Notes: Testing vision capability\n",
+      "================================================================================\n",
+      "  Page 1/12: 6.8s\n",
+      "  Page 2/12: 7.3s\n",
+      "  Page 3/12: 9.0s\n",
+      "  Page 4/12: 9.7s\n",
+      "❌ Failed: ERROR: Error code: 500 - {'statusCode': 500, 'message': 'Internal server error', 'activityId': '4a2cd3c9-b21e-4efd-a273-1a4a5e99e2c4'}\n",
+      "\n",
+      "================================================================================\n",
+      "✅ VLM OCR Benchmarking complete!\n",
+      "================================================================================\n"
      ]
     }
    ],
@@ -552,7 +578,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -564,7 +590,7 @@
       "📊 VLM OCR BENCHMARKING RESULTS\n",
       "====================================================================================================\n",
       "  Model   CSR   WSR   CER   WER  Response_Time Rating\n",
-      "GPT-4.1 85.86 67.61 14.14 32.39         133.49  ⭐⭐⭐⭐⭐\n",
+      "GPT-4.1 86.17 68.19 13.83 31.81         120.73  ⭐⭐⭐⭐⭐\n",
       "====================================================================================================\n"
      ]
     }
@@ -586,13 +612,6 @@
     "print(\"=\"*100)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 7. Visualizations"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -603,8 +622,8 @@
     "import os\n",
     "from pathlib import Path\n",
     "\n",
-    "# Create output directory\n",
-    "output_dir = Path('output/vlm_ocr_benchmark')\n",
+    "# Create output directory - using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'vlm_ocr_benchmark'\n",
     "output_dir.mkdir(parents=True, exist_ok=True)\n",
     "\n",
     "fig, axes = plt.subplots(2, 2, figsize=(16, 10))\n",
@@ -662,133 +681,6 @@
     "print(f\"\\n✅ Visualization saved to '{output_dir}/results.png'\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 8. Winner Analysis and Recommendations"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "====================================================================================================\n",
-      "🏆 FINAL RANKINGS\n",
-      "====================================================================================================\n",
-      " Rank   Model   CSR   WSR   CER   WER  Response_Time Rating\n",
-      "    1 GPT-4.1 85.86 67.61 14.14 32.39         133.49  ⭐⭐⭐⭐⭐\n",
-      "====================================================================================================\n",
-      "\n",
-      "====================================================================================================\n",
-      "💡 RECOMMENDATIONS FOR HACKATHON\n",
-      "====================================================================================================\n",
-      "\n",
-      "🥇 BEST OVERALL: GPT-4.1 ⭐⭐⭐⭐⭐\n",
-      "   CSR: 85.86% (Character Success)\n",
-      "   WSR: 67.61% (Word Success)\n",
-      "   CER: 14.14% (Character Error)\n",
-      "   WER: 32.39% (Word Error)\n",
-      "   Time: 133.49s for 12 pages\n",
-      "   Notes: Excellent OCR\n",
-      "\n",
-      "⚡ FASTEST: GPT-4.1\n",
-      "   Time: 133.49s\n",
-      "   CSR: 85.86%\n",
-      "\n",
-      "====================================================================================================\n",
-      "📝 HACKATHON SCORING IMPACT\n",
-      "====================================================================================================\n",
-      "\n",
-      "OCR Quality = 50% of total hackathon score\n",
-      "\n",
-      "Using GPT-4.1:\n",
-      "  - CSR: 85.86% × 50% = 42.93 points\n",
-      "  - This is 85.9% accuracy on character-level OCR\n",
-      "\n",
-      "⚠️ GOOD - Consider optimizing prompt or trying other models\n",
-      "\n",
-      "====================================================================================================\n",
-      "🎯 FINAL RECOMMENDATION\n",
-      "====================================================================================================\n",
-      "\n",
-      "Use: GPT-4.1\n",
-      "Reason: Highest accuracy (85.86% CSR) for hackathon OCR benchmark\n",
-      "Implementation: Use vision API with same prompt as tested\n",
-      "====================================================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Rankings\n",
-    "rankings = df[['Model', 'CSR', 'WSR', 'CER', 'WER', 'Response_Time', 'Rating']].copy()\n",
-    "rankings.insert(0, 'Rank', range(1, len(rankings) + 1))\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"🏆 FINAL RANKINGS\")\n",
-    "print(\"=\"*100)\n",
-    "print(rankings.to_string(index=False))\n",
-    "print(\"=\"*100)\n",
-    "\n",
-    "# Winner\n",
-    "best_model = df.iloc[0]\n",
-    "fastest_model = df.loc[df['Response_Time'].idxmin()]\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"💡 RECOMMENDATIONS FOR HACKATHON\")\n",
-    "print(\"=\"*100)\n",
-    "\n",
-    "print(f\"\\n🥇 BEST OVERALL: {best_model['Model']} {best_model['Rating']}\")\n",
-    "print(f\"   CSR: {best_model['CSR']:.2f}% (Character Success)\")\n",
-    "print(f\"   WSR: {best_model['WSR']:.2f}% (Word Success)\")\n",
-    "print(f\"   CER: {best_model['CER']:.2f}% (Character Error)\")\n",
-    "print(f\"   WER: {best_model['WER']:.2f}% (Word Error)\")\n",
-    "print(f\"   Time: {best_model['Response_Time']:.2f}s for {len(images)} pages\")\n",
-    "print(f\"   Notes: {best_model['Notes']}\")\n",
-    "\n",
-    "print(f\"\\n⚡ FASTEST: {fastest_model['Model']}\")\n",
-    "print(f\"   Time: {fastest_model['Response_Time']:.2f}s\")\n",
-    "print(f\"   CSR: {fastest_model['CSR']:.2f}%\")\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"📝 HACKATHON SCORING IMPACT\")\n",
-    "print(\"=\"*100)\n",
-    "print(\"\\nOCR Quality = 50% of total hackathon score\")\n",
-    "print(f\"\\nUsing {best_model['Model']}:\")\n",
-    "print(f\"  - CSR: {best_model['CSR']:.2f}% × 50% = {best_model['CSR'] * 0.5:.2f} points\")\n",
-    "print(f\"  - This is {best_model['CSR']:.1f}% accuracy on character-level OCR\")\n",
-    "\n",
-    "if best_model['CSR'] >= 95:\n",
-    "    print(\"\\n✅ EXCELLENT - This will score very high on OCR!\")\n",
-    "elif best_model['CSR'] >= 90:\n",
-    "    print(\"\\n✅ VERY GOOD - Strong OCR performance!\")\n",
-    "elif best_model['CSR'] >= 85:\n",
-    "    print(\"\\n⚠️ GOOD - Consider optimizing prompt or trying other models\")\n",
-    "else:\n",
-    "    print(\"\\n⚠️ NEEDS IMPROVEMENT - Try other models or adjust parameters\")\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"🎯 FINAL RECOMMENDATION\")\n",
-    "print(\"=\"*100)\n",
-    "print(f\"\\nUse: {best_model['Model']}\")\n",
-    "print(f\"Reason: Highest accuracy ({best_model['CSR']:.2f}% CSR) for hackathon OCR benchmark\")\n",
-    "print(f\"Implementation: Use vision API with same prompt as tested\")\n",
-    "print(\"=\"*100)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 9. Export Results"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -798,7 +690,8 @@
     "# Save results\n",
     "from pathlib import Path\n",
     "\n",
-    "output_dir = Path('output/vlm_ocr_benchmark')\n",
+    "# Using dynamic path\n",
+    "output_dir = OUTPUT_DIR / 'vlm_ocr_benchmark'\n",
     "output_dir.mkdir(parents=True, exist_ok=True)\n",
     "\n",
     "df.to_csv(output_dir / 'detailed_results.csv', index=False, encoding='utf-8')\n",
@@ -821,50 +714,8 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "====================================================================================================\n",
-      "📝 SAMPLE TEXT COMPARISON (First 500 characters)\n",
-      "====================================================================================================\n",
-      "\n",
-      "🎯 GROUND TRUTH:\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "XÜLASƏ\n",
-      "\n",
-      "Bu tədqiqat Aşağı Kür çökəkliyi (AKÇ) və Bakı arxipelaqı (BA) daxil olmaqla Cənubi Xəzər çökəkliyi sistemində faydalı qazıntıların mənşəyinin paleotektonik, paleocoğrafi şərait və geodinamik rejimlə necə əlaqələndiyini, eləcə də Gec Miosendən etibarən Ərəbistan plitəsinin təsiri ilə formalaşan kollizion proseslərin bölgənin struktur-morfoloji və termal inkişafına nə dərəcədə yönverici rol oynadığını kompleks şəkildə qiymətləndirir. Seismotektonik göstəricilərin, çöküntütoplanma sürətləri\n",
-      "\n",
-      "🤖 GPT-4.1 (CSR: 85.86%):\n",
-      "----------------------------------------------------------------------------------------------------\n",
-      "Xülasə\n",
-      "Bu məqalə Apşə Kər çökəkliyi (AKÇ) və Bakı arxipelaqı (BA) daxil olmaqla Cənubi Xəzər çökəkliyi sistemində faydalı qazıntıların mənşəyinin paleotektonik, paleocoğrafi şərait və geodinamik rejim\n",
-      "\n",
-      "====================================================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Show comparison of first 500 characters\n",
-    "print(\"\\n\" + \"=\"*100)\n",
-    "print(\"📝 SAMPLE TEXT COMPARISON (First 500 characters)\")\n",
-    "print(\"=\"*100)\n",
-    "\n",
-    "print(\"\\n🎯 GROUND TRUTH:\")\n",
-    "print(\"-\" * 100)\n",
-    "print(ground_truth[:500])\n",
-    "\n",
-    "for _, row in df.iterrows():\n",
-    "    print(f\"\\n🤖 {row['Model']} (CSR: {row['CSR']:.2f}%):\")\n",
-    "    print(\"-\" * 100)\n",
-    "    # Get first 500 chars from extracted text\n",
-    "    preview = row['Extracted_Preview'] if len(row['Extracted_Preview']) >= 500 else row['Extracted_Preview']\n",
-    "    print(preview[:500])\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*100)"
-   ]
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {