Upload main.ipynb

main.ipynb

@@ -0,0 +1,848 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2cec985b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data split into 70 training lines and 18 validation lines from ai_ethics_data.txt.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:07:19,311] A new study created in memory with name: gpt2_finetuning_ai_ethics\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "--- Starting Optuna hyperparameter optimization ---\n",
+      "A new study created in memory with name: gpt2_finetuning_ai_ethics\n",
+      "A new study created in memory with name: gpt2_finetuning_ai_ethics\n",
+      "A new study created in memory with name: gpt2_finetuning_ai_ethics\n",
+      "A new study created in memory with name: gpt2_finetuning_ai_ethics\n",
+      "\n",
+      "--- Starting training for trial 0 ---\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='42' max='42' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [42/42 01:01, Epoch 6/6]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.896929</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.772568</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.693300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.668611</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.660016</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.654525</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "There were missing keys in the checkpoint model loaded: ['lm_head.weight'].\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [1/1 : < :]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:08:28,746] Trial 0 finished with value: 14.218224606871981 and parameters: {'learning_rate': 4.257432830042159e-05, 'num_train_epochs': 6, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 1, 'weight_decay': 0.006021467020511296}. Best is trial 0 with value: 14.218224606871981.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trial 0 finished. Perplexity: 14.22\n",
+      "Trial 0 finished with value: 14.218224606871981 and parameters: {'learning_rate': 4.257432830042159e-05, 'num_train_epochs': 6, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 1, 'weight_decay': 0.006021467020511296}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 0 finished with value: 14.218224606871981 and parameters: {'learning_rate': 4.257432830042159e-05, 'num_train_epochs': 6, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 1, 'weight_decay': 0.006021467020511296}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 0 finished with value: 14.218224606871981 and parameters: {'learning_rate': 4.257432830042159e-05, 'num_train_epochs': 6, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 1, 'weight_decay': 0.006021467020511296}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 0 finished with value: 14.218224606871981 and parameters: {'learning_rate': 4.257432830042159e-05, 'num_train_epochs': 6, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 1, 'weight_decay': 0.006021467020511296}. Best is trial 0 with value: 14.218224606871981.\n",
+      "\n",
+      "--- Starting training for trial 1 ---\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='6' max='6' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [6/6 00:31, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>3.138857</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>3.090950</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>3.071213</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "There were missing keys in the checkpoint model loaded: ['lm_head.weight'].\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [1/1 : < :]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:09:11,356] Trial 1 finished with value: 21.568049007800685 and parameters: {'learning_rate': 1.984906349341712e-05, 'num_train_epochs': 3, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 4, 'weight_decay': 0.08258462935496225}. Best is trial 0 with value: 14.218224606871981.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trial 1 finished. Perplexity: 21.57\n",
+      "Trial 1 finished with value: 21.568049007800685 and parameters: {'learning_rate': 1.984906349341712e-05, 'num_train_epochs': 3, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 4, 'weight_decay': 0.08258462935496225}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 1 finished with value: 21.568049007800685 and parameters: {'learning_rate': 1.984906349341712e-05, 'num_train_epochs': 3, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 4, 'weight_decay': 0.08258462935496225}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 1 finished with value: 21.568049007800685 and parameters: {'learning_rate': 1.984906349341712e-05, 'num_train_epochs': 3, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 4, 'weight_decay': 0.08258462935496225}. Best is trial 0 with value: 14.218224606871981.\n",
+      "Trial 1 finished with value: 21.568049007800685 and parameters: {'learning_rate': 1.984906349341712e-05, 'num_train_epochs': 3, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 4, 'weight_decay': 0.08258462935496225}. Best is trial 0 with value: 14.218224606871981.\n",
+      "\n",
+      "--- Starting training for trial 2 ---\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='65' max='65' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [65/65 01:33, Epoch 5/5]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.830866</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.704903</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.650187</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>2.572700</td>\n",
+       "      <td>2.637062</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>2.572700</td>\n",
+       "      <td>2.630425</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "There were missing keys in the checkpoint model loaded: ['lm_head.weight'].\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [1/1 : < :]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:10:53,203] Trial 2 finished with value: 13.87967048839188 and parameters: {'learning_rate': 3.934727515529697e-05, 'num_train_epochs': 5, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.013016196478728849}. Best is trial 2 with value: 13.87967048839188.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trial 2 finished. Perplexity: 13.88\n",
+      "Trial 2 finished with value: 13.87967048839188 and parameters: {'learning_rate': 3.934727515529697e-05, 'num_train_epochs': 5, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.013016196478728849}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 2 finished with value: 13.87967048839188 and parameters: {'learning_rate': 3.934727515529697e-05, 'num_train_epochs': 5, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.013016196478728849}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 2 finished with value: 13.87967048839188 and parameters: {'learning_rate': 3.934727515529697e-05, 'num_train_epochs': 5, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.013016196478728849}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 2 finished with value: 13.87967048839188 and parameters: {'learning_rate': 3.934727515529697e-05, 'num_train_epochs': 5, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.013016196478728849}. Best is trial 2 with value: 13.87967048839188.\n",
+      "\n",
+      "--- Starting training for trial 3 ---\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='130' max='130' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [130/130 02:31, Epoch 10/10]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>3.009625</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.892823</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.799477</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>2.906200</td>\n",
+       "      <td>2.756954</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>2.906200</td>\n",
+       "      <td>2.726986</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>2.906200</td>\n",
+       "      <td>2.708734</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>2.906200</td>\n",
+       "      <td>2.693299</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>2.366800</td>\n",
+       "      <td>2.687046</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>2.366800</td>\n",
+       "      <td>2.682608</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>2.366800</td>\n",
+       "      <td>2.680843</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "There were missing keys in the checkpoint model loaded: ['lm_head.weight'].\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [1/1 : < :]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:13:34,833] Trial 3 finished with value: 14.597388429401072 and parameters: {'learning_rate': 1.4797090243167628e-05, 'num_train_epochs': 10, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.042400374321387935}. Best is trial 2 with value: 13.87967048839188.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trial 3 finished. Perplexity: 14.60\n",
+      "Trial 3 finished with value: 14.597388429401072 and parameters: {'learning_rate': 1.4797090243167628e-05, 'num_train_epochs': 10, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.042400374321387935}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 3 finished with value: 14.597388429401072 and parameters: {'learning_rate': 1.4797090243167628e-05, 'num_train_epochs': 10, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.042400374321387935}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 3 finished with value: 14.597388429401072 and parameters: {'learning_rate': 1.4797090243167628e-05, 'num_train_epochs': 10, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.042400374321387935}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 3 finished with value: 14.597388429401072 and parameters: {'learning_rate': 1.4797090243167628e-05, 'num_train_epochs': 10, 'per_device_train_batch_batch_size': 2, 'gradient_accumulation_steps': 1, 'weight_decay': 0.042400374321387935}. Best is trial 2 with value: 13.87967048839188.\n",
+      "\n",
+      "--- Starting training for trial 4 ---\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='16' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [16/16 00:39, Epoch 4/4]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>3.007116</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.916104</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.852073</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>2.832791</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "There were missing keys in the checkpoint model loaded: ['lm_head.weight'].\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [1/1 : < :]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[I 2025-07-16 15:14:24,143] Trial 4 finished with value: 16.992815093757653 and parameters: {'learning_rate': 3.624712025325302e-05, 'num_train_epochs': 4, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 2, 'weight_decay': 0.021517733040790188}. Best is trial 2 with value: 13.87967048839188.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trial 4 finished. Perplexity: 16.99\n",
+      "Trial 4 finished with value: 16.992815093757653 and parameters: {'learning_rate': 3.624712025325302e-05, 'num_train_epochs': 4, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 2, 'weight_decay': 0.021517733040790188}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 4 finished with value: 16.992815093757653 and parameters: {'learning_rate': 3.624712025325302e-05, 'num_train_epochs': 4, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 2, 'weight_decay': 0.021517733040790188}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 4 finished with value: 16.992815093757653 and parameters: {'learning_rate': 3.624712025325302e-05, 'num_train_epochs': 4, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 2, 'weight_decay': 0.021517733040790188}. Best is trial 2 with value: 13.87967048839188.\n",
+      "Trial 4 finished with value: 16.992815093757653 and parameters: {'learning_rate': 3.624712025325302e-05, 'num_train_epochs': 4, 'per_device_train_batch_batch_size': 4, 'gradient_accumulation_steps': 2, 'weight_decay': 0.021517733040790188}. Best is trial 2 with value: 13.87967048839188.\n",
+      "\n",
+      "--- Hyperparameter optimization complete ---\n",
+      "Best trial:\n",
+      "  Value (Perplexity): 13.88\n",
+      "  Params: \n",
+      "    learning_rate: 3.934727515529697e-05\n",
+      "    num_train_epochs: 5\n",
+      "    per_device_train_batch_batch_size: 2\n",
+      "    gradient_accumulation_steps: 1\n",
+      "    weight_decay: 0.013016196478728849\n",
+      "\n",
+      "--- Retraining final model with best hyperparameters ---\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\Anaconda\\Lib\\site-packages\\transformers\\data\\datasets\\language_modeling.py:53: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm.py\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='80' max='80' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [80/80 01:34, Epoch 5/5]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Final model fine-tuning complete and saved.\n",
+      "\n",
+      "--- Generating text with the fine-tuned model ---\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Prompt: The ethical implications of AI are\n",
+      "Generated Text:\n",
+      "The ethical implications of AI are complex, and the ethical consequences of autonomous systems are often complex.\n",
+      "Ethical Implications of Artificial Intelligence\n",
+      "AI ethics are important for several reasons:\n",
+      "1. It enables individuals to make informed decisions about their own well-being and their well being. This enables them to better understand and address societal challenges. 2. AI technologies can help individuals better manage their personal and social lives. 3. They can be used to address social and economic inequalities. 4. Ethical\n",
+      "\n",
+      "Cleaned up temporary data split files.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import GPT2Tokenizer, GPT2LMHeadModel, TextDataset, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",
+    "import math\n",
+    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
+    "import os\n",
+    "import random\n",
+    "import optuna\n",
+    "import logging\n",
+    "import sys\n",
+    "\n",
+    "optuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))\n",
+    "\n",
+    "file_path = \"ai_ethics_data.txt\"\n",
+    "\n",
+    "with open(file_path, \"r\", encoding=\"utf-8\") as f:\n",
+    "    full_text_content = f.read()\n",
+    "\n",
+    "train_file_path = \"ai_ethics_train.txt\"\n",
+    "val_file_path = \"ai_ethics_val.txt\"\n",
+    "\n",
+    "lines = full_text_content.strip().split('\\n')\n",
+    "lines = [line for line in lines if line.strip()]\n",
+    "random.shuffle(lines)\n",
+    "\n",
+    "train_size = int(len(lines) * 0.8)\n",
+    "train_lines = lines[:train_size]\n",
+    "val_lines = lines[train_size:]\n",
+    "\n",
+    "with open(train_file_path, \"w\", encoding=\"utf-8\") as f:\n",
+    "    f.write(\"\\n\".join(train_lines))\n",
+    "\n",
+    "with open(val_file_path, \"w\", encoding=\"utf-8\") as f:\n",
+    "    f.write(\"\\n\".join(val_lines))\n",
+    "\n",
+    "print(f\"Data split into {len(train_lines)} training lines and {len(val_lines)} validation lines from {file_path}.\")\n",
+    "\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
+    "\n",
+    "# 1. Fine-tuning the GPT-2 Model\n",
+    "\n",
+    "def model_init():\n",
+    "    model = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
+    "    model.resize_token_embeddings(len(tokenizer))\n",
+    "    model.config.pad_token_id = tokenizer.pad_token_id\n",
+    "    return model\n",
+    "\n",
+    "train_dataset = TextDataset(\n",
+    "    tokenizer=tokenizer,\n",
+    "    file_path=train_file_path,\n",
+    "    block_size=128 \n",
+    ")\n",
+    "val_dataset = TextDataset(\n",
+    "    tokenizer=tokenizer,\n",
+    "    file_path=val_file_path,\n",
+    "    block_size=128\n",
+    ")\n",
+    "\n",
+    "# Data collator for language modeling\n",
+    "data_collator = DataCollatorForLanguageModeling(\n",
+    "    tokenizer=tokenizer, mlm=False\n",
+    ")\n",
+    "\n",
+    "# 2. Hyperparameter Tuning with Optuna\n",
+    "\n",
+    "def objective(trial):\n",
+    "    learning_rate = trial.suggest_float(\"learning_rate\", 1e-5, 5e-5, log=True)\n",
+    "    num_train_epochs = trial.suggest_int(\"num_train_epochs\", 3, 10) # Reduced max epochs for faster tuning\n",
+    "    per_device_train_batch_size = trial.suggest_categorical(\"per_device_train_batch_batch_size\", [2, 4])\n",
+    "    gradient_accumulation_steps = trial.suggest_categorical(\"gradient_accumulation_steps\", [1, 2, 4])\n",
+    "    weight_decay = trial.suggest_float(\"weight_decay\", 0.0, 0.1)\n",
+    "\n",
+    "    output_dir = f\"./gpt2-finetuned-ai-ethics-trial-{trial.number}\"\n",
+    "\n",
+    "    training_args = TrainingArguments(\n",
+    "        output_dir=output_dir,\n",
+    "        overwrite_output_dir=True,\n",
+    "        num_train_epochs=num_train_epochs,\n",
+    "        per_device_train_batch_size=per_device_train_batch_size,\n",
+    "        gradient_accumulation_steps=gradient_accumulation_steps,\n",
+    "        save_steps=500, \n",
+    "        save_total_limit=1, \n",
+    "        prediction_loss_only=True,\n",
+    "        logging_dir='./logs',\n",
+    "        logging_steps=50,\n",
+    "        learning_rate=learning_rate,\n",
+    "        weight_decay=weight_decay,\n",
+    "\n",
+    "        eval_strategy=\"epoch\",\n",
+    "        save_strategy=\"epoch\", \n",
+    "        load_best_model_at_end=True, \n",
+    "        metric_for_best_model=\"eval_loss\", \n",
+    "        report_to=\"none\",\n",
+    "    )\n",
+    "\n",
+    "    trainer = Trainer(\n",
+    "        model_init=model_init, \n",
+    "        args=training_args,\n",
+    "        data_collator=data_collator,\n",
+    "        train_dataset=train_dataset,\n",
+    "        eval_dataset=val_dataset, \n",
+    "    )\n",
+    "\n",
+    "    print(f\"\\n--- Starting training for trial {trial.number} ---\")\n",
+    "    trainer.train()\n",
+    "\n",
+    "    results = trainer.evaluate(eval_dataset=val_dataset)\n",
+    "    eval_loss = results['eval_loss']\n",
+    "    perplexity = math.exp(eval_loss) if eval_loss < float('inf') else float('inf')\n",
+    "\n",
+    "    print(f\"Trial {trial.number} finished. Perplexity: {perplexity:.2f}\")\n",
+    "    return perplexity\n",
+    "\n",
+    "print(\"\\n--- Starting Optuna hyperparameter optimization ---\")\n",
+    "study = optuna.create_study(direction=\"minimize\", study_name=\"gpt2_finetuning_ai_ethics\")\n",
+    "study.optimize(objective, n_trials=5)\n",
+    "\n",
+    "print(\"\\n--- Hyperparameter optimization complete ---\")\n",
+    "print(\"Best trial:\")\n",
+    "trial = study.best_trial\n",
+    "print(f\"  Value (Perplexity): {trial.value:.2f}\")\n",
+    "print(\"  Params: \")\n",
+    "for key, value in trial.params.items():\n",
+    "    print(f\"    {key}: {value}\")\n",
+    "\n",
+    "# 3. Retrain with Best Hyperparameters (Final Model)\n",
+    "\n",
+    "print(\"\\n--- Retraining final model with best hyperparameters ---\")\n",
+    "\n",
+    "best_params = study.best_trial.params\n",
+    "\n",
+    "final_model = model_init()\n",
+    "\n",
+    "final_train_dataset = TextDataset(\n",
+    "    tokenizer=tokenizer,\n",
+    "    file_path=file_path,\n",
+    "    block_size=128\n",
+    ")\n",
+    "\n",
+    "final_training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2-finetuned-ai-ethics-final\",\n",
+    "    overwrite_output_dir=True,\n",
+    "    num_train_epochs=best_params[\"num_train_epochs\"],\n",
+    "    per_device_train_batch_size=best_params[\"per_device_train_batch_batch_size\"],\n",
+    "    gradient_accumulation_steps=best_params[\"gradient_accumulation_steps\"],\n",
+    "    save_steps=500, \n",
+    "    save_total_limit=2,\n",
+    "    prediction_loss_only=True,\n",
+    "    logging_dir='./logs_final',\n",
+    "    logging_steps=100,\n",
+    "    learning_rate=best_params[\"learning_rate\"],\n",
+    "    weight_decay=best_params[\"weight_decay\"],\n",
+    "    eval_strategy=\"no\", \n",
+    "    save_strategy=\"epoch\", \n",
+    "    load_best_model_at_end=False, \n",
+    "    report_to=\"none\",\n",
+    ")\n",
+    "\n",
+    "final_trainer = Trainer(\n",
+    "    model=final_model,\n",
+    "    args=final_training_args,\n",
+    "    data_collator=data_collator,\n",
+    "    train_dataset=final_train_dataset,\n",
+    ")\n",
+    "\n",
+    "final_trainer.train()\n",
+    "final_trainer.save_model(\"./gpt2-finetuned-ai-ethics-final\")\n",
+    "print(\"Final model fine-tuning complete and saved.\")\n",
+    "\n",
+    "# 4. Basic Text Generation Example (using the final model)\n",
+    "print(\"\\n--- Generating text with the fine-tuned model ---\")\n",
+    "\n",
+    "loaded_tokenizer = GPT2Tokenizer.from_pretrained(\"./gpt2-finetuned-ai-ethics-final\")\n",
+    "loaded_model = GPT2LMHeadModel.from_pretrained(\"./gpt2-finetuned-ai-ethics-final\")\n",
+    "\n",
+    "if loaded_tokenizer.pad_token is None:\n",
+    "    loaded_tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
+    "    loaded_model.resize_token_embeddings(len(loaded_tokenizer))\n",
+    "loaded_model.config.pad_token_id = loaded_tokenizer.pad_token_id\n",
+    "\n",
+    "prompt = \"The ethical implications of AI are\"\n",
+    "input_ids = loaded_tokenizer.encode(prompt, return_tensors='pt')\n",
+    "\n",
+    "output = loaded_model.generate(\n",
+    "    input_ids,\n",
+    "    max_length=100,\n",
+    "    num_return_sequences=1,\n",
+    "    no_repeat_ngram_size=2,\n",
+    "    top_k=50,\n",
+    "    top_p=0.95,\n",
+    "    temperature=0.7,\n",
+    "    pad_token_id=loaded_tokenizer.pad_token_id \n",
+    ")\n",
+    "\n",
+    "generated_text = loaded_tokenizer.decode(output[0], skip_special_tokens=True)\n",
+    "print(f\"Prompt: {prompt}\")\n",
+    "print(f\"Generated Text:\\n{generated_text}\")\n",
+    "\n",
+    "try:\n",
+    "    import nltk\n",
+    "    nltk.data.find('tokenizers/punkt')\n",
+    "except nltk.downloader.DownloadError:\n",
+    "    nltk.download('punkt')\n",
+    "\n",
+    "reference = [[\"The responsible development and deployment of AI are paramount for societal well-being.\"]]\n",
+    "candidate = generated_text.split()\n",
+    "score = sentence_bleu(reference, candidate, smoothing_function=SmoothingFunction().method1)\n",
+    "print(f\"\\nBLEU Score (example): {score:.2f}\")\n",
+    "\n",
+    "os.remove(train_file_path)\n",
+    "os.remove(val_file_path)\n",
+    "print(\"\\nCleaned up temporary data split files.\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}