full fine tuning code

Fully Fine Tuning BERT for QA.ipynb

@@ -0,0 +1,497 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "66bc0f06",
+   "metadata": {},
+   "source": [
+    "##  Install Dependencies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c214dba3-9553-4668-8582-b5edb7c13492",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting transformers\n",
+      "  Downloading transformers-4.49.0-py3-none-any.whl.metadata (44 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.0/44.0 kB\u001b[0m \u001b[31m968.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting datasets\n",
+      "  Downloading datasets-3.3.2-py3-none-any.whl.metadata (19 kB)\n",
+      "Collecting peft\n",
+      "  Downloading peft-0.14.0-py3-none-any.whl.metadata (13 kB)\n",
+      "Collecting accelerate\n",
+      "  Downloading accelerate-1.4.0-py3-none-any.whl.metadata (19 kB)\n",
+      "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
+      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.9.0)\n",
+      "Collecting huggingface-hub<1.0,>=0.26.0 (from transformers)\n",
+      "  Downloading huggingface_hub-0.29.1-py3-none-any.whl.metadata (13 kB)\n",
+      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.24.1)\n",
+      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
+      "Collecting regex!=2019.12.17 (from transformers)\n",
+      "  Downloading regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.5/40.5 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
+      "Collecting tokenizers<0.22,>=0.21 (from transformers)\n",
+      "  Downloading tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
+      "Collecting safetensors>=0.4.1 (from transformers)\n",
+      "  Downloading safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
+      "Collecting tqdm>=4.27 (from transformers)\n",
+      "  Downloading tqdm-4.67.1-py3-none-any.whl.metadata (57 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hCollecting pyarrow>=15.0.0 (from datasets)\n",
+      "  Downloading pyarrow-19.0.1-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
+      "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
+      "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
+      "Collecting pandas (from datasets)\n",
+      "  Downloading pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (89 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.9/89.9 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hCollecting requests (from transformers)\n",
+      "  Downloading requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
+      "Collecting xxhash (from datasets)\n",
+      "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
+      "Collecting multiprocess<0.70.17 (from datasets)\n",
+      "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
+      "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets) (2023.4.0)\n",
+      "Collecting aiohttp (from datasets)\n",
+      "  Downloading aiohttp-3.11.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
+      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft) (5.9.6)\n",
+      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.4.0)\n",
+      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
+      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.0)\n",
+      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
+      "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
+      "Collecting aiohappyeyeballs>=2.3.0 (from aiohttp->datasets)\n",
+      "  Downloading aiohappyeyeballs-2.4.6-py3-none-any.whl.metadata (5.9 kB)\n",
+      "Collecting aiosignal>=1.1.2 (from aiohttp->datasets)\n",
+      "  Downloading aiosignal-1.3.2-py2.py3-none-any.whl.metadata (3.8 kB)\n",
+      "Collecting async-timeout<6.0,>=4.0 (from aiohttp->datasets)\n",
+      "  Downloading async_timeout-5.0.1-py3-none-any.whl.metadata (5.1 kB)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
+      "Collecting frozenlist>=1.1.1 (from aiohttp->datasets)\n",
+      "  Downloading frozenlist-1.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
+      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets)\n",
+      "  Downloading multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.0 kB)\n",
+      "Collecting propcache>=0.2.0 (from aiohttp->datasets)\n",
+      "  Downloading propcache-0.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (10 kB)\n",
+      "Collecting yarl<2.0,>=1.17.0 (from aiohttp->datasets)\n",
+      "  Downloading yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (69 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.2/69.2 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hINFO: pip is looking at multiple versions of huggingface-hub to determine which version is compatible with other requirements. This could take a while.\n",
+      "Collecting huggingface-hub<1.0,>=0.26.0 (from transformers)\n",
+      "  Downloading huggingface_hub-0.29.0-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.28.1-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.28.0-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.27.1-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.27.0-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.26.5-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.26.3-py3-none-any.whl.metadata (13 kB)\n",
+      "INFO: pip is still looking at multiple versions of huggingface-hub to determine which version is compatible with other requirements. This could take a while.\n",
+      "  Downloading huggingface_hub-0.26.2-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.26.1-py3-none-any.whl.metadata (13 kB)\n",
+      "  Downloading huggingface_hub-0.26.0-py3-none-any.whl.metadata (13 kB)\n",
+      "Collecting fsspec[http]<=2024.12.0,>=2023.1.0 (from datasets)\n",
+      "  Downloading fsspec-2024.12.0-py3-none-any.whl.metadata (11 kB)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.1.1)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.13)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.2)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
+      "Collecting pytz>=2020.1 (from pandas->datasets)\n",
+      "  Downloading pytz-2025.1-py2.py3-none-any.whl.metadata (22 kB)\n",
+      "Collecting tzdata>=2022.7 (from pandas->datasets)\n",
+      "  Downloading tzdata-2025.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
+      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
+      "Downloading transformers-4.49.0-py3-none-any.whl (10.0 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.0/10.0 MB\u001b[0m \u001b[31m46.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading datasets-3.3.2-py3-none-any.whl (485 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m485.4/485.4 kB\u001b[0m \u001b[31m157.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading peft-0.14.0-py3-none-any.whl (374 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m374.8/374.8 kB\u001b[0m \u001b[31m128.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading accelerate-1.4.0-py3-none-any.whl (342 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m342.1/342.1 kB\u001b[0m \u001b[31m123.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m54.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading fsspec-2024.12.0-py3-none-any.whl (183 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.9/183.9 kB\u001b[0m \u001b[31m79.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading huggingface_hub-0.29.1-py3-none-any.whl (468 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.0/468.0 kB\u001b[0m \u001b[31m156.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading aiohttp-3.11.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m251.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m65.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading pyarrow-19.0.1-cp310-cp310-manylinux_2_28_x86_64.whl (42.1 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 MB\u001b[0m \u001b[31m83.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (781 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m781.7/781.7 kB\u001b[0m \u001b[31m179.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m31.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (471 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m471.6/471.6 kB\u001b[0m \u001b[31m164.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m225.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading tqdm-4.67.1-py3-none-any.whl (78 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.5/78.5 kB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.1 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.1/13.1 MB\u001b[0m \u001b[31m72.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m90.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading aiohappyeyeballs-2.4.6-py3-none-any.whl (14 kB)\n",
+      "Downloading aiosignal-1.3.2-py2.py3-none-any.whl (7.6 kB)\n",
+      "Downloading async_timeout-5.0.1-py3-none-any.whl (6.2 kB)\n",
+      "Downloading frozenlist-1.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (241 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m241.9/241.9 kB\u001b[0m \u001b[31m89.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (124 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.6/124.6 kB\u001b[0m \u001b[31m61.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading propcache-0.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (205 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m205.4/205.4 kB\u001b[0m \u001b[31m92.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading pytz-2025.1-py2.py3-none-any.whl (507 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m507.9/507.9 kB\u001b[0m \u001b[31m168.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading tzdata-2025.1-py2.py3-none-any.whl (346 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m346.8/346.8 kB\u001b[0m \u001b[31m130.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hDownloading yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (319 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m319.7/319.7 kB\u001b[0m \u001b[31m106.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: pytz, xxhash, tzdata, tqdm, safetensors, requests, regex, pyarrow, propcache, multidict, fsspec, frozenlist, dill, async-timeout, aiohappyeyeballs, yarl, pandas, multiprocess, huggingface-hub, aiosignal, tokenizers, aiohttp, accelerate, transformers, peft, datasets\n",
+      "  Attempting uninstall: requests\n",
+      "    Found existing installation: requests 2.31.0\n",
+      "    Uninstalling requests-2.31.0:\n",
+      "      Successfully uninstalled requests-2.31.0\n",
+      "  Attempting uninstall: fsspec\n",
+      "    Found existing installation: fsspec 2023.4.0\n",
+      "    Uninstalling fsspec-2023.4.0:\n",
+      "      Successfully uninstalled fsspec-2023.4.0\n",
+      "Successfully installed accelerate-1.4.0 aiohappyeyeballs-2.4.6 aiohttp-3.11.13 aiosignal-1.3.2 async-timeout-5.0.1 datasets-3.3.2 dill-0.3.8 frozenlist-1.5.0 fsspec-2024.12.0 huggingface-hub-0.29.1 multidict-6.1.0 multiprocess-0.70.16 pandas-2.2.3 peft-0.14.0 propcache-0.3.0 pyarrow-19.0.1 pytz-2025.1 regex-2024.11.6 requests-2.32.3 safetensors-0.5.3 tokenizers-0.21.0 tqdm-4.67.1 transformers-4.49.0 tzdata-2025.1 xxhash-3.5.0 yarl-1.18.3\n",
+      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
+      "\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%capture\n",
+    "\n",
+    "%pip install transformers datasets peft accelerate torch"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "367aeaca",
+   "metadata": {},
+   "source": [
+    "## Import Modules"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7df95981",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import torch\n",
+    "from transformers import BertTokenizerFast, BertForQuestionAnswering, Trainer, TrainingArguments, pipeline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e991901",
+   "metadata": {},
+   "source": [
+    "## Pre-Process The Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "282f340e-f1ba-4933-af49-0642863c01e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1594: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
+      "  warnings.warn(\n",
+      "/tmp/ipykernel_373/3878564307.py:83: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='585' max='585' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [585/585 02:00, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>0.309300</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=585, training_loss=0.26500008636050754, metrics={'train_runtime': 120.4137, 'train_samples_per_second': 77.632, 'train_steps_per_second': 4.858, 'total_flos': 2442602081968128.0, 'train_loss': 0.26500008636050754, 'epoch': 3.0})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Initialize the tokenizer\n",
+    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n",
+    "\n",
+    "def preprocess_data(data):\n",
+    "    tokenized_data = []\n",
+    "    for item in data:\n",
+    "        # Tokenize the question and context together with offset mapping\n",
+    "        inputs = tokenizer(\n",
+    "            item['question'],\n",
+    "            item['context'],\n",
+    "            max_length=512,\n",
+    "            truncation=True,\n",
+    "            padding='max_length',\n",
+    "            return_offsets_mapping=True,  # This is crucial\n",
+    "            return_tensors='pt'\n",
+    "        )\n",
+    "\n",
+    "        offset_mapping = inputs.pop('offset_mapping')  # Extract offset mapping\n",
+    "        input_ids = inputs['input_ids'].squeeze()  # Remove batch dimension\n",
+    "\n",
+    "        # Convert character indices to token indices for the answer\n",
+    "        start_char = item['answer_start_index']\n",
+    "        end_char = item['answer_end_index']\n",
+    "\n",
+    "        start_token_idx, end_token_idx = None, None\n",
+    "\n",
+    "        for i, (start, end) in enumerate(offset_mapping.squeeze().tolist()):\n",
+    "            if start_char >= start and start_char < end:\n",
+    "                start_token_idx = i\n",
+    "            if end_char > start and end_char <= end:\n",
+    "                end_token_idx = i\n",
+    "                break  # Stop once the end position is found\n",
+    "\n",
+    "        # Ensure valid token indices\n",
+    "        if start_token_idx is None or end_token_idx is None:\n",
+    "            continue  # Skip this example if indices are not found\n",
+    "\n",
+    "        tokenized_data.append({\n",
+    "            'input_ids': input_ids,\n",
+    "            'attention_mask': inputs['attention_mask'].squeeze(),\n",
+    "            'token_type_ids': inputs['token_type_ids'].squeeze(),\n",
+    "            'start_positions': torch.tensor([start_token_idx]),\n",
+    "            'end_positions': torch.tensor([end_token_idx])\n",
+    "        })\n",
+    "    \n",
+    "    return tokenized_data\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bd7c66f6",
+   "metadata": {},
+   "source": [
+    "## Load The Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1238434",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load dataset\n",
+    "def load_dataset(file_path):\n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        data = json.load(file)\n",
+    "    return data\n",
+    "\n",
+    "# Load your dataset\n",
+    "data = load_dataset('qa_london_data.json')\n",
+    "\n",
+    "# Preprocess the data\n",
+    "tokenized_datasets = preprocess_data(data)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9a0359a5",
+   "metadata": {},
+   "source": [
+    "## Train the Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cceb106c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Prepare model\n",
+    "model = BertForQuestionAnswering.from_pretrained('bert-base-uncased')\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "model.to(device)\n",
+    "\n",
+    "# Training arguments\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./results\",\n",
+    "    evaluation_strategy=\"no\",  # Disable evaluation\n",
+    "    learning_rate=2e-5,\n",
+    "    per_device_train_batch_size=16,\n",
+    "    per_device_eval_batch_size=16,\n",
+    "    num_train_epochs=3,\n",
+    "    weight_decay=0.01,\n",
+    ")\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_datasets,  # Only training dataset\n",
+    "    tokenizer=tokenizer,\n",
+    ")\n",
+    "\n",
+    "trainer.train()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "91b95765",
+   "metadata": {},
+   "source": [
+    "## Save The Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "49a99e11-a3de-42a7-a245-a352f1e70bab",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('./fine_tuned_bert/tokenizer_config.json',\n",
+       " './fine_tuned_bert/special_tokens_map.json',\n",
+       " './fine_tuned_bert/vocab.txt',\n",
+       " './fine_tuned_bert/added_tokens.json',\n",
+       " './fine_tuned_bert/tokenizer.json')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.save_pretrained(\"./fine_tuned_bert\")\n",
+    "tokenizer.save_pretrained(\"./fine_tuned_bert\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "79a2aeca",
+   "metadata": {},
+   "source": [
+    "## Test The Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "aa2ebe37-fbc6-4093-bbdb-497b8ac50b89",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use cuda:0\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'score': 0.9995142817497253, 'start': 67, 'end': 78, 'answer': 'guided tour'}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/question_answering.py:391: FutureWarning: Passing a list of SQuAD examples to the pipeline is deprecated and will be removed in v5. Inputs should be passed using the `question` and `context` keyword arguments instead.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load the fine-tuned model\n",
+    "qa_pipeline = pipeline(\"question-answering\", model=\"./fine_tuned_bert\", tokenizer=\"./fine_tuned_bert\")\n",
+    "\n",
+    "# Test on a sample question\n",
+    "result = qa_pipeline({\n",
+    "    \"question\": \"To which category does the Christmas Lights by Night Open-Top Bus Tour belong?\",\n",
+    "    \"context\": \"Christmas Lights by Night Open-Top Bus Tour is an activity of type guided tour. It lasts 1.5 hours...\"\n",
+    "})\n",
+    "\n",
+    "print(result)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}