Full Fine Tuned Model

Gemma2_2B/inference.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -291,13 +291,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import torch\n",
     "from peft import PeftModel\n",
     "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
     "\n",
     "# Load the base model and tokenizer\n",
     "model_name = \"google/gemma-2-2b-it\"\n",
-    "device_map = {\"\": 0}  # Use GPU 0 for the model\n",
-    "\n",
     "# Load the fine-tuned model\n",
     "new_model = \"gemma-2-2b-ft/\"  # Replace with the path to your fine-tuned model"
    ]
@@ -310,7 +309,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8bf9b158501544f092a784849b8e402d",
+       "model_id": "21f72716997c42cfa2244677b36b85f8",
        "version_major": 2,
        "version_minor": 0
       },
@@ -320,18 +319,38 @@
      },
      "metadata": {},
      "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "('gemma2-TADBot\\\\tokenizer_config.json',\n",
+       " 'gemma2-TADBot\\\\special_tokens_map.json',\n",
+       " 'gemma2-TADBot\\\\tokenizer.json')"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "base_model = AutoModelForCausalLM.from_pretrained(\n",
-    "    model_name, device_map=device_map, cache_dir=\".cache/\")\n",
+    "    model_name,\n",
+    "    low_cpu_mem_usage=True,\n",
+    "    return_dict=True,\n",
+    "    torch_dtype=torch.float16,\n",
+    "    device_map=\"cpu\",\n",
+    "    cache_dir=\".cache/\"\n",
+    ")\n",
     "model = PeftModel.from_pretrained(base_model, new_model, cache_dir = \".cache/\")\n",
     "model = model.merge_and_unload()\n",
+    "model.save_pretrained(\"gemma2-TADBot\")\n",
     "\n",
     "# Reload tokenizer to save it\n",
     "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, cache_dir = \".cache/\")\n",
     "tokenizer.pad_token = tokenizer.eos_token\n",
-    "tokenizer.padding_side = \"right\"\n"
+    "tokenizer.padding_side = \"right\"\n",
+    "tokenizer.save_pretrained(\"gemma2-TADBot\")"
    ]
   },
   {