Update: first

T5.ipynb

@@ -0,0 +1,158 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2c4eea53",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'OutStream' object has no attribute 'reconfigure'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mAttributeError\u001b[39m                            Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msys\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mstdout\u001b[49m\u001b[43m.\u001b[49m\u001b[43mreconfigure\u001b[49m(encoding=\u001b[33m'\u001b[39m\u001b[33mutf-8\u001b[39m\u001b[33m'\u001b[39m)\n",
+      "\u001b[31mAttributeError\u001b[39m: 'OutStream' object has no attribute 'reconfigure'"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "sys.stdout.reconfigure(encoding='utf-8')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c2185684",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting simpleT5\n",
+      "  Using cached simplet5-0.1.4.tar.gz (7.3 kB)\n",
+      "  Installing build dependencies: started\n",
+      "  Installing build dependencies: finished with status 'done'\n",
+      "  Getting requirements to build wheel: started\n",
+      "  Getting requirements to build wheel: finished with status 'done'\n",
+      "  Preparing metadata (pyproject.toml): started\n",
+      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
+      "Requirement already satisfied: numpy in d:\\programming\\workspace_python\\study_llm\\venv\\lib\\site-packages (from simpleT5) (2.1.2)\n",
+      "Requirement already satisfied: pandas in d:\\programming\\workspace_python\\study_llm\\venv\\lib\\site-packages (from simpleT5) (2.3.1)\n",
+      "Collecting sentencepiece (from simpleT5)\n",
+      "  Using cached sentencepiece-0.2.0.tar.gz (2.6 MB)\n",
+      "  Installing build dependencies: started\n",
+      "  Installing build dependencies: finished with status 'done'\n",
+      "  Getting requirements to build wheel: started\n",
+      "  Getting requirements to build wheel: finished with status 'error'\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  error: subprocess-exited-with-error\n",
+      "  \n",
+      "  × Getting requirements to build wheel did not run successfully.\n",
+      "  │ exit code: 1\n",
+      "  ╰─> [48 lines of output]\n",
+      "      Traceback (most recent call last):\n",
+      "        File \u001b[35m\"D:\\Programming\\workspace_python\\Study_LLM\\venv\\Lib\\site-packages\\pip\\_vendor\\pyproject_hooks\\_in_process\\_in_process.py\"\u001b[0m, line \u001b[35m389\u001b[0m, in \u001b[35m<module>\u001b[0m\n",
+      "          \u001b[31mmain\u001b[0m\u001b[1;31m()\u001b[0m\n",
+      "          \u001b[31m~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n",
+      "        File \u001b[35m\"D:\\Programming\\workspace_python\\Study_LLM\\venv\\Lib\\site-packages\\pip\\_vendor\\pyproject_hooks\\_in_process\\_in_process.py\"\u001b[0m, line \u001b[35m373\u001b[0m, in \u001b[35mmain\u001b[0m\n",
+      "          json_out[\"return_val\"] = \u001b[31mhook\u001b[0m\u001b[1;31m(**hook_input[\"kwargs\"])\u001b[0m\n",
+      "                                   \u001b[31m~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"D:\\Programming\\workspace_python\\Study_LLM\\venv\\Lib\\site-packages\\pip\\_vendor\\pyproject_hooks\\_in_process\\_in_process.py\"\u001b[0m, line \u001b[35m143\u001b[0m, in \u001b[35mget_requires_for_build_wheel\u001b[0m\n",
+      "          return hook(config_settings)\n",
+      "        File \u001b[35m\"C:\\Users\\han\\AppData\\Local\\Temp\\pip-build-env-72ht70wk\\overlay\\Lib\\site-packages\\setuptools\\build_meta.py\"\u001b[0m, line \u001b[35m331\u001b[0m, in \u001b[35mget_requires_for_build_wheel\u001b[0m\n",
+      "          return \u001b[31mself._get_build_requires\u001b[0m\u001b[1;31m(config_settings, requirements=[])\u001b[0m\n",
+      "                 \u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Users\\han\\AppData\\Local\\Temp\\pip-build-env-72ht70wk\\overlay\\Lib\\site-packages\\setuptools\\build_meta.py\"\u001b[0m, line \u001b[35m301\u001b[0m, in \u001b[35m_get_build_requires\u001b[0m\n",
+      "          \u001b[31mself.run_setup\u001b[0m\u001b[1;31m()\u001b[0m\n",
+      "          \u001b[31m~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Users\\han\\AppData\\Local\\Temp\\pip-build-env-72ht70wk\\overlay\\Lib\\site-packages\\setuptools\\build_meta.py\"\u001b[0m, line \u001b[35m512\u001b[0m, in \u001b[35mrun_setup\u001b[0m\n",
+      "          \u001b[31msuper().run_setup\u001b[0m\u001b[1;31m(setup_script=setup_script)\u001b[0m\n",
+      "          \u001b[31m~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Users\\han\\AppData\\Local\\Temp\\pip-build-env-72ht70wk\\overlay\\Lib\\site-packages\\setuptools\\build_meta.py\"\u001b[0m, line \u001b[35m317\u001b[0m, in \u001b[35mrun_setup\u001b[0m\n",
+      "          \u001b[31mexec\u001b[0m\u001b[1;31m(code, locals())\u001b[0m\n",
+      "          \u001b[31m~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"<string>\"\u001b[0m, line \u001b[35m128\u001b[0m, in \u001b[35m<module>\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.13_3.13.1520.0_x64__qbz5n2kfra8p0\\Lib\\subprocess.py\"\u001b[0m, line \u001b[35m414\u001b[0m, in \u001b[35mcheck_call\u001b[0m\n",
+      "          retcode = call(*popenargs, **kwargs)\n",
+      "        File \u001b[35m\"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.13_3.13.1520.0_x64__qbz5n2kfra8p0\\Lib\\subprocess.py\"\u001b[0m, line \u001b[35m395\u001b[0m, in \u001b[35mcall\u001b[0m\n",
+      "          with \u001b[31mPopen\u001b[0m\u001b[1;31m(*popenargs, **kwargs)\u001b[0m as p:\n",
+      "               \u001b[31m~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.13_3.13.1520.0_x64__qbz5n2kfra8p0\\Lib\\subprocess.py\"\u001b[0m, line \u001b[35m1039\u001b[0m, in \u001b[35m__init__\u001b[0m\n",
+      "          \u001b[31mself._execute_child\u001b[0m\u001b[1;31m(args, executable, preexec_fn, close_fds,\u001b[0m\n",
+      "          \u001b[31m~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "                              \u001b[1;31mpass_fds, cwd, env,\u001b[0m\n",
+      "                              \u001b[1;31m^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "          ...<5 lines>...\n",
+      "                              \u001b[1;31mgid, gids, uid, umask,\u001b[0m\n",
+      "                              \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "                              \u001b[1;31mstart_new_session, process_group)\u001b[0m\n",
+      "                              \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "        File \u001b[35m\"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.13_3.13.1520.0_x64__qbz5n2kfra8p0\\Lib\\subprocess.py\"\u001b[0m, line \u001b[35m1554\u001b[0m, in \u001b[35m_execute_child\u001b[0m\n",
+      "          hp, ht, pid, tid = \u001b[31m_winapi.CreateProcess\u001b[0m\u001b[1;31m(executable, args,\u001b[0m\n",
+      "                             \u001b[31m~~~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "                                   \u001b[1;31m# no special security\u001b[0m\n",
+      "                                   \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n",
+      "          ...<4 lines>...\n",
+      "                                   \u001b[1;31mcwd,\u001b[0m\n",
+      "                                   \u001b[1;31m^^^^\u001b[0m\n",
+      "                                   \u001b[1;31mstartupinfo)\u001b[0m\n",
+      "                                   \u001b[1;31m^^^^^^^^^^^^\u001b[0m\n",
+      "      \u001b[1;35mFileNotFoundError\u001b[0m: \u001b[35m[WinError 2] 吏\\x80\\xec젙\\xeb맂 \\xed뙆\\xec씪\\xec쓣 李얠쓣 \\xec닔 \\xec뾾\\xec뒿\\xeb땲\\xeb떎\u001b[0m\n",
+      "      [end of output]\n",
+      "  \n",
+      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
+      "error: subprocess-exited-with-error\n",
+      "\n",
+      "× Getting requirements to build wheel did not run successfully.\n",
+      "│ exit code: 1\n",
+      "╰─> See above for output.\n",
+      "\n",
+      "note: This error originates from a subprocess, and is likely not a problem with pip.\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install simpleT5 "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "566cf25f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

config.json/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 2,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "use_cache": true,
+  "vocab_size": 11954
+}

glm.ipynb

@@ -0,0 +1,569 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "03ab65cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tokenizers.models import BPE\n",
+    "from tokenizers import Tokenizer\n",
+    "from tokenizers.decoders import ByteLevel as ByteLevelDecoder\n",
+    "from tokenizers.normalizers import Sequence, Lowercase\n",
+    "from tokenizers.pre_tokenizers import ByteLevel\n",
+    "from tokenizers.trainers import BpeTrainer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "efe37f35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = Tokenizer(BPE())\n",
+    "tokenizer.normalizer = Sequence([Lowercase()])\n",
+    "tokenizer.pre_tokenizer = ByteLevel()\n",
+    "tokenizer.decoder = ByteLevelDecoder()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "6a596c74",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer = BpeTrainer(vocab_size = 50000, initial_alphabet=ByteLevel.alphabet(), special_tokens=['<s>', '<pad>', '</s>','</unk>', '<mask>'])\n",
+    "tokenizer.train([\"../../datasets/austen-emma.txt\"], trainer)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "id": "00138f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.save(\"tokenizer_gpt/tokenizer.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "id": "ca9061de",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import (GPT2TokenizerFast, GPT2Config, GPT2LMHeadModel)\n",
+    "tokenizer_gpt = GPT2TokenizerFast.from_pretrained(\"tokenizer_gpt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "id": "4e5f77ea",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 48,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer_gpt.add_special_tokens({\n",
+    "    \"eos_token\": \"</s>\",\n",
+    "    \"bos_token\": \"<s>\",\n",
+    "    \"unk_token\": \"<unk>\",\n",
+    "    \"pad_token\": \"<pad>\",\n",
+    "    \"mask_token\": \"<mask>\"\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "id": "ab84b4f2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[0, 469, 293, 225, 2]"
+      ]
+     },
+     "execution_count": 49,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer_gpt.encode(\"<s> thisis </s>\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "id": "2cda86e9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[0, 469, 361, 225, 2]"
+      ]
+     },
+     "execution_count": 50,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer_gpt.encode(\"<s> this is </s>\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "id": "5d7d2260",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = GPT2Config(\n",
+    "    vocab_size = tokenizer_gpt.vocab_size,\n",
+    "    bos_token_id = tokenizer_gpt.bos_token_id,\n",
+    "    eos_token_id = tokenizer_gpt.eos_token_id\n",
+    ")\n",
+    "\n",
+    "model = GPT2LMHeadModel(config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "id": "45d2da31",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2Config {\n",
+       "  \"activation_function\": \"gelu_new\",\n",
+       "  \"attn_pdrop\": 0.1,\n",
+       "  \"bos_token_id\": 0,\n",
+       "  \"embd_pdrop\": 0.1,\n",
+       "  \"eos_token_id\": 2,\n",
+       "  \"initializer_range\": 0.02,\n",
+       "  \"layer_norm_epsilon\": 1e-05,\n",
+       "  \"model_type\": \"gpt2\",\n",
+       "  \"n_embd\": 768,\n",
+       "  \"n_head\": 12,\n",
+       "  \"n_inner\": null,\n",
+       "  \"n_layer\": 12,\n",
+       "  \"n_positions\": 1024,\n",
+       "  \"reorder_and_upcast_attn\": false,\n",
+       "  \"resid_pdrop\": 0.1,\n",
+       "  \"scale_attn_by_inverse_layer_idx\": false,\n",
+       "  \"scale_attn_weights\": true,\n",
+       "  \"summary_activation\": null,\n",
+       "  \"summary_first_dropout\": 0.1,\n",
+       "  \"summary_proj_to_labels\": true,\n",
+       "  \"summary_type\": \"cls_index\",\n",
+       "  \"summary_use_proj\": true,\n",
+       "  \"transformers_version\": \"4.54.1\",\n",
+       "  \"use_cache\": true,\n",
+       "  \"vocab_size\": 11954\n",
+       "}"
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "id": "692b12c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(11954, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2Attention(\n",
+       "          (c_attn): Conv1D(nf=2304, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=768)\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D(nf=3072, nx=768)\n",
+       "          (c_proj): Conv1D(nf=768, nx=3072)\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=11954, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "49b1f246",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"../../datasets/austen-emma.txt\", \"r\", encoding='utf-8') as f:\n",
+    "    content = f.readlines()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "id": "306508f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "content_p = []\n",
+    "for c in content:\n",
+    "    if len(c) > 10:\n",
+    "        content_p.append(c.strip())\n",
+    "content_p = ' '.join(content_p) + tokenizer_gpt.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "id": "4536854a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenized_content = tokenizer_gpt.encode(content_p)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "id": "a1b0aacc",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "195221"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(tokenized_content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "b475fdde",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_len = 100\n",
+    "examples = []\n",
+    "for i in range(0, len(tokenized_content) - sample_len + 1):\n",
+    "    examples.append(\n",
+    "        tokenized_content[i:i+ sample_len]\n",
+    "    )\n",
+    "\n",
+    "train_data = []\n",
+    "labels = []\n",
+    "for example in examples:\n",
+    "    train_data.append(example[:-1])\n",
+    "    labels.append(example[1:])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "id": "ed046662",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from torch.utils.data import TensorDataset, DataLoader\n",
+    "import torch.nn.functional as F\n",
+    "\n",
+    "buffer = 500\n",
+    "batch_size = 64\n",
+    "\n",
+    "train_data = torch.Tensor(train_data).to(dtype=torch.long).cuda()\n",
+    "labels = torch.Tensor(labels).to(dtype=torch.long).cuda()\n",
+    "dataset = TensorDataset(train_data, labels)\n",
+    "\n",
+    "loader = DataLoader(dataset, batch_size=batch_size, drop_last=True, shuffle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "dc687478",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch.optim import Adam\n",
+    "import torch.nn as nn\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "model = model.cuda()\n",
+    "optimizer = Adam(model.parameters(), lr=3e-5, eps=1e-08)\n",
+    "criterion = nn.CrossEntropyLoss()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "99b192b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|          | 0/3048 [00:00<?, ?it/s]C:\\Users\\han\\AppData\\Local\\Temp\\ipykernel_32488\\611308887.py:17: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
+      "  pred = F.softmax(pred)\n",
+      "100%|██████████| 3048/3048 [07:58<00:00,  6.37it/s, loss: 0.0061, aucc: 0.4009]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor(0.0085, device='cuda:0', grad_fn=<DivBackward0>)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "ename": "TypeError",
+     "evalue": "len() takes exactly one argument (0 given)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mTypeError\u001b[39m                                 Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[79]\u001b[39m\u001b[32m, line 26\u001b[39m\n\u001b[32m     23\u001b[39m     progress.set_postfix_str(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mloss: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mloss.cpu().detach().numpy()\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m.04f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m, aucc: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00maucc.cpu()/cnt\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m.04f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m     25\u001b[39m \u001b[38;5;28mprint\u001b[39m(losses / \u001b[38;5;28mlen\u001b[39m(loader))\n\u001b[32m---> \u001b[39m\u001b[32m26\u001b[39m \u001b[38;5;28mprint\u001b[39m(aucc / \u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n",
+      "\u001b[31mTypeError\u001b[39m: len() takes exactly one argument (0 given)"
+     ]
+    }
+   ],
+   "source": [
+    "model.train()\n",
+    "# for epoch in tqdm(range(0, 1)):\n",
+    "losses = 0.\n",
+    "aucc = 0\n",
+    "cnt = 0\n",
+    "progress = tqdm(loader)\n",
+    "for x, y in progress:\n",
+    "    pred = model(x).logits\n",
+    "    \n",
+    "    y = F.one_hot(y, num_classes=tokenizer_gpt.vocab_size)\n",
+    "    \n",
+    "    loss = criterion(pred.to(dtype=torch.float32), y.to(dtype=torch.float32))\n",
+    "    optimizer.zero_grad()\n",
+    "    loss.backward()\n",
+    "    optimizer.step()\n",
+    "    \n",
+    "    pred = F.softmax(pred)\n",
+    "    y = torch.argmax(y, dim=2)\n",
+    "    aucc += torch.sum(pred == y)\n",
+    "    cnt += (batch_size*99)\n",
+    "    losses += loss\n",
+    "    progress.set_postfix_str(f\"loss: {loss.cpu().detach().numpy():.04f}, aucc: {aucc.cpu()/cnt:.04f}\")\n",
+    "    \n",
+    "print(losses / len(loader))\n",
+    "print(aucc / cnt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "5f0c3c7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate(start, model):\n",
+    "    input_token_ids = tokenizer_gpt.encode(start, return_tensors='pt').cuda()\n",
+    "    output = model.generate(\n",
+    "        input_token_ids,\n",
+    "        max_length= 500,\n",
+    "        num_beams = 5,\n",
+    "        temperature=0.7,\n",
+    "        no_repeat_ngram_size=2,\n",
+    "        num_return_sequences=1\n",
+    "    )\n",
+    "    return tokenizer_gpt.decode(output[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "id": "7acad8eb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The following generation flags are not valid and may be ignored: ['temperature']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'  yes!\" cried emma.--\"my dearest harriet, mr. weston, who had been walking away from hartfield, he would never have seen him before; and while she did not quite understand how it might be supposed that she could give up the idea of any body\\'s coming to such a thing by his manners. \"i am afraid,\" he replied, \"that you must be the greatest pleasure.  you do not know what your father would have heard.\" \"oh! yes, my dear--but i dare say i am sure i shall think they will think you are very much obliged to be sure.--but this is an excellent miss smith, however, indeed; but there is being _you are quite enough to make one of course.--i can hardly ever hear of every thing to see nothing else.\" she comes in love with me, sir?--well--a very bad.--he is coming over this morning and yet quite complete in the whole.\" emma could not likely to call upon the same glance at all these words, if they walked on that sort of his feelings:  i must take care about ten months ago, perhaps--what an old acquaintance with my father--and so far off in spite of the smallest degree or two men, i was forced to bring a few weeks ago as far as well, unless they are going to wait for _she is a great regard for a young lady\\'s conduct, though her husband and therefore.\" chapter xvi and mrs. frank churchill came out of having given me--he came back again, when they were all my mother might have done?  she walked up in fact.--the same evening it was sitting down again--the case at this day?\" \"miss woodhouse\\'s eyes to keep your feelings did you walked off, and then?\" emma\\'s situation. chapter xiv some young woman\\'s manners to give me!  he seemed to feel that period at weymouth.\"  and nobody else, papa) the worst of hearing every body else--\" she came over-morrow.--they are so pleased with all that moment.--a little boys appeared till within half-day or four weeks, or three times a beautiful creature into the oddest creature!\" chapter xvii when she might not making a moment--\"the change?--you must wait a mile off!--but if this evening without being taken place, moreover, but one subject after making no account.--it seems every other people could never saw me; he made her mind was giving themselves off; her eyes made up these young'"
+      ]
+     },
+     "execution_count": 92,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "generate(\" \", model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 93,
+   "id": "b169491a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import (WEIGHTS_NAME, CONFIG_NAME)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "id": "690df94b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'pytorch_model.bin'"
+      ]
+     },
+     "execution_count": 94,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "WEIGHTS_NAME"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "id": "b6d192d9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'config.json'"
+      ]
+     },
+     "execution_count": 95,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "CONFIG_NAME"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 96,
+   "id": "e4abd96c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.save_pretrained(WEIGHTS_NAME)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "id": "d719459b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config.save_pretrained(CONFIG_NAME)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pytorch_model.bin/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 2,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "use_cache": true,
+  "vocab_size": 11954
+}

pytorch_model.bin/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "transformers_version": "4.54.1"
+}

pytorch_model.bin/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9efd942375a58ba5fd94dda20b8cdff0abbe41d52ae86752b2d44ad56bf86c
+size 380107392