Duplicate from IkIzma/Transformer_Homework

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

FineTune.ipynb

@@ -0,0 +1,877 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "BO7MEGbb6mtB"
+   },
+   "source": [
+    "# Finetune \n",
+    "Finetuning RuGPTs model with huggingface.\n",
+    "\n",
+    "## Install env"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true,
+    "id": "Xyhc5yrzR75j"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Cloning into 'transformers'...\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Processing /home/kamil/Documents/SHAD/ML/Part 2/Seminar 7/transformers\n",
+      "  Installing build dependencies: started\n",
+      "  Installing build dependencies: finished with status 'done'\n",
+      "  Getting requirements to build wheel: started\n",
+      "  Getting requirements to build wheel: finished with status 'done'\n",
+      "  Preparing metadata (pyproject.toml): started\n",
+      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
+      "Requirement already satisfied: tqdm>=4.27 in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (4.65.0)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (2023.3.23)\n",
+      "Requirement already satisfied: requests in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (2.28.2)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers==4.29.0.dev0) (5.4.1)\n",
+      "Requirement already satisfied: filelock in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (3.10.6)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (0.13.3)\n",
+      "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers==4.29.0.dev0) (21.3)\n",
+      "Requirement already satisfied: numpy>=1.17 in /usr/lib/python3/dist-packages (from transformers==4.29.0.dev0) (1.21.5)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /home/kamil/.local/lib/python3.10/site-packages (from transformers==4.29.0.dev0) (0.13.4)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/kamil/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers==4.29.0.dev0) (4.5.0)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers==4.29.0.dev0) (2020.6.20)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/kamil/.local/lib/python3.10/site-packages (from requests->transformers==4.29.0.dev0) (3.0.1)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers==4.29.0.dev0) (1.26.5)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers==4.29.0.dev0) (3.3)\n",
+      "Building wheels for collected packages: transformers\n",
+      "  Building wheel for transformers (pyproject.toml): started\n",
+      "  Building wheel for transformers (pyproject.toml): finished with status 'done'\n",
+      "  Created wheel for transformers: filename=transformers-4.29.0.dev0-py3-none-any.whl size=6929166 sha256=280057264eb46bc68355d5c5a1a4d2caff1da9951d55bacbaa62463cbf73296c\n",
+      "  Stored in directory: /tmp/pip-ephem-wheel-cache-xt8a8mve/wheels/a5/d3/d1/e281e4412399bfd2f44bb86274ac4204a7d53b596a501f2ad1\n",
+      "Successfully built transformers\n",
+      "Installing collected packages: transformers\n",
+      "  Attempting uninstall: transformers\n",
+      "    Found existing installation: transformers 4.27.4\n",
+      "    Uninstalling transformers-4.27.4:\n",
+      "      Successfully uninstalled transformers-4.27.4\n",
+      "Successfully installed transformers-4.29.0.dev0\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%bash\n",
+    "git clone https://github.com/huggingface/transformers\n",
+    "cd transformers\n",
+    "pip install ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true,
+    "id": "Os4vOL5LTOmk"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting datasets\n",
+      "  Downloading datasets-2.11.0-py3-none-any.whl (468 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/lib/python3/dist-packages (from datasets) (1.21.5)\n",
+      "Requirement already satisfied: pyarrow>=8.0.0 in /home/kamil/.local/lib/python3.10/site-packages (from datasets) (11.0.0)\n",
+      "Requirement already satisfied: tqdm>=4.62.1 in /home/kamil/.local/lib/python3.10/site-packages (from datasets) (4.65.0)\n",
+      "Requirement already satisfied: packaging in /usr/lib/python3/dist-packages (from datasets) (21.3)\n",
+      "Collecting dill<0.3.7,>=0.3.0\n",
+      "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: pandas in /home/kamil/.local/lib/python3.10/site-packages (from datasets) (1.5.3)\n",
+      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /home/kamil/.local/lib/python3.10/site-packages (from datasets) (0.13.4)\n",
+      "Collecting aiohttp\n",
+      "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m0m eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /home/kamil/.local/lib/python3.10/site-packages (from datasets) (2.28.2)\n",
+      "Collecting fsspec[http]>=2021.11.1\n",
+      "  Downloading fsspec-2023.4.0-py3-none-any.whl (153 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.0/154.0 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets) (5.4.1)\n",
+      "Collecting xxhash\n",
+      "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting responses<0.19\n",
+      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
+      "Collecting multiprocess\n",
+      "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets) (3.0.1)\n",
+      "Collecting multidict<7.0,>=4.5\n",
+      "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets) (21.2.0)\n",
+      "Collecting async-timeout<5.0,>=4.0.0a3\n",
+      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
+      "Collecting frozenlist>=1.1.1\n",
+      "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
+      "  Downloading yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.0/264.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hCollecting aiosignal>=1.1.2\n",
+      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/kamil/.local/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n",
+      "Requirement already satisfied: filelock in /home/kamil/.local/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.10.6)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (1.26.5)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (3.3)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2020.6.20)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas->datasets) (2022.1)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/kamil/.local/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
+      "Installing collected packages: xxhash, multidict, fsspec, frozenlist, dill, async-timeout, yarl, responses, multiprocess, aiosignal, aiohttp, datasets\n",
+      "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.11.0 dill-0.3.6 frozenlist-1.3.3 fsspec-2023.4.0 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true,
+    "id": "m1P6WSIeTdV5"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Collecting evaluate\n",
+      "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
+      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: xxhash in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (3.2.0)\n",
+      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (2023.4.0)\n",
+      "Requirement already satisfied: responses<0.19 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (0.18.0)\n",
+      "Requirement already satisfied: packaging in /usr/lib/python3/dist-packages (from evaluate) (21.3)\n",
+      "Requirement already satisfied: tqdm>=4.62.1 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (4.65.0)\n",
+      "Requirement already satisfied: huggingface-hub>=0.7.0 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (0.13.4)\n",
+      "Requirement already satisfied: numpy>=1.17 in /usr/lib/python3/dist-packages (from evaluate) (1.21.5)\n",
+      "Requirement already satisfied: dill in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (0.3.6)\n",
+      "Requirement already satisfied: requests>=2.19.0 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (2.28.2)\n",
+      "Requirement already satisfied: datasets>=2.0.0 in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (2.11.0)\n",
+      "Requirement already satisfied: multiprocess in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (0.70.14)\n",
+      "Requirement already satisfied: pandas in /home/kamil/.local/lib/python3.10/site-packages (from evaluate) (1.5.3)\n",
+      "Requirement already satisfied: aiohttp in /home/kamil/.local/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.4)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets>=2.0.0->evaluate) (5.4.1)\n",
+      "Requirement already satisfied: pyarrow>=8.0.0 in /home/kamil/.local/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\n",
+      "Requirement already satisfied: filelock in /home/kamil/.local/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (3.10.6)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/kamil/.local/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/kamil/.local/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.0.1)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/lib/python3/dist-packages (from requests>=2.19.0->evaluate) (1.26.5)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->evaluate) (2020.6.20)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->evaluate) (3.3)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/kamil/.local/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas->evaluate) (2022.1)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (21.2.0)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.8.2)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/kamil/.local/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n",
+      "Installing collected packages: evaluate\n",
+      "Successfully installed evaluate-0.4.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install evaluate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "id": "WJZtWu8u6nwL"
+   },
+   "outputs": [],
+   "source": [
+    "!mkdir models/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "WqwZiumW8WbZ"
+   },
+   "source": [
+    "## Download files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "collapsed": true,
+    "id": "j51bKtQW6nyY"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2023-04-16 19:47:12--  https://www.dropbox.com/s/oa3v9c7g9bp40xw/train.txt?dl=0\n",
+      "Resolving www.dropbox.com (www.dropbox.com)... 162.125.70.18, 2620:100:6027:18::a27d:4812\n",
+      "Connecting to www.dropbox.com (www.dropbox.com)|162.125.70.18|:443... connected.\n",
+      "HTTP request sent, awaiting response... 302 Found\n",
+      "Location: /s/raw/oa3v9c7g9bp40xw/train.txt [following]\n",
+      "--2023-04-16 19:47:13--  https://www.dropbox.com/s/raw/oa3v9c7g9bp40xw/train.txt\n",
+      "Reusing existing connection to www.dropbox.com:443.\n",
+      "HTTP request sent, awaiting response... 302 Found\n",
+      "Location: https://uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com/cd/0/inline/B6QRy9JQtzcR-y7uMF3TBS26D_9WsPQhmzXoWmGuHLgFMVq5YeUy4XIvymTf-coW8njd463mquV6DZB7LKdlznygflsCZHNIJ0A8Hf_yyRl2y5rb63wSIyvyBbANSc5DBKvhD4HSmZ-G8GDlRmEf3CXz-PP4jpoQFXwvDZCbIGlStw/file# [following]\n",
+      "--2023-04-16 19:47:13--  https://uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com/cd/0/inline/B6QRy9JQtzcR-y7uMF3TBS26D_9WsPQhmzXoWmGuHLgFMVq5YeUy4XIvymTf-coW8njd463mquV6DZB7LKdlznygflsCZHNIJ0A8Hf_yyRl2y5rb63wSIyvyBbANSc5DBKvhD4HSmZ-G8GDlRmEf3CXz-PP4jpoQFXwvDZCbIGlStw/file\n",
+      "Resolving uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com (uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com)... 162.125.70.15, 2620:100:6028:15::a27d:470f\n",
+      "Connecting to uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com (uc5788429f15c026c306ed6aa7c0.dl.dropboxusercontent.com)|162.125.70.15|:443... connected.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 1654900 (1,6M) [text/plain]\n",
+      "Saving to: ‘train.txt’\n",
+      "\n",
+      "train.txt           100%[===================>]   1,58M  8,43MB/s    in 0,2s    \n",
+      "\n",
+      "2023-04-16 19:47:14 (8,43 MB/s) - ‘train.txt’ saved [1654900/1654900]\n",
+      "\n",
+      "--2023-04-16 19:47:14--  https://www.dropbox.com/s/mworl3ld6r3bg62/valid.txt?dl=0\n",
+      "Resolving www.dropbox.com (www.dropbox.com)... 162.125.70.18, 2620:100:6027:18::a27d:4812\n",
+      "Connecting to www.dropbox.com (www.dropbox.com)|162.125.70.18|:443... connected.\n",
+      "HTTP request sent, awaiting response... 302 Found\n",
+      "Location: /s/raw/mworl3ld6r3bg62/valid.txt [following]\n",
+      "--2023-04-16 19:47:14--  https://www.dropbox.com/s/raw/mworl3ld6r3bg62/valid.txt\n",
+      "Reusing existing connection to www.dropbox.com:443.\n",
+      "HTTP request sent, awaiting response... 302 Found\n",
+      "Location: https://uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com/cd/0/inline/B6QZm3htPxEoOiKlbNIGQz27I0gnkhm3CfT9DoU9qR3VUmFjo8_GWcsquYc01t4LT6WYRj4t70Sw9Z9DhdBPq4ZFpgiGfN4TyCf4Hav48iIButfo1Aaa31uqnVavn3dRVXKM2CZ5ewiMDDEGDexFnB-ZPHZyomgPCjDRtkdkMvfP7g/file# [following]\n",
+      "--2023-04-16 19:47:15--  https://uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com/cd/0/inline/B6QZm3htPxEoOiKlbNIGQz27I0gnkhm3CfT9DoU9qR3VUmFjo8_GWcsquYc01t4LT6WYRj4t70Sw9Z9DhdBPq4ZFpgiGfN4TyCf4Hav48iIButfo1Aaa31uqnVavn3dRVXKM2CZ5ewiMDDEGDexFnB-ZPHZyomgPCjDRtkdkMvfP7g/file\n",
+      "Resolving uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com (uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com)... 162.125.70.15, 2620:100:6026:15::a27d:460f\n",
+      "Connecting to uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com (uc5ee48fa1d36195fd1fe094947e.dl.dropboxusercontent.com)|162.125.70.15|:443... connected.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 167021 (163K) [text/plain]\n",
+      "Saving to: ‘valid.txt’\n",
+      "\n",
+      "valid.txt           100%[===================>] 163,11K  --.-KB/s    in 0,08s   \n",
+      "\n",
+      "2023-04-16 19:47:15 (2,02 MB/s) - ‘valid.txt’ saved [167021/167021]\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "!wget -O train.txt https://www.dropbox.com/s/oa3v9c7g9bp40xw/train.txt?dl=0\n",
+    "!wget -O valid.txt https://www.dropbox.com/s/mworl3ld6r3bg62/valid.txt?dl=0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "zoyX62qN_38l"
+   },
+   "source": [
+    "## Train \n",
+    "The following code download model and tokenizer from huggingface and finetune model for generating essays."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": true,
+    "id": "OCIERP8AS1Dl"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "04/16/2023 19:47:40 - WARNING - __main__ - Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
+      "04/16/2023 19:47:40 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n",
+      "_n_gpu=1,\n",
+      "adafactor=False,\n",
+      "adam_beta1=0.9,\n",
+      "adam_beta2=0.999,\n",
+      "adam_epsilon=1e-08,\n",
+      "auto_find_batch_size=False,\n",
+      "bf16=False,\n",
+      "bf16_full_eval=False,\n",
+      "data_seed=None,\n",
+      "dataloader_drop_last=False,\n",
+      "dataloader_num_workers=0,\n",
+      "dataloader_pin_memory=True,\n",
+      "ddp_bucket_cap_mb=None,\n",
+      "ddp_find_unused_parameters=None,\n",
+      "ddp_timeout=1800,\n",
+      "debug=[],\n",
+      "deepspeed=None,\n",
+      "disable_tqdm=False,\n",
+      "do_eval=True,\n",
+      "do_predict=False,\n",
+      "do_train=True,\n",
+      "eval_accumulation_steps=None,\n",
+      "eval_delay=0,\n",
+      "eval_steps=None,\n",
+      "evaluation_strategy=no,\n",
+      "fp16=False,\n",
+      "fp16_backend=auto,\n",
+      "fp16_full_eval=False,\n",
+      "fp16_opt_level=O1,\n",
+      "fsdp=[],\n",
+      "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
+      "fsdp_min_num_params=0,\n",
+      "fsdp_transformer_layer_cls_to_wrap=None,\n",
+      "full_determinism=False,\n",
+      "gradient_accumulation_steps=1,\n",
+      "gradient_checkpointing=False,\n",
+      "greater_is_better=None,\n",
+      "group_by_length=False,\n",
+      "half_precision_backend=auto,\n",
+      "hub_model_id=None,\n",
+      "hub_private_repo=False,\n",
+      "hub_strategy=every_save,\n",
+      "hub_token=<HUB_TOKEN>,\n",
+      "ignore_data_skip=False,\n",
+      "include_inputs_for_metrics=False,\n",
+      "jit_mode_eval=False,\n",
+      "label_names=None,\n",
+      "label_smoothing_factor=0.0,\n",
+      "learning_rate=5e-05,\n",
+      "length_column_name=length,\n",
+      "load_best_model_at_end=False,\n",
+      "local_rank=-1,\n",
+      "log_level=passive,\n",
+      "log_level_replica=warning,\n",
+      "log_on_each_node=True,\n",
+      "logging_dir=models/essays2/runs/Apr16_19-47-40_kamil-desktop,\n",
+      "logging_first_step=False,\n",
+      "logging_nan_inf_filter=True,\n",
+      "logging_steps=500,\n",
+      "logging_strategy=steps,\n",
+      "lr_scheduler_type=linear,\n",
+      "max_grad_norm=1.0,\n",
+      "max_steps=-1,\n",
+      "metric_for_best_model=None,\n",
+      "mp_parameters=,\n",
+      "no_cuda=False,\n",
+      "num_train_epochs=3.0,\n",
+      "optim=adamw_hf,\n",
+      "optim_args=None,\n",
+      "output_dir=models/essays2,\n",
+      "overwrite_output_dir=False,\n",
+      "past_index=-1,\n",
+      "per_device_eval_batch_size=1,\n",
+      "per_device_train_batch_size=1,\n",
+      "prediction_loss_only=False,\n",
+      "push_to_hub=False,\n",
+      "push_to_hub_model_id=None,\n",
+      "push_to_hub_organization=None,\n",
+      "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
+      "ray_scope=last,\n",
+      "remove_unused_columns=True,\n",
+      "report_to=[],\n",
+      "resume_from_checkpoint=None,\n",
+      "run_name=models/essays2,\n",
+      "save_on_each_node=False,\n",
+      "save_safetensors=False,\n",
+      "save_steps=500,\n",
+      "save_strategy=steps,\n",
+      "save_total_limit=None,\n",
+      "seed=42,\n",
+      "sharded_ddp=[],\n",
+      "skip_memory_metrics=True,\n",
+      "tf32=None,\n",
+      "torch_compile=False,\n",
+      "torch_compile_backend=None,\n",
+      "torch_compile_mode=None,\n",
+      "torchdynamo=None,\n",
+      "tpu_metrics_debug=False,\n",
+      "tpu_num_cores=None,\n",
+      "use_ipex=False,\n",
+      "use_legacy_prediction_loop=False,\n",
+      "use_mps_device=False,\n",
+      "warmup_ratio=0.0,\n",
+      "warmup_steps=0,\n",
+      "weight_decay=0.0,\n",
+      "xpu_backend=None,\n",
+      ")\n",
+      "04/16/2023 19:47:40 - INFO - datasets.builder - Using custom data configuration default-94a5e2bc6bcfdc2e\n",
+      "04/16/2023 19:47:40 - INFO - datasets.info - Loading Dataset Infos from /home/kamil/.local/lib/python3.10/site-packages/datasets/packaged_modules/text\n",
+      "04/16/2023 19:47:40 - INFO - datasets.builder - Generating dataset text (/home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2)\n",
+      "Downloading and preparing dataset text/default to /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2...\n",
+      "Downloading data files: 100%|██████████████████| 2/2 [00:00<00:00, 18517.90it/s]\n",
+      "04/16/2023 19:47:40 - INFO - datasets.download.download_manager - Downloading took 0.0 min\n",
+      "04/16/2023 19:47:40 - INFO - datasets.download.download_manager - Checksum Computation took 0.0 min\n",
+      "Extracting data files: 100%|█████████████████████| 2/2 [00:00<00:00, 228.71it/s]\n",
+      "04/16/2023 19:47:40 - INFO - datasets.builder - Generating train split\n",
+      "04/16/2023 19:47:40 - INFO - datasets.builder - Generating validation split\n",
+      "04/16/2023 19:47:40 - INFO - datasets.utils.info_utils - Unable to verify splits sizes.\n",
+      "Dataset text downloaded and prepared to /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2. Subsequent calls will reuse this data.\n",
+      "100%|███████████████████████████████████████████| 2/2 [00:00<00:00, 1228.20it/s]\n",
+      "Downloading (…)lve/main/config.json: 100%|██████| 608/608 [00:00<00:00, 832kB/s]\n",
+      "[INFO|configuration_utils.py:668] 2023-04-16 19:47:41,750 >> loading configuration file config.json from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/config.json\n",
+      "[INFO|configuration_utils.py:720] 2023-04-16 19:47:41,751 >> Model config GPT2Config {\n",
+      "  \"_name_or_path\": \"sberbank-ai/rugpt3small_based_on_gpt2\",\n",
+      "  \"activation_function\": \"gelu_new\",\n",
+      "  \"architectures\": [\n",
+      "    \"GPT2LMHeadModel\"\n",
+      "  ],\n",
+      "  \"attn_pdrop\": 0.1,\n",
+      "  \"bos_token_id\": 50256,\n",
+      "  \"embd_pdrop\": 0.1,\n",
+      "  \"eos_token_id\": 50256,\n",
+      "  \"gradient_checkpointing\": false,\n",
+      "  \"initializer_range\": 0.02,\n",
+      "  \"layer_norm_epsilon\": 1e-05,\n",
+      "  \"model_type\": \"gpt2\",\n",
+      "  \"n_ctx\": 2048,\n",
+      "  \"n_embd\": 768,\n",
+      "  \"n_head\": 12,\n",
+      "  \"n_inner\": null,\n",
+      "  \"n_layer\": 12,\n",
+      "  \"n_positions\": 2048,\n",
+      "  \"reorder_and_upcast_attn\": false,\n",
+      "  \"resid_pdrop\": 0.1,\n",
+      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
+      "  \"scale_attn_weights\": true,\n",
+      "  \"summary_activation\": null,\n",
+      "  \"summary_first_dropout\": 0.1,\n",
+      "  \"summary_proj_to_labels\": true,\n",
+      "  \"summary_type\": \"cls_index\",\n",
+      "  \"summary_use_proj\": true,\n",
+      "  \"transformers_version\": \"4.29.0.dev0\",\n",
+      "  \"use_cache\": true,\n",
+      "  \"vocab_size\": 50264\n",
+      "}\n",
+      "\n",
+      "[INFO|tokenization_auto.py:502] 2023-04-16 19:47:42,302 >> Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
+      "[INFO|configuration_utils.py:668] 2023-04-16 19:47:42,851 >> loading configuration file config.json from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/config.json\n",
+      "[INFO|configuration_utils.py:720] 2023-04-16 19:47:42,852 >> Model config GPT2Config {\n",
+      "  \"_name_or_path\": \"sberbank-ai/rugpt3small_based_on_gpt2\",\n",
+      "  \"activation_function\": \"gelu_new\",\n",
+      "  \"architectures\": [\n",
+      "    \"GPT2LMHeadModel\"\n",
+      "  ],\n",
+      "  \"attn_pdrop\": 0.1,\n",
+      "  \"bos_token_id\": 50256,\n",
+      "  \"embd_pdrop\": 0.1,\n",
+      "  \"eos_token_id\": 50256,\n",
+      "  \"gradient_checkpointing\": false,\n",
+      "  \"initializer_range\": 0.02,\n",
+      "  \"layer_norm_epsilon\": 1e-05,\n",
+      "  \"model_type\": \"gpt2\",\n",
+      "  \"n_ctx\": 2048,\n",
+      "  \"n_embd\": 768,\n",
+      "  \"n_head\": 12,\n",
+      "  \"n_inner\": null,\n",
+      "  \"n_layer\": 12,\n",
+      "  \"n_positions\": 2048,\n",
+      "  \"reorder_and_upcast_attn\": false,\n",
+      "  \"resid_pdrop\": 0.1,\n",
+      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
+      "  \"scale_attn_weights\": true,\n",
+      "  \"summary_activation\": null,\n",
+      "  \"summary_first_dropout\": 0.1,\n",
+      "  \"summary_proj_to_labels\": true,\n",
+      "  \"summary_type\": \"cls_index\",\n",
+      "  \"summary_use_proj\": true,\n",
+      "  \"transformers_version\": \"4.29.0.dev0\",\n",
+      "  \"use_cache\": true,\n",
+      "  \"vocab_size\": 50264\n",
+      "}\n",
+      "\n",
+      "Downloading (…)olve/main/vocab.json: 100%|█| 1.71M/1.71M [00:00<00:00, 3.73MB/s]\n",
+      "Downloading (…)olve/main/merges.txt: 100%|█| 1.27M/1.27M [00:00<00:00, 5.74MB/s]\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file vocab.json from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/vocab.json\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file merges.txt from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/merges.txt\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file tokenizer.json from cache at None\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file added_tokens.json from cache at None\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file special_tokens_map.json from cache at None\n",
+      "[INFO|tokenization_utils_base.py:1809] 2023-04-16 19:47:47,652 >> loading file tokenizer_config.json from cache at None\n",
+      "[INFO|configuration_utils.py:668] 2023-04-16 19:47:47,652 >> loading configuration file config.json from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/config.json\n",
+      "[INFO|configuration_utils.py:720] 2023-04-16 19:47:47,653 >> Model config GPT2Config {\n",
+      "  \"_name_or_path\": \"sberbank-ai/rugpt3small_based_on_gpt2\",\n",
+      "  \"activation_function\": \"gelu_new\",\n",
+      "  \"architectures\": [\n",
+      "    \"GPT2LMHeadModel\"\n",
+      "  ],\n",
+      "  \"attn_pdrop\": 0.1,\n",
+      "  \"bos_token_id\": 50256,\n",
+      "  \"embd_pdrop\": 0.1,\n",
+      "  \"eos_token_id\": 50256,\n",
+      "  \"gradient_checkpointing\": false,\n",
+      "  \"initializer_range\": 0.02,\n",
+      "  \"layer_norm_epsilon\": 1e-05,\n",
+      "  \"model_type\": \"gpt2\",\n",
+      "  \"n_ctx\": 2048,\n",
+      "  \"n_embd\": 768,\n",
+      "  \"n_head\": 12,\n",
+      "  \"n_inner\": null,\n",
+      "  \"n_layer\": 12,\n",
+      "  \"n_positions\": 2048,\n",
+      "  \"reorder_and_upcast_attn\": false,\n",
+      "  \"resid_pdrop\": 0.1,\n",
+      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
+      "  \"scale_attn_weights\": true,\n",
+      "  \"summary_activation\": null,\n",
+      "  \"summary_first_dropout\": 0.1,\n",
+      "  \"summary_proj_to_labels\": true,\n",
+      "  \"summary_type\": \"cls_index\",\n",
+      "  \"summary_use_proj\": true,\n",
+      "  \"transformers_version\": \"4.29.0.dev0\",\n",
+      "  \"use_cache\": true,\n",
+      "  \"vocab_size\": 50264\n",
+      "}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[INFO|configuration_utils.py:668] 2023-04-16 19:47:47,725 >> loading configuration file config.json from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/config.json\n",
+      "[INFO|configuration_utils.py:720] 2023-04-16 19:47:47,725 >> Model config GPT2Config {\n",
+      "  \"_name_or_path\": \"sberbank-ai/rugpt3small_based_on_gpt2\",\n",
+      "  \"activation_function\": \"gelu_new\",\n",
+      "  \"architectures\": [\n",
+      "    \"GPT2LMHeadModel\"\n",
+      "  ],\n",
+      "  \"attn_pdrop\": 0.1,\n",
+      "  \"bos_token_id\": 50256,\n",
+      "  \"embd_pdrop\": 0.1,\n",
+      "  \"eos_token_id\": 50256,\n",
+      "  \"gradient_checkpointing\": false,\n",
+      "  \"initializer_range\": 0.02,\n",
+      "  \"layer_norm_epsilon\": 1e-05,\n",
+      "  \"model_type\": \"gpt2\",\n",
+      "  \"n_ctx\": 2048,\n",
+      "  \"n_embd\": 768,\n",
+      "  \"n_head\": 12,\n",
+      "  \"n_inner\": null,\n",
+      "  \"n_layer\": 12,\n",
+      "  \"n_positions\": 2048,\n",
+      "  \"reorder_and_upcast_attn\": false,\n",
+      "  \"resid_pdrop\": 0.1,\n",
+      "  \"scale_attn_by_inverse_layer_idx\": false,\n",
+      "  \"scale_attn_weights\": true,\n",
+      "  \"summary_activation\": null,\n",
+      "  \"summary_first_dropout\": 0.1,\n",
+      "  \"summary_proj_to_labels\": true,\n",
+      "  \"summary_type\": \"cls_index\",\n",
+      "  \"summary_use_proj\": true,\n",
+      "  \"transformers_version\": \"4.29.0.dev0\",\n",
+      "  \"use_cache\": true,\n",
+      "  \"vocab_size\": 50264\n",
+      "}\n",
+      "\n",
+      "[WARNING|logging.py:280] 2023-04-16 19:47:47,765 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
+      "Downloading pytorch_model.bin: 100%|█████████| 551M/551M [00:36<00:00, 15.2MB/s]\n",
+      "[INFO|modeling_utils.py:2534] 2023-04-16 19:48:24,907 >> loading weights file pytorch_model.bin from cache at /home/kamil/.cache/huggingface/hub/models--sberbank-ai--rugpt3small_based_on_gpt2/snapshots/d64244b316057f71e745cc92be1dcfe7853d9d18/pytorch_model.bin\n",
+      "[INFO|configuration_utils.py:575] 2023-04-16 19:48:25,102 >> Generate config GenerationConfig {\n",
+      "  \"_from_model_config\": true,\n",
+      "  \"bos_token_id\": 50256,\n",
+      "  \"eos_token_id\": 50256,\n",
+      "  \"transformers_version\": \"4.29.0.dev0\"\n",
+      "}\n",
+      "\n",
+      "[INFO|modeling_utils.py:3190] 2023-04-16 19:48:26,046 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
+      "\n",
+      "[INFO|modeling_utils.py:3198] 2023-04-16 19:48:26,046 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at sberbank-ai/rugpt3small_based_on_gpt2.\n",
+      "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
+      "[INFO|modeling_utils.py:2839] 2023-04-16 19:48:26,570 >> Generation config file not found, using a generation config created from the model config.\n",
+      "Running tokenizer on dataset:   0%|              | 0/720 [00:00<?, ? examples/s]04/16/2023 19:48:26 - INFO - datasets.arrow_dataset - Caching processed dataset at /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-02f845f719a14b5e.arrow\n",
+      "Running tokenizer on dataset:   0%|               | 0/80 [00:00<?, ? examples/s]04/16/2023 19:48:26 - INFO - datasets.arrow_dataset - Caching processed dataset at /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-b826039bbf723e44.arrow\n",
+      "Grouping texts in chunks of 2048:   0%|          | 0/720 [00:00<?, ? examples/s]04/16/2023 19:48:26 - INFO - datasets.arrow_dataset - Caching processed dataset at /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-218f03f685e5499a.arrow\n",
+      "Grouping texts in chunks of 2048:   0%|           | 0/80 [00:00<?, ? examples/s]04/16/2023 19:48:27 - INFO - datasets.arrow_dataset - Caching processed dataset at /home/kamil/.cache/huggingface/datasets/text/default-94a5e2bc6bcfdc2e/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-ad1660cb69988af0.arrow\n",
+      "Downloading builder script: 100%|██████████| 4.20k/4.20k [00:00<00:00, 4.48MB/s]\n",
+      "/home/kamil/.local/lib/python3.10/site-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+      "  warnings.warn(\n",
+      "[INFO|trainer.py:1769] 2023-04-16 19:48:28,570 >> ***** Running training *****\n",
+      "[INFO|trainer.py:1770] 2023-04-16 19:48:28,570 >>   Num examples = 92\n",
+      "[INFO|trainer.py:1771] 2023-04-16 19:48:28,570 >>   Num Epochs = 3\n",
+      "[INFO|trainer.py:1772] 2023-04-16 19:48:28,570 >>   Instantaneous batch size per device = 1\n",
+      "[INFO|trainer.py:1773] 2023-04-16 19:48:28,570 >>   Total train batch size (w. parallel, distributed & accumulation) = 1\n",
+      "[INFO|trainer.py:1774] 2023-04-16 19:48:28,570 >>   Gradient Accumulation steps = 1\n",
+      "[INFO|trainer.py:1775] 2023-04-16 19:48:28,570 >>   Total optimization steps = 276\n",
+      "[INFO|trainer.py:1776] 2023-04-16 19:48:28,570 >>   Number of trainable parameters = 125,231,616\n",
+      "  0%|                                                   | 0/276 [00:00<?, ?it/s]Traceback (most recent call last):\n",
+      "  File \"/home/kamil/Documents/SHAD/ML/Part 2/Seminar 7/run_clm.py\", line 635, in <module>\n",
+      "    main()\n",
+      "  File \"/home/kamil/Documents/SHAD/ML/Part 2/Seminar 7/run_clm.py\", line 583, in main\n",
+      "    train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/trainer.py\", line 1662, in train\n",
+      "    return inner_training_loop(\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/trainer.py\", line 1929, in _inner_training_loop\n",
+      "    tr_loss_step = self.training_step(model, inputs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/trainer.py\", line 2699, in training_step\n",
+      "    loss = self.compute_loss(model, inputs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/trainer.py\", line 2731, in compute_loss\n",
+      "    outputs = model(**inputs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
+      "    return forward_call(*args, **kwargs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py\", line 1075, in forward\n",
+      "    transformer_outputs = self.transformer(\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
+      "    return forward_call(*args, **kwargs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py\", line 899, in forward\n",
+      "    outputs = block(\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
+      "    return forward_call(*args, **kwargs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py\", line 389, in forward\n",
+      "    attn_outputs = self.attn(\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
+      "    return forward_call(*args, **kwargs)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py\", line 330, in forward\n",
+      "    attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)\n",
+      "  File \"/home/kamil/.local/lib/python3.10/site-packages/transformers/models/gpt2/modeling_gpt2.py\", line 185, in _attn\n",
+      "    attn_weights = attn_weights / torch.full(\n",
+      "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 192.00 MiB (GPU 0; 7.79 GiB total capacity; 6.07 GiB already allocated; 171.81 MiB free; 6.09 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n",
+      "  0%|                                                   | 0/276 [00:01<?, ?it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "!python3 run_clm.py \\\n",
+    "    --model_name_or_path sberbank-ai/rugpt3small_based_on_gpt2 \\\n",
+    "    --train_file train.txt \\\n",
+    "    --validation_file valid.txt \\\n",
+    "    --per_device_train_batch_size 1 \\\n",
+    "    --per_device_eval_batch_size 1 \\\n",
+    "    --block_size 2048 \\\n",
+    "    --dataset_config_name plain_text \\\n",
+    "    --do_train \\\n",
+    "    --do_eval \\\n",
+    "    --output_dir models/essays2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "QvgntLymArg3"
+   },
+   "source": [
+    "## Evaluate model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "id": "csHcDJXFDdaW"
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "id": "TJxPg-cJDhAB"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<torch._C.Generator at 0x7fe5314a4c50>"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.random.seed(42)\n",
+    "torch.manual_seed(42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "id": "AkUrzKsy_16F"
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import GPT2LMHeadModel, GPT2Tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "id": "x_EMbgO0BTvb"
+   },
+   "outputs": [],
+   "source": [
+    "tok = GPT2Tokenizer.from_pretrained(\"models/essays\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "id": "Fjy0GAuQBYpA"
+   },
+   "outputs": [],
+   "source": [
+    "model = GPT2LMHeadModel.from_pretrained(\"models/essays\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {
+    "collapsed": true,
+    "id": "irh4H-HDBb6V"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50264, 768)\n",
+       "    (wpe): Embedding(2048, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2Attention(\n",
+       "          (c_attn): Conv1D()\n",
+       "          (c_proj): Conv1D()\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D()\n",
+       "          (c_proj): Conv1D()\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=50264, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.cuda()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {
+    "id": "hQY6A5q7Bd4O"
+   },
+   "outputs": [],
+   "source": [
+    "text = \"<s>Тема: «В чем смысл жизни?»\\nСочинение: \"\n",
+    "inpt = tok.encode(text, return_tensors=\"pt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {
+    "id": "1gfJFmeOBj_t"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
+     ]
+    }
+   ],
+   "source": [
+    "out = model.generate(inpt.cuda(), max_length=200, repetition_penalty=5.0, do_sample=True, top_k=5, top_p=0.95, temperature=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 123
+    },
+    "id": "gWZ9SUCxB2Ki",
+    "outputId": "31d8e1a3-376f-4f27-bd11-ba59a44983eb"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<s>Тема: «В чем смысл жизни?»\n",
+      "Сочинение: 📹Как часто в наше время мы слышим фразу \"жить надо так, чтобы было хорошо всем\". Однако не все могут себе позволить жить по-другому. В современном мире многие люди хотят изменить свою жизнь к лучшему и сделать ее комфортной для всех без исключения граждан нашей страны.</span] (по пьесе Мольера) \n",
+      " Существование системы образования является одним из важнейших условий становления цивилизованного общества на земле – формирования личности гражданина как носителя социально значимых ценностей.Начнем с определения понятия образование - наука о человеке или его способности организовывать свои действия во внешней среде посредством усвоения знаний об окружающем нас обществе; рассмотрим основные функции обучения, которые выполняет школьное учреждение : формирование у детей культуры общения со взрослыми людьми через обучение умению общаться при помощи словаря иностранных слов — это важнейший фактор социализации человека от рождения до смерти || • воспитание нравственности между детьми дошкольного возраста /под ред Н\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tok.decode(out[0]))"
+   ]
+  }
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "name": "RuGPT3FinetuneHF.ipynb",
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

Generate_text.py

@@ -0,0 +1,28 @@
+import numpy as np
+import torch
+
+np.random.seed(17)
+torch.manual_seed(17)
+
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+def load_tokenizer_and_model(model_name_or_path, device):
+    return GPT2Tokenizer.from_pretrained(model_name_or_path), GPT2LMHeadModel.from_pretrained(model_name_or_path).to(device)
+
+def generate(
+    model, tok, text, device, 
+    do_sample=True, max_length=200, repetition_penalty=5.0,
+    top_k=5, top_p=0.95, temperature=1,
+    num_beams=None,
+    no_repeat_ngram_size=3
+    ):
+    input_ids = tok.encode(text, return_tensors="pt").to(device)
+    out = model.generate(
+        input_ids.to(device),
+        max_length=max_length,
+        repetition_penalty=repetition_penalty,
+        do_sample=do_sample,
+        top_k=top_k, top_p=top_p, temperature=temperature,
+        num_beams=num_beams, no_repeat_ngram_size=no_repeat_ngram_size
+        )
+    return list(map(tok.decode, out))

README.md

@@ -0,0 +1,17 @@
+---
+title: Transformer Homework
+emoji: 😻
+colorFrom: gray
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.17.0
+app_file: app.py
+pinned: false
+license: openrail
+duplicated_from: IkIzma/Transformer_Homework
+---
+
+Эта модель https://huggingface.co/ai-forever/rugpt3small_based_on_gpt2, которая была дообучена на датасете сочинений https://www.dropbox.com/s/oa3v9c7g9bp40xw. Весь код а также модель с датасетом представлены в репозитории.
+Основной скрипт для finetune это run_clm.py, который запускается внутри ноутбука FineTune.ipynb
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,30 @@
+import streamlit as st
+from Generate_text import generate, load_tokenizer_and_model
+
+device = "cpu"
+tok, model = load_tokenizer_and_model("models/essays", device)
+
+st.markdown("## Генератор сочинений")
+st.markdown("Это приложение представляет из себя примитивный генератор сочинений. В качестве основной модели был взят трансформер RuGPT от Сбера https://huggingface.co/ai-forever/rugpt3small_based_on_gpt2. После этого на том же HF был скачан датасет с сочинениями на русском языке, который использовался для finetuning'а (см. ReadMe)")
+
+st.markdown("Для работы с приложением укажите максимальный размер сочинения, который хотите получить. Далее напишите тему. Для генерации сочинения длиной в 200 токенов потребуется подождать 6-7 минут")
+st.markdown("Модель rugpt3small_based_on_gpt2 маленькая и датасет сочинений, на котором она дообучалась, тоже маленький, поэтому не стоит рассчитывать, что получившееся сочинение будет хорошо отражать выбранную тему). Но, по крайней мере, текст получается осмысленным и его стиль вполне соответствует стилю написания сочинений.")
+st.markdown("Чем меньше макисмальная длина, тем хуже получается итоговый результат. Относительно неплохие сочинения получаются при длине 500+, но по времни такая генерация занимает десятки минут")
+with st.columns(3)[1]:
+    st.markdown("<img width=200px src='https://ps-static.cdn-tinkoff.ru/static/ai-pushkin/portrait-2021-12-10-12-24-56.png'>", unsafe_allow_html=True)
+# ^-- можно показывать пользователю текст, картинки, ограниченное подмножество html - всё как в jupyter
+
+max_len = st.text_area("Максимальная длина")
+text = st.text_area("Тема сочинения")
+# ^-- показать текстовое поле. В поле text лежит строка, которая находится там в данный момент
+thesis = "<s>Тема:" + "«" + text + "»." + "\nСочинение: " 
+
+if max_len != "":
+    begin_index = len(thesis)
+    max_len = int(max_len)
+    generated = generate(model, tok, thesis, device, max_length=max_len, num_beams=10)
+    end_index = (generated[0]).find("</s>")
+    
+    st.markdown(f"Сочинение:\n")
+    st.markdown(generated[0][begin_index:end_index])
+# выводим результаты модели в текстовое поле, на потеху пользователю

models/essays/README.md

@@ -0,0 +1,55 @@
+---
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: essays
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# essays
+
+This model is a fine-tuned version of [sberbank-ai/rugpt3small_based_on_gpt2](https://huggingface.co/sberbank-ai/rugpt3small_based_on_gpt2) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.7676
+- Accuracy: 0.4758
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.29.0.dev0
+- Pytorch 1.9.1+cu111
+- Datasets 2.11.0
+- Tokenizers 0.13.3

models/essays/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|endoftext|>": 50257
+}

models/essays/all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.4757639906638441,
+    "eval_loss": 2.767648458480835,
+    "eval_runtime": 1.0081,
+    "eval_samples": 9,
+    "eval_samples_per_second": 8.928,
+    "eval_steps_per_second": 8.928,
+    "perplexity": 15.921150708373386,
+    "train_loss": 2.7605772156646284,
+    "train_runtime": 87.21,
+    "train_samples": 92,
+    "train_samples_per_second": 3.165,
+    "train_steps_per_second": 3.165
+}

models/essays/config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "sberbank-ai/rugpt3small_based_on_gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 2048,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50264
+}

models/essays/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 3.0,
+    "eval_accuracy": 0.4757639906638441,
+    "eval_loss": 2.767648458480835,
+    "eval_runtime": 1.0081,
+    "eval_samples": 9,
+    "eval_samples_per_second": 8.928,
+    "eval_steps_per_second": 8.928,
+    "perplexity": 15.921150708373386
+}

models/essays/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.29.0.dev0"
+}

models/essays/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f145dc6ca97655187b357bc9b9484ecde90f445362bae691bc45c2faf0d587
+size 551312489

models/essays/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/essays/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/essays/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 2.7605772156646284,
+    "train_runtime": 87.21,
+    "train_samples": 92,
+    "train_samples_per_second": 3.165,
+    "train_steps_per_second": 3.165
+}

models/essays/trainer_state.json

@@ -0,0 +1,25 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "global_step": 276,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 3.0,
+      "step": 276,
+      "total_flos": 288466403328000.0,
+      "train_loss": 2.7605772156646284,
+      "train_runtime": 87.21,
+      "train_samples_per_second": 3.165,
+      "train_steps_per_second": 3.165
+    }
+  ],
+  "max_steps": 276,
+  "num_train_epochs": 3,
+  "total_flos": 288466403328000.0,
+  "trial_name": null,
+  "trial_params": null
+}

models/essays/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc90d13a3c1e167dedc3e673a03634beb1998f6a5a2653d6c61b9729498f4d7
+size 3567

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch

run_clm.py

@@ -0,0 +1,635 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+
+import logging
+import math
+import os
+import sys
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Optional
+
+import datasets
+import evaluate
+import torch
+from datasets import load_dataset
+
+import transformers
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    is_torch_tpu_available,
+    set_seed,
+)
+from transformers.testing_utils import CaptureLogger
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils.versions import require_version
+
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.29.0.dev0")
+
+require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
+
+logger = logging.getLogger(__name__)
+
+
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
+    )
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
+                "with private models)."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+    low_cpu_mem_usage: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded."
+                "set True will benefit LLM loading time and RAM consumption."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
+    block_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional input sequence length after tokenization. "
+                "The training dataset will be truncated in block of this size for training. "
+                "Default to the model max input length for single sentence inputs (take into account special tokens)."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+
+    def __post_init__(self):
+        if self.streaming:
+            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")
+
+        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
+            raise ValueError("Need either a dataset name or a training/validation file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            use_auth_token=True if model_args.use_auth_token else None,
+            streaming=data_args.streaming,
+        )
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+                streaming=data_args.streaming,
+            )
+            raw_datasets["train"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+                streaming=data_args.streaming,
+            )
+    else:
+        data_files = {}
+        dataset_args = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        extension = (
+            data_args.train_file.split(".")[-1]
+            if data_args.train_file is not None
+            else data_args.validation_file.split(".")[-1]
+        )
+        if extension == "txt":
+            extension = "text"
+            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
+        raw_datasets = load_dataset(
+            extension,
+            data_files=data_files,
+            cache_dir=model_args.cache_dir,
+            use_auth_token=True if model_args.use_auth_token else None,
+            **dataset_args,
+        )
+        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+                **dataset_args,
+            )
+            raw_datasets["train"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+                **dataset_args,
+            )
+
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "use_auth_token": True if model_args.use_auth_token else None,
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = AutoModelForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            use_auth_token=True if model_args.use_auth_token else None,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=model_args.low_cpu_mem_usage,
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
+    # on a small vocab and want a smaller embedding size, remove this test.
+    embedding_size = model.get_input_embeddings().weight.shape[0]
+    if len(tokenizer) > embedding_size:
+        model.resize_token_embeddings(len(tokenizer))
+
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    if training_args.do_train:
+        column_names = list(raw_datasets["train"].features)
+    else:
+        column_names = list(raw_datasets["validation"].features)
+    text_column_name = "text" if "text" in column_names else column_names[0]
+
+    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
+    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
+
+    def tokenize_function(examples):
+        with CaptureLogger(tok_logger) as cl:
+            output = tokenizer(examples[text_column_name])
+        # clm input could be much much longer than block_size
+        if "Token indices sequence length is longer than the" in cl.out:
+            tok_logger.warning(
+                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
+                " before being passed to the model."
+            )
+        return output
+
+    with training_args.main_process_first(desc="dataset map tokenization"):
+        if not data_args.streaming:
+            tokenized_datasets = raw_datasets.map(
+                tokenize_function,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc="Running tokenizer on dataset",
+            )
+        else:
+            tokenized_datasets = raw_datasets.map(
+                tokenize_function,
+                batched=True,
+                remove_columns=column_names,
+            )
+
+    if data_args.block_size is None:
+        block_size = tokenizer.model_max_length
+        if block_size > 1024:
+            logger.warning(
+                "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
+                " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
+                " override this default with `--block_size xxx`."
+            )
+            block_size = 1024
+    else:
+        if data_args.block_size > tokenizer.model_max_length:
+            logger.warning(
+                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model"
+                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
+            )
+        block_size = min(data_args.block_size, tokenizer.model_max_length)
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
+        # customize this part to your needs.
+        if total_length >= block_size:
+            total_length = (total_length // block_size) * block_size
+        # Split by chunks of max_len.
+        result = {
+            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+            for k, t in concatenated_examples.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
+    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
+    # to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
+
+    with training_args.main_process_first(desc="grouping texts together"):
+        if not data_args.streaming:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc=f"Grouping texts in chunks of {block_size}",
+            )
+        else:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+            )
+
+    if training_args.do_train:
+        if "train" not in tokenized_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = lm_datasets["train"]
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+
+    if training_args.do_eval:
+        if "validation" not in tokenized_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = lm_datasets["validation"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+
+        def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
+            return logits.argmax(dim=-1)
+
+        metric = evaluate.load("accuracy")
+
+        def compute_metrics(eval_preds):
+            preds, labels = eval_preds
+            # preds have the same shape as the labels, after the argmax(-1) has been calculated
+            # by preprocess_logits_for_metrics but we need to shift the labels
+            labels = labels[:, 1:].reshape(-1)
+            preds = preds[:, :-1].reshape(-1)
+            return metric.compute(predictions=preds, references=labels)
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        # Data collator will default to DataCollatorWithPadding, so we change it.
+        data_collator=default_data_collator,
+        compute_metrics=compute_metrics if training_args.do_eval and not is_torch_tpu_available() else None,
+        preprocess_logits_for_metrics=preprocess_logits_for_metrics
+        if training_args.do_eval and not is_torch_tpu_available()
+        else None,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        metrics = train_result.metrics
+
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
+    if data_args.dataset_name is not None:
+        kwargs["dataset_tags"] = data_args.dataset_name
+        if data_args.dataset_config_name is not None:
+            kwargs["dataset_args"] = data_args.dataset_config_name
+            kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
+        else:
+            kwargs["dataset"] = data_args.dataset_name
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()