Upload convert_to_long.ipynb

convert_to_long.ipynb

@@ -0,0 +1,1091 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "426ead53-211f-4f07-b327-ce1f75f923e0",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting transformers\n",
+      "  Downloading transformers-4.41.2-py3-none-any.whl (9.1 MB)\n",
+      "\u001b[K     |████████████████████████████████| 9.1 MB 26.1 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting sentencepiece\n",
+      "  Downloading sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
+      "\u001b[K     |████████████████████████████████| 1.3 MB 94.2 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting filelock\n",
+      "  Downloading filelock-3.15.4-py3-none-any.whl (16 kB)\n",
+      "Collecting numpy>=1.17\n",
+      "  Downloading numpy-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.3 MB)\n",
+      "\u001b[K     |████████████████████████████████| 19.3 MB 97.7 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /home/user/miniconda/lib/python3.9/site-packages (from transformers) (6.0.1)\n",
+      "Collecting safetensors>=0.4.1\n",
+      "  Downloading safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
+      "\u001b[K     |████████████████████████████████| 1.2 MB 87.6 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting tokenizers<0.20,>=0.19\n",
+      "  Downloading tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
+      "\u001b[K     |████████████████████████████████| 3.6 MB 111.0 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting regex!=2019.12.17\n",
+      "  Downloading regex-2024.5.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (774 kB)\n",
+      "\u001b[K     |████████████████████████████████| 774 kB 96.6 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /home/user/miniconda/lib/python3.9/site-packages (from transformers) (24.1)\n",
+      "Collecting huggingface-hub<1.0,>=0.23.0\n",
+      "  Downloading huggingface_hub-0.23.4-py3-none-any.whl (402 kB)\n",
+      "\u001b[K     |████████████████████████████████| 402 kB 118.3 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /home/user/miniconda/lib/python3.9/site-packages (from transformers) (4.61.2)\n",
+      "Requirement already satisfied: requests in /home/user/miniconda/lib/python3.9/site-packages (from transformers) (2.32.3)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/user/miniconda/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (4.12.2)\n",
+      "Collecting fsspec>=2023.5.0\n",
+      "  Downloading fsspec-2024.6.0-py3-none-any.whl (176 kB)\n",
+      "\u001b[K     |████████████████████████████████| 176 kB 127.1 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: urllib3<3,>=1.21.1 in /home/user/miniconda/lib/python3.9/site-packages (from requests->transformers) (1.26.6)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/user/miniconda/lib/python3.9/site-packages (from requests->transformers) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /home/user/miniconda/lib/python3.9/site-packages (from requests->transformers) (2.10)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /home/user/miniconda/lib/python3.9/site-packages (from requests->transformers) (2021.5.30)\n",
+      "Installing collected packages: fsspec, filelock, huggingface-hub, tokenizers, safetensors, regex, numpy, transformers, sentencepiece\n",
+      "Successfully installed filelock-3.15.4 fsspec-2024.6.0 huggingface-hub-0.23.4 numpy-2.0.0 regex-2024.5.15 safetensors-0.4.3 sentencepiece-0.2.0 tokenizers-0.19.1 transformers-4.41.2\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install transformers sentencepiece"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5bee7472-9458-428b-b315-2a6579ecbcd7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0051538944244384766,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "config.json",
+       "rate": null,
+       "total": 615,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b071a2288c7848b9a86e9c12e7c5105c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/615 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.004002094268798828,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "model.safetensors",
+       "rate": null,
+       "total": 1115567652,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "524b814ec5164db5a86203078b303a9c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoModel\n",
+    "\n",
+    "xlm_base = AutoModel.from_pretrained(\"xlm-roberta-base\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "3c5bfded-aa37-4c1b-8fe8-c6fe64e184e7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0036783218383789062,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "config.json",
+       "rate": null,
+       "total": 616,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "67617bf347cf46bb95d1559e756b3e64",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/616 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.00390625,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "model.safetensors",
+       "rate": null,
+       "total": 2244817354,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7cd0c7418dc4483083054e541b6dd1ac",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/2.24G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "xlm_large = AutoModel.from_pretrained(\"xlm-roberta-large\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ba1b9da7-eafe-4c81-b47d-63ae94585b37",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "XLMRobertaModel(\n",
+       "  (embeddings): XLMRobertaEmbeddings(\n",
+       "    (word_embeddings): Embedding(250002, 768, padding_idx=1)\n",
+       "    (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
+       "    (token_type_embeddings): Embedding(1, 768)\n",
+       "    (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "    (dropout): Dropout(p=0.1, inplace=False)\n",
+       "  )\n",
+       "  (encoder): XLMRobertaEncoder(\n",
+       "    (layer): ModuleList(\n",
+       "      (0-11): 12 x XLMRobertaLayer(\n",
+       "        (attention): XLMRobertaAttention(\n",
+       "          (self): XLMRobertaSelfAttention(\n",
+       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "          (output): XLMRobertaSelfOutput(\n",
+       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (intermediate): XLMRobertaIntermediate(\n",
+       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          (intermediate_act_fn): GELUActivation()\n",
+       "        )\n",
+       "        (output): XLMRobertaOutput(\n",
+       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (pooler): XLMRobertaPooler(\n",
+       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "    (activation): Tanh()\n",
+       "  )\n",
+       ")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "xlm_base"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "53fcc03e-d6f5-41c3-9dc2-2922e6747896",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[ 0.0578, -0.0071, -0.0068,  ...,  0.0061, -0.0260, -0.0291],\n",
+       "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
+       "        [-0.1564, -0.0728, -0.2477,  ..., -0.0778, -0.3088, -0.0090],\n",
+       "        ...,\n",
+       "        [ 0.0118,  0.0458, -0.0054,  ..., -0.0865,  0.0374,  0.0040],\n",
+       "        [ 0.0525, -0.0270, -0.0141,  ..., -0.0552,  0.0349,  0.0274],\n",
+       "        [-0.0479, -0.0293,  0.1079,  ..., -0.0825,  0.2908,  0.0861]])"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "old_embeddings = xlm_base.embeddings.position_embeddings.weight.data\n",
+    "old_embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "fc69704e-51a6-489f-8b6a-b776d1027a2a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([514, 768])"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "old_embeddings.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "714faf1f-ec8c-4e39-986f-5624658c8a9d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "\n",
+    "new_embeddings = torch.zeros((2050, 768))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "4eb8dfff-57a7-4e2c-a060-54d7ec84e2d7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "new_embeddings[:514, :] = old_embeddings.clone()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "0a4dcbb8-20f1-476c-83b0-199d3e406888",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "514 1026\n",
+      "1026 1538\n",
+      "1538 2050\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_pos = 514\n",
+    "\n",
+    "for i in range(3):\n",
+    "    start_idx = num_pos+512*i\n",
+    "    end_idx = start_idx + 512\n",
+    "    new_embeddings[start_idx:end_idx, :] = old_embeddings[2:, :].clone()\n",
+    "    print(start_idx, end_idx)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "6530a6fa-fe8e-4d37-9e35-1272643d04c4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of XLMRobertaModel were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized because the shapes did not match:\n",
+      "- roberta.embeddings.position_embeddings.weight: found shape torch.Size([514, 768]) in the checkpoint and torch.Size([2050, 768]) in the model instantiated\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "xlm_base = AutoModel.from_pretrained(\"xlm-roberta-base\", max_position_embeddings=2050, ignore_mismatched_sizes=True)\n",
+    "\n",
+    "xlm_base.embeddings.position_embeddings.weight.data = new_embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "ae8250bb-2d81-4ad9-b281-84d9ad3d9114",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "with torch.no_grad():\n",
+    "    xlm_base(input_ids=seq_2048)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "1e77aea0-65ff-403c-87cc-14afd87d7646",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([2050, 768])\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[ 0.0578, -0.0071, -0.0068,  ...,  0.0061, -0.0260, -0.0291],\n",
+       "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
+       "        [-0.1564, -0.0728, -0.2477,  ..., -0.0778, -0.3088, -0.0090],\n",
+       "        ...,\n",
+       "        [ 0.0118,  0.0458, -0.0054,  ..., -0.0865,  0.0374,  0.0040],\n",
+       "        [ 0.0525, -0.0270, -0.0141,  ..., -0.0552,  0.0349,  0.0274],\n",
+       "        [-0.0479, -0.0293,  0.1079,  ..., -0.0825,  0.2908,  0.0861]])"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(xlm_base.embeddings.position_embeddings.weight.data.shape)\n",
+    "xlm_base.embeddings.position_embeddings.weight.data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "74297602-9c8e-4341-96cc-b26435046082",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "514 1026\n",
+      "1026 1538\n",
+      "1538 2050\n"
+     ]
+    }
+   ],
+   "source": [
+    "old_embeddings = xlm_large.embeddings.position_embeddings.weight.data\n",
+    "\n",
+    "new_embeddings = torch.zeros((2050, old_embeddings.shape[1]))\n",
+    "\n",
+    "new_embeddings[:514, :] = old_embeddings.clone()\n",
+    "\n",
+    "num_pos = 514\n",
+    "\n",
+    "for i in range(3):\n",
+    "    start_idx = num_pos+512*i\n",
+    "    end_idx = start_idx + 512\n",
+    "    new_embeddings[start_idx:end_idx, :] = old_embeddings[2:, :].clone()\n",
+    "    print(start_idx, end_idx)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "57ccd5e8-667a-4fae-be04-55c14aa1a316",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of XLMRobertaModel were not initialized from the model checkpoint at xlm-roberta-large and are newly initialized because the shapes did not match:\n",
+      "- roberta.embeddings.position_embeddings.weight: found shape torch.Size([514, 1024]) in the checkpoint and torch.Size([2050, 1024]) in the model instantiated\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "xlm_large = AutoModel.from_pretrained(\"xlm-roberta-large\", max_position_embeddings=2050, ignore_mismatched_sizes=True)\n",
+    "\n",
+    "xlm_large.embeddings.position_embeddings.weight.data = new_embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "1b584a7b-19b0-45ac-824c-57e37bf2bf75",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
+      "Token is valid (permission: write).\n",
+      "Your token has been saved to /home/user/.cache/huggingface/token\n",
+      "Login successful\n"
+     ]
+    }
+   ],
+   "source": [
+    "!huggingface-cli login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "3b5f110a-c30d-4b83-997b-384cf159f1c4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.004299163818359375,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "model.safetensors",
+       "rate": null,
+       "total": 2245898632,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d1193e52abce44fdba074b5996560954",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/2.25G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/nbroad/xlm-roberta-large-2048/commit/8270d498aad31c695866c0fbcf5c7eb932b69590', commit_message='Upload model', commit_description='', oid='8270d498aad31c695866c0fbcf5c7eb932b69590', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "xlm_large.push_to_hub(\"nbroad/xlm-roberta-large-2048\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "b0448ee6-e471-44f3-ac8e-e18ed66adcbb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.003611326217651367,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "model.safetensors",
+       "rate": null,
+       "total": 1116915696,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2c2058a26f1641ea8645220c91f50d73",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/nbroad/xlm-roberta-base-2048/commit/4daea3ba9080f535f1fc819cbc76453248252f6b', commit_message='Upload model', commit_description='', oid='4daea3ba9080f535f1fc819cbc76453248252f6b', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "xlm_base.push_to_hub(\"nbroad/xlm-roberta-base-2048\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "5bae84dc-a5c1-40cd-9eaf-a42e0a4276f3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-base\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "1bbf303d-8613-43dd-9a1e-4fb597920b78",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "tokenizer.model_max_length = 2048"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "07add3fd-e539-43ef-be9b-9409e5f44e40",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0037915706634521484,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "README.md",
+       "rate": null,
+       "total": 5174,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a2b9909d0e784de0bc03e1edc6b8e21a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/5.17k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0038919448852539062,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "sentencepiece.bpe.model",
+       "rate": null,
+       "total": 5069051,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9d22217b68554e9ab3c67be3dce5d9a3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0034050941467285156,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "Upload 2 LFS files",
+       "rate": null,
+       "total": 2,
+       "unit": "it",
+       "unit_divisor": 1000,
+       "unit_scale": false
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e0f6e1f6662c44a1b434ee81980cbe97",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Upload 2 LFS files:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0035247802734375,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "tokenizer.json",
+       "rate": null,
+       "total": 17082734,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6db7d931fd2846ad9fa36c0612b9aa71",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/nbroad/xlm-roberta-base-2048/commit/82129a4b55ea013dd3fa2816c24696814b08a6b4', commit_message='Upload tokenizer', commit_description='', oid='82129a4b55ea013dd3fa2816c24696814b08a6b4', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.push_to_hub(\"nbroad/xlm-roberta-base-2048\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "2ab6599e-3621-469f-8bdd-389cf7f60088",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0036215782165527344,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "tokenizer_config.json",
+       "rate": null,
+       "total": 25,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4c30d397f9344f2ca227d117618d25ce",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.003586292266845703,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "sentencepiece.bpe.model",
+       "rate": null,
+       "total": 5069051,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d094dd6bccd84006b0d7347d8dbdccfe",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.003679037094116211,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "tokenizer.json",
+       "rate": null,
+       "total": 9096718,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ff510cc6c58c42dc93c74d607543b734",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/9.10M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.003905057907104492,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "README.md",
+       "rate": null,
+       "total": 5174,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b1d4be8d4e744082b9f48d6a92cc5f6c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/5.17k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0074939727783203125,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "Upload 2 LFS files",
+       "rate": null,
+       "total": 2,
+       "unit": "it",
+       "unit_divisor": 1000,
+       "unit_scale": false
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e32b5391525c4b448fadf8bbaf434f91",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Upload 2 LFS files:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.0040264129638671875,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "sentencepiece.bpe.model",
+       "rate": null,
+       "total": 5069051,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d9260ad639034eacbc3ee0f3cf970751",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/json": {
+       "ascii": false,
+       "bar_format": null,
+       "colour": null,
+       "elapsed": 0.003367185592651367,
+       "initial": 0,
+       "n": 0,
+       "ncols": null,
+       "nrows": null,
+       "postfix": null,
+       "prefix": "tokenizer.json",
+       "rate": null,
+       "total": 17082734,
+       "unit": "B",
+       "unit_divisor": 1000,
+       "unit_scale": true
+      },
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "71db7613c7d347128e7ddb745e1e5f6e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "ename": "AttributeError",
+     "evalue": "'CommitInfo' object has no attribute 'push_to_hub'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[45], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m AutoTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxlm-roberta-large\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mpush_to_hub(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbroad/xlm-roberta-large-2048\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      2\u001b[0m tokenizer\u001b[38;5;241m.\u001b[39mmodel_max_length \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2048\u001b[39m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpush_to_hub\u001b[49m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbroad/xlm-roberta-large-2048\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'CommitInfo' object has no attribute 'push_to_hub'"
+     ]
+    }
+   ],
+   "source": [
+    "tokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-large\").push_to_hub(\"nbroad/xlm-roberta-large-2048\")\n",
+    "tokenizer.model_max_length = 2048\n",
+    "tokenizer.push_to_hub(\"nbroad/xlm-roberta-large-2048\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f64c2a48-0850-4520-a7bc-f8870f570066",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}