{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "!pip install huggingface_hub\n",
    "!pip install transformers\n",
    "!pip install accelerate\n",
    "!pip install sentencepiece\n",
    "!pip install datasets\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "data": {
      "text/plain": "Fetching 7 files:   0%|          | 0/7 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b747b1d796954a0b8e58eaeaf08c27dd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "'/home/michael/.cache/huggingface/hub/models--microsoft--deberta-v3-small/snapshots/a36c739020e01763fe789b4b85e2df55d6180012'"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "model_name = 'microsoft/deberta-v3-small'\n",
    "snapshot_download(model_name)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:41.467075866Z",
     "start_time": "2023-07-21T02:52:40.942594675Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "/home/michael/mambaforge/lib/python3.10/site-packages/transformers/convert_slow_tokenizer.py:454: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification,AutoTokenizer\n",
    "tokz = AutoTokenizer.from_pretrained(model_name)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:42.935172136Z",
     "start_time": "2023-07-21T02:52:41.341048053Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "['▁This', '▁is', '▁a', '▁test', '▁of', '▁the', '▁token', 'izer']"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokz.tokenize('This is a test of the tokenizer')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:42.936703036Z",
     "start_time": "2023-07-21T02:52:42.925853299Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def tok_func(x): return tokz(x[\"text\"])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:43.007261101Z",
     "start_time": "2023-07-21T02:52:42.934508146Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Download the emotion text files from the tweet_eval dataset"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset tweet_eval (/home/michael/.cache/huggingface/datasets/tweet_eval/emotion/1.1.0/12aee5282b8784f3e95459466db4cdf45c6bf49719c25cdb0743d71ed0410343)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset_builder\n",
    "builder = load_dataset_builder('tweet_eval', 'emotion')\n",
    "builder.download_and_prepare()\n",
    "ds = builder.as_dataset(split='train')\n",
    "vds = builder.as_dataset(split='validation')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:44.475452380Z",
     "start_time": "2023-07-21T02:52:42.945039871Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/michael/.cache/huggingface/datasets/tweet_eval/emotion/1.1.0/12aee5282b8784f3e95459466db4cdf45c6bf49719c25cdb0743d71ed0410343/cache-d18e190fc8dabba8.arrow\n"
     ]
    }
   ],
   "source": [
    "from datasets import *\n",
    "\n",
    "def prep_ds(data_source):\n",
    "  data_source = data_source.cast(Features({\"text\": Value(\"string\"), \"label\": Value(\"float32\")}))\n",
    "  return data_source.map(tok_func, batched=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:44.476532635Z",
     "start_time": "2023-07-21T02:52:44.460302418Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/michael/.cache/huggingface/datasets/tweet_eval/emotion/1.1.0/12aee5282b8784f3e95459466db4cdf45c6bf49719c25cdb0743d71ed0410343/cache-41920982e7f44aa8.arrow\n"
     ]
    }
   ],
   "source": [
    "tok_ds = ds.map(tok_func, batched=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:44.542254280Z",
     "start_time": "2023-07-21T02:52:44.469675002Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at /home/michael/.cache/huggingface/datasets/tweet_eval/emotion/1.1.0/12aee5282b8784f3e95459466db4cdf45c6bf49719c25cdb0743d71ed0410343/cache-7873464903b05ba9.arrow and /home/michael/.cache/huggingface/datasets/tweet_eval/emotion/1.1.0/12aee5282b8784f3e95459466db4cdf45c6bf49719c25cdb0743d71ed0410343/cache-5ca9a3af06e40878.arrow\n"
     ]
    }
   ],
   "source": [
    "dds = tok_ds.train_test_split(0.25, seed=42)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:44.640911948Z",
     "start_time": "2023-07-21T02:52:44.495005606Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def corr(x,y): return np.corrcoef(x,y)[0][1]\n",
    "def corr_d(eval_pred): return {'pearson': corr(*eval_pred)}"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:44.676598747Z",
     "start_time": "2023-07-21T02:52:44.503337608Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments,Trainer\n",
    "bs = 128\n",
    "epochs = 3\n",
    "lr = 8e-5\n",
    "\n",
    "args = TrainingArguments('outputs', learning_rate=lr, warmup_ratio=0.1, lr_scheduler_type='cosine', fp16=True,\n",
    "    evaluation_strategy=\"epoch\", per_device_train_batch_size=bs, per_device_eval_batch_size=bs*2,\n",
    "    num_train_epochs=epochs, weight_decay=0.01, report_to=None)\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:46.554518613Z",
     "start_time": "2023-07-21T02:52:44.549850534Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at microsoft/deberta-v3-small were not used when initializing DebertaV2ForSequenceClassification: ['mask_predictions.classifier.bias', 'lm_predictions.lm_head.dense.weight', 'mask_predictions.LayerNorm.bias', 'mask_predictions.dense.bias', 'mask_predictions.classifier.weight', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.bias', 'mask_predictions.dense.weight', 'mask_predictions.LayerNorm.weight']\n",
      "- This IS expected if you are initializing DebertaV2ForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2ForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at microsoft/deberta-v3-small and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight', 'classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "id2label = {0:'anger', 1: 'joy', 2: 'optimism', 3: 'sadness'}\n",
    "label2id = {v: k for k, v in id2label.items()}\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=4, id2label=id2label, label2id=label2id)\n",
    "trainer = Trainer(model, args, train_dataset=dds['train'], eval_dataset=dds['test'],\n",
    "                  tokenizer=tokz, compute_metrics=corr_d)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:52:48.872454377Z",
     "start_time": "2023-07-21T02:52:46.557996403Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/michael/mambaforge/lib/python3.10/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a DebertaV2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n    <div>\n      \n      <progress value='2' max='308' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [  2/308 : < :, Epoch 0.01/4]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 18.00 MiB (GPU 0; 3.81 GiB total capacity; 3.57 GiB already allocated; 4.69 MiB free; 3.66 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mOutOfMemoryError\u001B[0m                          Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[12], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[43mtrainer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/trainer.py:1645\u001B[0m, in \u001B[0;36mTrainer.train\u001B[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001B[0m\n\u001B[1;32m   1640\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodel_wrapped \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodel\n\u001B[1;32m   1642\u001B[0m inner_training_loop \u001B[38;5;241m=\u001B[39m find_executable_batch_size(\n\u001B[1;32m   1643\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_inner_training_loop, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_train_batch_size, args\u001B[38;5;241m.\u001B[39mauto_find_batch_size\n\u001B[1;32m   1644\u001B[0m )\n\u001B[0;32m-> 1645\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43minner_training_loop\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m   1646\u001B[0m \u001B[43m    \u001B[49m\u001B[43margs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1647\u001B[0m \u001B[43m    \u001B[49m\u001B[43mresume_from_checkpoint\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mresume_from_checkpoint\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1648\u001B[0m \u001B[43m    \u001B[49m\u001B[43mtrial\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtrial\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1649\u001B[0m \u001B[43m    \u001B[49m\u001B[43mignore_keys_for_eval\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mignore_keys_for_eval\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1650\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/trainer.py:1938\u001B[0m, in \u001B[0;36mTrainer._inner_training_loop\u001B[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001B[0m\n\u001B[1;32m   1935\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcontrol \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcallback_handler\u001B[38;5;241m.\u001B[39mon_step_begin(args, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstate, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcontrol)\n\u001B[1;32m   1937\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39maccelerator\u001B[38;5;241m.\u001B[39maccumulate(model):\n\u001B[0;32m-> 1938\u001B[0m     tr_loss_step \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtraining_step\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minputs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1940\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\n\u001B[1;32m   1941\u001B[0m     args\u001B[38;5;241m.\u001B[39mlogging_nan_inf_filter\n\u001B[1;32m   1942\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m is_torch_tpu_available()\n\u001B[1;32m   1943\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m (torch\u001B[38;5;241m.\u001B[39misnan(tr_loss_step) \u001B[38;5;129;01mor\u001B[39;00m torch\u001B[38;5;241m.\u001B[39misinf(tr_loss_step))\n\u001B[1;32m   1944\u001B[0m ):\n\u001B[1;32m   1945\u001B[0m     \u001B[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001B[39;00m\n\u001B[1;32m   1946\u001B[0m     tr_loss \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m tr_loss \u001B[38;5;241m/\u001B[39m (\u001B[38;5;241m1\u001B[39m \u001B[38;5;241m+\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstate\u001B[38;5;241m.\u001B[39mglobal_step \u001B[38;5;241m-\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_globalstep_last_logged)\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/trainer.py:2759\u001B[0m, in \u001B[0;36mTrainer.training_step\u001B[0;34m(self, model, inputs)\u001B[0m\n\u001B[1;32m   2756\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m loss_mb\u001B[38;5;241m.\u001B[39mreduce_mean()\u001B[38;5;241m.\u001B[39mdetach()\u001B[38;5;241m.\u001B[39mto(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39margs\u001B[38;5;241m.\u001B[39mdevice)\n\u001B[1;32m   2758\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcompute_loss_context_manager():\n\u001B[0;32m-> 2759\u001B[0m     loss \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcompute_loss\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minputs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   2761\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39margs\u001B[38;5;241m.\u001B[39mn_gpu \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m   2762\u001B[0m     loss \u001B[38;5;241m=\u001B[39m loss\u001B[38;5;241m.\u001B[39mmean()  \u001B[38;5;66;03m# mean() to average on multi-gpu parallel training\u001B[39;00m\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/trainer.py:2784\u001B[0m, in \u001B[0;36mTrainer.compute_loss\u001B[0;34m(self, model, inputs, return_outputs)\u001B[0m\n\u001B[1;32m   2782\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   2783\u001B[0m     labels \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m-> 2784\u001B[0m outputs \u001B[38;5;241m=\u001B[39m \u001B[43mmodel\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43minputs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   2785\u001B[0m \u001B[38;5;66;03m# Save past state if it exists\u001B[39;00m\n\u001B[1;32m   2786\u001B[0m \u001B[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001B[39;00m\n\u001B[1;32m   2787\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39margs\u001B[38;5;241m.\u001B[39mpast_index \u001B[38;5;241m>\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m:\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/accelerate/utils/operations.py:581\u001B[0m, in \u001B[0;36mconvert_outputs_to_fp32.<locals>.forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    580\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mforward\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m--> 581\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mmodel_forward\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/accelerate/utils/operations.py:569\u001B[0m, in \u001B[0;36mConvertOutputsToFp32.__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    568\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m--> 569\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m convert_to_fp32(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmodel_forward\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m)\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/amp/autocast_mode.py:14\u001B[0m, in \u001B[0;36mautocast_decorator.<locals>.decorate_autocast\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m     11\u001B[0m \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(func)\n\u001B[1;32m     12\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mdecorate_autocast\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m     13\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m autocast_instance:\n\u001B[0;32m---> 14\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/models/deberta_v2/modeling_deberta_v2.py:1310\u001B[0m, in \u001B[0;36mDebertaV2ForSequenceClassification.forward\u001B[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001B[0m\n\u001B[1;32m   1302\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124mr\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m   1303\u001B[0m \u001B[38;5;124;03mlabels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):\u001B[39;00m\n\u001B[1;32m   1304\u001B[0m \u001B[38;5;124;03m    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,\u001B[39;00m\n\u001B[1;32m   1305\u001B[0m \u001B[38;5;124;03m    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If\u001B[39;00m\n\u001B[1;32m   1306\u001B[0m \u001B[38;5;124;03m    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).\u001B[39;00m\n\u001B[1;32m   1307\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m   1308\u001B[0m return_dict \u001B[38;5;241m=\u001B[39m return_dict \u001B[38;5;28;01mif\u001B[39;00m return_dict \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mconfig\u001B[38;5;241m.\u001B[39muse_return_dict\n\u001B[0;32m-> 1310\u001B[0m outputs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdeberta\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m   1311\u001B[0m \u001B[43m    \u001B[49m\u001B[43minput_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1312\u001B[0m \u001B[43m    \u001B[49m\u001B[43mtoken_type_ids\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtoken_type_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1313\u001B[0m \u001B[43m    \u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mattention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1314\u001B[0m \u001B[43m    \u001B[49m\u001B[43mposition_ids\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mposition_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1315\u001B[0m \u001B[43m    \u001B[49m\u001B[43minputs_embeds\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minputs_embeds\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1316\u001B[0m \u001B[43m    \u001B[49m\u001B[43moutput_attentions\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43moutput_attentions\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1317\u001B[0m \u001B[43m    \u001B[49m\u001B[43moutput_hidden_states\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43moutput_hidden_states\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1318\u001B[0m \u001B[43m    \u001B[49m\u001B[43mreturn_dict\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreturn_dict\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1319\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1321\u001B[0m encoder_layer \u001B[38;5;241m=\u001B[39m outputs[\u001B[38;5;241m0\u001B[39m]\n\u001B[1;32m   1322\u001B[0m pooled_output \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mpooler(encoder_layer)\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/models/deberta_v2/modeling_deberta_v2.py:1082\u001B[0m, in \u001B[0;36mDebertaV2Model.forward\u001B[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001B[0m\n\u001B[1;32m   1072\u001B[0m     token_type_ids \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mzeros(input_shape, dtype\u001B[38;5;241m=\u001B[39mtorch\u001B[38;5;241m.\u001B[39mlong, device\u001B[38;5;241m=\u001B[39mdevice)\n\u001B[1;32m   1074\u001B[0m embedding_output \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39membeddings(\n\u001B[1;32m   1075\u001B[0m     input_ids\u001B[38;5;241m=\u001B[39minput_ids,\n\u001B[1;32m   1076\u001B[0m     token_type_ids\u001B[38;5;241m=\u001B[39mtoken_type_ids,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1079\u001B[0m     inputs_embeds\u001B[38;5;241m=\u001B[39minputs_embeds,\n\u001B[1;32m   1080\u001B[0m )\n\u001B[0;32m-> 1082\u001B[0m encoder_outputs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mencoder\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m   1083\u001B[0m \u001B[43m    \u001B[49m\u001B[43membedding_output\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1084\u001B[0m \u001B[43m    \u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1085\u001B[0m \u001B[43m    \u001B[49m\u001B[43moutput_hidden_states\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m   1086\u001B[0m \u001B[43m    \u001B[49m\u001B[43moutput_attentions\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43moutput_attentions\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1087\u001B[0m \u001B[43m    \u001B[49m\u001B[43mreturn_dict\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreturn_dict\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1088\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1089\u001B[0m encoded_layers \u001B[38;5;241m=\u001B[39m encoder_outputs[\u001B[38;5;241m1\u001B[39m]\n\u001B[1;32m   1091\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mz_steps \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m:\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/models/deberta_v2/modeling_deberta_v2.py:520\u001B[0m, in \u001B[0;36mDebertaV2Encoder.forward\u001B[0;34m(self, hidden_states, attention_mask, output_hidden_states, output_attentions, query_states, relative_pos, return_dict)\u001B[0m\n\u001B[1;32m    511\u001B[0m     output_states \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mutils\u001B[38;5;241m.\u001B[39mcheckpoint\u001B[38;5;241m.\u001B[39mcheckpoint(\n\u001B[1;32m    512\u001B[0m         create_custom_forward(layer_module),\n\u001B[1;32m    513\u001B[0m         next_kv,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    517\u001B[0m         rel_embeddings,\n\u001B[1;32m    518\u001B[0m     )\n\u001B[1;32m    519\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 520\u001B[0m     output_states \u001B[38;5;241m=\u001B[39m \u001B[43mlayer_module\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    521\u001B[0m \u001B[43m        \u001B[49m\u001B[43mnext_kv\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    522\u001B[0m \u001B[43m        \u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    523\u001B[0m \u001B[43m        \u001B[49m\u001B[43mquery_states\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mquery_states\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    524\u001B[0m \u001B[43m        \u001B[49m\u001B[43mrelative_pos\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrelative_pos\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    525\u001B[0m \u001B[43m        \u001B[49m\u001B[43mrel_embeddings\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrel_embeddings\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    526\u001B[0m \u001B[43m        \u001B[49m\u001B[43moutput_attentions\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43moutput_attentions\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    527\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    529\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m output_attentions:\n\u001B[1;32m    530\u001B[0m     output_states, att_m \u001B[38;5;241m=\u001B[39m output_states\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/models/deberta_v2/modeling_deberta_v2.py:372\u001B[0m, in \u001B[0;36mDebertaV2Layer.forward\u001B[0;34m(self, hidden_states, attention_mask, query_states, relative_pos, rel_embeddings, output_attentions)\u001B[0m\n\u001B[1;32m    370\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m output_attentions:\n\u001B[1;32m    371\u001B[0m     attention_output, att_matrix \u001B[38;5;241m=\u001B[39m attention_output\n\u001B[0;32m--> 372\u001B[0m intermediate_output \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mintermediate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mattention_output\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    373\u001B[0m layer_output \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moutput(intermediate_output, attention_output)\n\u001B[1;32m    374\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m output_attentions:\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/models/deberta_v2/modeling_deberta_v2.py:325\u001B[0m, in \u001B[0;36mDebertaV2Intermediate.forward\u001B[0;34m(self, hidden_states)\u001B[0m\n\u001B[1;32m    323\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mforward\u001B[39m(\u001B[38;5;28mself\u001B[39m, hidden_states: torch\u001B[38;5;241m.\u001B[39mTensor) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m torch\u001B[38;5;241m.\u001B[39mTensor:\n\u001B[1;32m    324\u001B[0m     hidden_states \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdense(hidden_states)\n\u001B[0;32m--> 325\u001B[0m     hidden_states \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mintermediate_act_fn\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhidden_states\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    326\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m hidden_states\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "File \u001B[0;32m~/mambaforge/lib/python3.10/site-packages/transformers/activations.py:78\u001B[0m, in \u001B[0;36mGELUActivation.forward\u001B[0;34m(self, input)\u001B[0m\n\u001B[1;32m     77\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mforward\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;28minput\u001B[39m: Tensor) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tensor:\n\u001B[0;32m---> 78\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mact\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m)\u001B[49m\n",
      "\u001B[0;31mOutOfMemoryError\u001B[0m: CUDA out of memory. Tried to allocate 18.00 MiB (GPU 0; 3.81 GiB total capacity; 3.57 GiB already allocated; 4.69 MiB free; 3.66 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-21T02:54:24.815549633Z",
     "start_time": "2023-07-21T02:52:49.041959305Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "tok_vds = prep_ds(vds)\n",
    "\n",
    "model.to('cuda')\n",
    "predictions = trainer.predict(tok_vds)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-07-21T02:54:24.815103648Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'trainer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[1], line 3\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtransformers\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m pipeline\n\u001B[0;32m----> 3\u001B[0m classifier \u001B[38;5;241m=\u001B[39m pipeline(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msentiment-analysis\u001B[39m\u001B[38;5;124m\"\u001B[39m, model\u001B[38;5;241m=\u001B[39m\u001B[43mtrainer\u001B[49m)\n\u001B[1;32m      5\u001B[0m classifier(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mBrett Favre is an idiot\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "\u001B[0;31mNameError\u001B[0m: name 'trainer' is not defined"
     ]
    }
   ],
   "source": [
    "!pip install xformers\n",
    "\n",
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"text-classification\", model=model.to('cpu'), tokenizer=tokz)\n",
    "\n",
    "classifier('Brett Favre is an idiot')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-23T04:50:39.032580490Z",
     "start_time": "2023-07-23T04:50:35.736802269Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}