first model version

.ipynb_checkpoints/part4-checkpoint.ipynb

@@ -309,10 +309,52 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "06ef92c9",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "repo.git_pull()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3442a913",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "To https://huggingface.co/SebastianS/dummy-model\n",
+      "   91d9c6c..bc13ef6  main -> main\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'https://huggingface.co/SebastianS/dummy-model/commit/bc13ef64436e852b999af0315b661eebf6fd6a42'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "repo.git_add()\n",
+    "repo.git_commit(\"added this file\")\n",
+    "repo.git_push()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f036dfdd",
+   "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],

config.json

@@ -1,1335 +1,27 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Processing data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import torch\n",
-    "from torch.utils.data import DataLoader\n",
-    "from transformers import get_scheduler, TrainingArguments, Trainer, DataCollatorWithPadding, AdamW, AutoTokenizer, AutoModelForSequenceClassification\n",
-    "from datasets import load_dataset\n",
-    "import gc\n",
-    "import numpy as np\n",
-    "from datasets import load_metric\n",
-    "import random\n",
-    "import os\n",
-    "from tqdm.auto import tqdm"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "os.environ['CUDA_LAUNCH_BLOCKING'] = '1'"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# reset GPU memory\n",
-    "gc.collect()\n",
-    "torch.cuda.empty_cache()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'AutoTokenizer' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "\u001b[1;32m<ipython-input-3-f5793421e6ee>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mcheckpoint\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"bert-base-uncased\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtokenizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[1;31mNameError\u001b[0m: name 'AutoTokenizer' is not defined"
-     ]
-    }
-   ],
-   "source": [
-    "checkpoint = \"bert-base-uncased\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    }
-   ],
-   "source": [
-    "checkpoint = \"bert-base-uncased\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sequences = [\n",
-    "  \"I've been waiting for a HuggingFace course my whole life.\",\n",
-    "  \"This course is amazing!\",\n",
-    "]\n",
-    "batch = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
-    "batch[\"labels\"] = torch.tensor([1, 1])\n",
-    "optimizer = AdamW(model.parameters())\n",
-    "loss = model(**batch).loss\n",
-    "loss.backward()\n",
-    "optimizer.step()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reusing dataset glue (C:\\Users\\1seba\\.cache\\huggingface\\datasets\\glue\\mrpc\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
-     ]
-    }
-   ],
-   "source": [
-    "raw_datasets = load_dataset(\"glue\",\"mrpc\")\n",
-    "raw_train_dataset = raw_datasets['train']\n",
-    "# print(raw_train_dataset.features)\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
-    "# # WHY CANT WE PASS THE DIFFERENT SENTENCES TOGETHER\n",
-    "# tokenized_sentences_1 = tokenizer(raw_train_dataset[15]['sentence1'])\n",
-    "# tokenized_sentences_2 = tokenizer(raw_train_dataset[15]['sentence2'])\n",
-    "# print(tokenizer.decode(tokenized_sentences_1.input_ids), tokenizer.decode(tokenized_sentences_2.input_ids))\n",
-    "# inputs = tokenizer(raw_train_dataset[15]['sentence1'], raw_train_dataset[15]['sentence2'])\n",
-    "# print(tokenizer.decode(inputs.input_ids))\n",
-    "inputs = tokenizer(raw_train_dataset['sentence1'], raw_train_dataset['sentence2'], padding=True, truncation=True)\n",
-    "\n",
-    "# tokenized_datasets = raw_datasets.map(tokenize_function, batched=False)\n",
-    "# print(tokenized_datasets['train'].features)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['input_ids', 'token_type_ids', 'attention_mask']"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "list(inputs.keys())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 4/4 [00:01<00:00,  3.69ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00, 16.42ba/s]\n",
-      "100%|██████████| 2/2 [00:00<00:00,  6.22ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "def tokenize_function(example):\n",
-    "    tokenized = tokenizer(example['sentence1'], example['sentence2'], truncation=True)\n",
-    "#     tokenized['input_ids'] = ['CHANGED!' for item in tokenized['input_ids']]\n",
-    "    return tokenized\n",
-    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[50, 59, 47, 67, 59, 50, 62, 32]"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "samples = tokenized_datasets[\"train\"][:8]\n",
-    "samples = {k: v for k, v in samples.items() if k not in [\"idx\", \"sentence1\", \"sentence2\"]}\n",
-    "[len(x) for x in samples[\"input_ids\"]]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'attention_mask': torch.Size([8, 67]),\n",
-       " 'input_ids': torch.Size([8, 67]),\n",
-       " 'token_type_ids': torch.Size([8, 67]),\n",
-       " 'labels': torch.Size([8])}"
-      ]
-     },
-     "execution_count": 37,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "batch = data_collator(samples)\n",
-    "{k: v.shape for k, v in batch.items()}"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Challenge 1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from torch.utils.data import DataLoader"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "samples = tokenized_datasets['test'][:8]\n",
-    "samples = {k: samples[k] for k in list(samples.keys()) if k not in [\"idx\", \"sentence1\", \"sentence2\"]}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "padded_samples = data_collator(samples)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
-    "train_dataloader = DataLoader(tokenized_datasets['test'], batch_size=16, shuffle=True, collate_fn=data_collator)\n",
-    "for batch in train_dataloader:\n",
-    "  print(batch['input_ids'].shape())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Challenge 2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reusing dataset glue (C:\\Users\\1seba\\.cache\\huggingface\\datasets\\glue\\sst2\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
-     ]
-    }
-   ],
-   "source": [
-    "raw_dataset_sst2 = load_dataset(\"glue\",\"sst2\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 68/68 [00:03<00:00, 18.46ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00, 16.67ba/s]\n",
-      "100%|██████████| 2/2 [00:00<00:00, 16.67ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "dataset_to_tokenize = raw_dataset_sst2\n",
-    "def tokenize_dynamic(example):\n",
-    "    dynamic_sentence_list = [x for x in list(example.keys()) if x not in ['label', 'idx']]\n",
-    "    if len(dynamic_sentence_list) == 1:\n",
-    "        return tokenizer(example[dynamic_sentence_list[0]], truncation=True)\n",
-    "    else:\n",
-    "        return tokenizer(example[dynamic_sentence_list[0]], example[dynamic_sentence_list[1]], truncation=True)\n",
-    "tokenized_datasets = dataset_to_tokenize.map(tokenize_dynamic, batched=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "samples = tokenized_datasets['train'][:8]\n",
-    "samples = {k: samples[k] for k in list(samples.keys()) if k not in [\"idx\", \"sentence\", \"sentence1\", \"sentence2\"]}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 74,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "padded_data = data_collator(samples)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Fine-tuning a model with Trainer API"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 33,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reusing dataset glue (C:\\Users\\1seba\\.cache\\huggingface\\datasets\\glue\\mrpc\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
-      "100%|██████████| 4/4 [00:00<00:00,  5.85ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00, 14.49ba/s]\n",
-      "100%|██████████| 2/2 [00:00<00:00,  6.37ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "# set up so far\n",
-    "from datasets import load_dataset\n",
-    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
-    "\n",
-    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
-    "checkpoint = \"bert-base-uncased\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
-    "\n",
-    "def tokenize_function(example):\n",
-    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
-    "\n",
-    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import TrainingArguments\n",
-    "from transformers import AutoModelForSequenceClassification"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 34,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "training_args = TrainingArguments(\"test-trainer\")\n",
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 4/4 [00:00<00:00,  4.14ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00,  9.71ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "train_dataset = tokenized_datasets[\"train\"].filter(percentageOfItems)\n",
-    "validation_dataset = tokenized_datasets[\"validation\"].filter(percentageOfItems)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 42,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    training_args,\n",
-    "    train_dataset=train_dataset,\n",
-    "    eval_dataset=validation_dataset,\n",
-    "    data_collator=data_collator,\n",
-    "    tokenizer=tokenizer,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "  0%|          | 0/132 [01:31<?, ?it/s]\n",
-      "100%|██████████| 132/132 [00:44<00:00,  2.97it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'train_runtime': 44.4012, 'train_samples_per_second': 2.973, 'epoch': 3.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TrainOutput(global_step=132, training_loss=0.4154145789868904, metrics={'train_runtime': 44.4012, 'train_samples_per_second': 2.973, 'epoch': 3.0})"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "trainer.train()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 48,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      " 80%|████████  | 4/5 [00:00<00:00,  9.37it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(37, 2) (37,)\n"
-     ]
-    }
-   ],
-   "source": [
-    "predictions = trainer.predict(validation_dataset)\n",
-    "print(predictions.predictions.shape, predictions.label_ids.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "from datasets import load_metric"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 49,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "preds = np.argmax(predictions.predictions, axis=-1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 51,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'accuracy': 0.8378378378378378, 'f1': 0.8928571428571429}"
-      ]
-     },
-     "execution_count": 51,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "metric = load_metric(\"glue\", \"mrpc\")\n",
-    "metric.compute(predictions=preds, references=predictions.label_ids)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 52,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def compute_metrics(eval_preds):\n",
-    "  metric = load_metric(\"glue\", \"mrpc\")\n",
-    "  logits, labels = eval_preds\n",
-    "  predictions = np.argmax(logits, axis=-1)\n",
-    "  return metric.compute(predictions=predictions, references=labels)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    }
-   ],
-   "source": [
-    "training_args = TrainingArguments(\"test-trainer\", evaluation_strategy=\"epoch\")\n",
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
-    "\n",
-    "trainer = Trainer(\n",
-    "    model,\n",
-    "    training_args,\n",
-    "    train_dataset=train_dataset,\n",
-    "    eval_dataset=validation_dataset,\n",
-    "    data_collator=data_collator,\n",
-    "    tokenizer=tokenizer,\n",
-    "    compute_metrics=compute_metrics\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 66,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "  1%|          | 1/132 [00:19<43:22, 19.87s/it]\n",
-      "100%|██████████| 5/5 [00:00<00:00, 17.23it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.5742557048797607, 'eval_accuracy': 0.7027027027027027, 'eval_f1': 0.8070175438596492, 'eval_runtime': 0.9927, 'eval_samples_per_second': 37.273, 'epoch': 1.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 5/5 [00:00<00:00, 17.03it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.4739842414855957, 'eval_accuracy': 0.7837837837837838, 'eval_f1': 0.8620689655172413, 'eval_runtime': 0.9255, 'eval_samples_per_second': 39.977, 'epoch': 2.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 5/5 [00:00<00:00, 16.95it/s]\n",
-      "                                             \n",
-      "100%|██████████| 132/132 [00:46<00:00,  2.81it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.5759992599487305, 'eval_accuracy': 0.7567567567567568, 'eval_f1': 0.8474576271186441, 'eval_runtime': 0.8269, 'eval_samples_per_second': 44.745, 'epoch': 3.0}\n",
-      "{'train_runtime': 46.927, 'train_samples_per_second': 2.813, 'epoch': 3.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TrainOutput(global_step=132, training_loss=0.39838010614568536, metrics={'train_runtime': 46.927, 'train_samples_per_second': 2.813, 'epoch': 3.0})"
-      ]
-     },
-     "execution_count": 66,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainer.train()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Challenge 3"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 2/2 [00:00<00:00,  7.19ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "# FILTER BREAKS THE LABELS ON THIS DATASET\n",
-    "a = tokenized_datasets['test'].filter(lambda example, index: index % 2 == 0, with_indices=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    }
-   ],
-   "source": [
-    "# use \"tokenized_datasets\" from challenge 2\n",
-    "checkpoint = \"bert-base-uncased\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
-    "training_args = TrainingArguments('test-trainer', evaluation_strategy='epoch')\n",
-    "train_shard = tokenized_datasets['train'].shard(num_shards=150, index=0)\n",
-    "validation_shard = tokenized_datasets['validation'].shard(num_shards=4, index=0)\n",
-    "metric_sst2 = load_metric('glue', 'sst2')\n",
-    "\n",
-    "# def compute_metrics(eval_preds):\n",
-    "#   metric = load_metric(\"glue\", \"mrpc\")\n",
-    "#   logits, labels = eval_preds\n",
-    "#   predictions = np.argmax(logits, axis=-1)\n",
-    "#   return metric.compute(predictions=predictions, references=labels)\n",
-    "def compute_metrics (eval_preds):\n",
-    "  metric_sst2 = load_metric('glue', 'sst2')\n",
-    "  logits, labels = eval_preds\n",
-    "  predictions = np.argmax(logits, axis=-1)\n",
-    "  return metric_sst2.compute(predictions=predictions, references=labels)\n",
-    "\n",
-    "trainer = Trainer(\n",
-    "  model,\n",
-    "  training_args,\n",
-    "  train_dataset=train_shard,\n",
-    "  eval_dataset=validation_shard,\n",
-    "  data_collator=data_collator,\n",
-    "  tokenizer=tokenizer,\n",
-    "  compute_metrics=compute_metrics\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      " 33%|███▎      | 57/171 [00:35<00:58,  1.94it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.38222888112068176, 'eval_accuracy': 0.8302752293577982, 'eval_runtime': 3.3093, 'eval_samples_per_second': 65.875, 'epoch': 1.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      " 67%|██████▋   | 114/171 [01:09<00:29,  1.93it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.7558169364929199, 'eval_accuracy': 0.8165137614678899, 'eval_runtime': 3.5593, 'eval_samples_per_second': 61.248, 'epoch': 2.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "100%|██████████| 171/171 [01:42<00:00,  1.66it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'eval_loss': 0.5612818598747253, 'eval_accuracy': 0.8669724770642202, 'eval_runtime': 3.3543, 'eval_samples_per_second': 64.991, 'epoch': 3.0}\n",
-      "{'train_runtime': 102.7742, 'train_samples_per_second': 1.664, 'epoch': 3.0}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TrainOutput(global_step=171, training_loss=0.276075485854121, metrics={'train_runtime': 102.7742, 'train_samples_per_second': 1.664, 'epoch': 3.0})"
-      ]
-     },
-     "execution_count": 22,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainer.train()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# A Full Training"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reusing dataset glue (C:\\Users\\1seba\\.cache\\huggingface\\datasets\\glue\\mrpc\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
-      "100%|██████████| 4/4 [00:00<00:00,  7.09ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00, 16.39ba/s]\n",
-      "100%|██████████| 2/2 [00:00<00:00,  9.01ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "# setup\n",
-    "from datasets import load_dataset\n",
-    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
-    "\n",
-    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
-    "checkpoint = \"bert-base-uncased\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
-    "def tokenize_function(example):\n",
-    "  return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
-    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['attention_mask', 'input_ids', 'labels', 'token_type_ids']"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "tokenized_datasets = tokenized_datasets.remove_columns([\"idx\", \"sentence1\", \"sentence2\"])\n",
-    "tokenized_datasets = tokenized_datasets.rename_column('label', 'labels')\n",
-    "tokenized_datasets.set_format('torch')\n",
-    "tokenized_datasets['train'].column_names"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from torch.utils.data import DataLoader\n",
-    "train_dataloader = DataLoader(\n",
-    "  tokenized_datasets['train'].shard(num_shards=15, index=0), shuffle=True, batch_size=8, collate_fn=data_collator\n",
-    ")\n",
-    "eval_dataloader = DataLoader(\n",
-    "  tokenized_datasets['validation'].shard(num_shards=5, index=0), batch_size=8, collate_fn=data_collator\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 60,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'attention_mask': torch.Size([8, 64]),\n",
-       " 'input_ids': torch.Size([8, 64]),\n",
-       " 'labels': torch.Size([8]),\n",
-       " 'token_type_ids': torch.Size([8, 64])}"
-      ]
-     },
-     "execution_count": 60,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "for batch in train_dataloader:\n",
-    "  break\n",
-    "{k: v.shape for k, v in batch.items()}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    }
-   ],
-   "source": [
-    "from transformers import AutoModelForSequenceClassification\n",
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "tensor(0.5705, grad_fn=<NllLossBackward>) torch.Size([8, 2])\n"
-     ]
-    }
-   ],
-   "source": [
-    "outputs = model(**batch)\n",
-    "print(outputs.loss, outputs.logits.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 63,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import AdamW\n",
-    "optimizer = AdamW(model.parameters(), lr=5e-5)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 64,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "93\n"
-     ]
-    }
-   ],
-   "source": [
-    "from transformers import get_scheduler\n",
-    "num_epochs = 3\n",
-    "num_training_steps = num_epochs * len(train_dataloader)\n",
-    "lr_scheduler = get_scheduler(\n",
-    "  'linear',\n",
-    "  optimizer,\n",
-    "  num_warmup_steps=0,\n",
-    "  num_training_steps=num_training_steps,\n",
-    ")\n",
-    "print(num_training_steps)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 65,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "device(type='cuda')"
-      ]
-     },
-     "execution_count": 65,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import torch\n",
-    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
-    "model.to(device)\n",
-    "device"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 71,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 93/93 [08:50<00:00,  5.70s/it]\n",
-      "100%|██████████| 93/93 [00:28<00:00,  3.21it/s]"
-     ]
-    }
-   ],
-   "source": [
-    "from tqdm.auto import tqdm\n",
-    "progress_bar = tqdm(range(num_training_steps))\n",
-    "model.train()\n",
-    "for epoch in range(num_epochs):\n",
-    "  for batch in train_dataloader:\n",
-    "    batch = {k: v.to(device) for k, v in batch.items()}\n",
-    "    outputs = model(**batch)\n",
-    "    loss = outputs.loss\n",
-    "    loss.backward()\n",
-    "    optimizer.step()\n",
-    "    optimizer.zero_grad()\n",
-    "    progress_bar.update(1)\n",
-    "  \n",
-    "  # metric = load_metric('glue', 'mrpc')\n",
-    "  # model.eval()\n",
-    "  # for batch in eval_dataloader:\n",
-    "  #   batch = {k: v.to(device) for k, v in batch.items()}\n",
-    "  #   with torch.no_grad():\n",
-    "  #     outputs = model(**batch)\n",
-    "  #   logits = outputs.logits\n",
-    "  #   predictions = torch.argmax(logits, dim=-1)\n",
-    "  #   metric.add_batch(predictions=predictions, references=batch['labels'])\n",
-    "  # print(metric.compute())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 109,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'accuracy': 0.6463414634146342, 'f1': 0.7851851851851851}"
-      ]
-     },
-     "execution_count": 109,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from datasets import load_metric\n",
-    "metric = load_metric('glue', 'mrpc')\n",
-    "model.eval()\n",
-    "for batch in eval_dataloader:\n",
-    "  batch = {k: v.to(device) for k, v in batch.items()}\n",
-    "  with torch.no_grad():\n",
-    "    outputs = model(**batch)\n",
-    "  logits = outputs.logits\n",
-    "  predictions = torch.argmax(logits, dim=-1)\n",
-    "  metric.add_batch(predictions=predictions, references=batch['labels'])\n",
-    "metric.compute()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Challenge 1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Reusing dataset glue (C:\\Users\\1seba\\.cache\\huggingface\\datasets\\glue\\sst2\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
-      "100%|██████████| 68/68 [00:03<00:00, 20.33ba/s]\n",
-      "100%|██████████| 1/1 [00:00<00:00, 17.24ba/s]\n",
-      "100%|██████████| 2/2 [00:00<00:00, 16.53ba/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
-    "\n",
-    "sst2_datasets = load_dataset(\"glue\", \"sst2\")\n",
-    "def tokenize_function (example):\n",
-    "  return tokenizer(example['sentence'], truncation=True)\n",
-    "tokenized_datasets = sst2_datasets.map(tokenize_function, batched=True)\n",
-    "tokenized_datasets = tokenized_datasets.remove_columns([\"idx\", \"sentence\"])\n",
-    "tokenized_datasets = tokenized_datasets.rename_column('label', 'labels')\n",
-    "tokenized_datasets.set_format('torch')\n",
-    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
-    "train_dataset = DataLoader(\n",
-    "  tokenized_datasets['train'].shard(num_shards=180, index=0), shuffle=True, batch_size=8, collate_fn=data_collator\n",
-    ")\n",
-    "eval_dataset = DataLoader(\n",
-    "  tokenized_datasets['validation'].shard(num_shards=4, index=0), batch_size=8, collate_fn=data_collator\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 31,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
-      "100%|██████████| 141/141 [18:15<00:00,  7.77s/it]\n",
-      "100%|██████████| 141/141 [01:12<00:00,  2.21it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[{'accuracy': 0.7568807339449541}, {'accuracy': 0.8256880733944955}, {'accuracy': 0.8623853211009175}]\n"
-     ]
-    }
-   ],
-   "source": [
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
-    "model.to(device)\n",
-    "optimizer= AdamW(model.parameters(), 5e-5)\n",
-    "\n",
-    "num_epochs = 3\n",
-    "num_training_steps = num_epochs * len(train_dataset)\n",
-    "lr_scheduler = get_scheduler(\n",
-    "  'linear',\n",
-    "  optimizer=optimizer,\n",
-    "  num_warmup_steps=0,\n",
-    "  num_training_steps=num_training_steps,\n",
-    ")\n",
-    "\n",
-    "metrics = []\n",
-    "\n",
-    "progress_bar = tqdm(range(num_training_steps))\n",
-    "model.train()\n",
-    "for epoch in range(num_epochs):\n",
-    "  for batch in train_dataset:\n",
-    "    batch = {k: v.to(device) for k, v in batch.items()}\n",
-    "    outputs = model(**batch)\n",
-    "    loss = outputs.loss\n",
-    "    loss.backward()\n",
-    "    optimizer.step()\n",
-    "    lr_scheduler.step()\n",
-    "    optimizer.zero_grad()\n",
-    "    progress_bar.update(1)\n",
-    "\n",
-    "  metric= load_metric(\"glue\", \"sst2\")\n",
-    "  model.eval()\n",
-    "  for batch in eval_dataset:\n",
-    "    batch = {k: v.to(device) for k, v in batch.items()}\n",
-    "    with torch.no_grad():\n",
-    "        outputs = model(**batch)\n",
-    "    logits = outputs.logits\n",
-    "    predictions = torch.argmax(logits, dim=-1)\n",
-    "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
-    "  metrics.append(metric.compute())\n",
-    "\n",
-    "print(metrics)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## (end)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from accelerate import Accelerator\n",
-    "accelerator = Accelerator()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
-      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
-      "100%|██████████| 93/93 [01:11<00:00,  1.85it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[{'accuracy': 0.6707317073170732}, {'accuracy': 0.7073170731707317}, {'accuracy': 0.7560975609756098}]\n"
-     ]
-    }
-   ],
-   "source": [
-    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
-    "optimizer= AdamW(model.parameters(), 5e-5)\n",
-    "train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n",
-    "  train_dataloader, eval_dataloader, model, optimizer\n",
-    ")\n",
-    "\n",
-    "num_epochs = 3\n",
-    "num_training_steps = num_epochs * len(train_dataloader)\n",
-    "lr_scheduler = get_scheduler(\n",
-    "  'linear',\n",
-    "  optimizer=optimizer,\n",
-    "  num_warmup_steps=0,\n",
-    "  num_training_steps=num_training_steps,\n",
-    ")\n",
-    "\n",
-    "metrics = []\n",
-    "\n",
-    "progress_bar = tqdm(range(num_training_steps))\n",
-    "model.train()\n",
-    "for epoch in range(num_epochs):\n",
-    "  for batch in train_dataloader:\n",
-    "    outputs = model(**batch)\n",
-    "    loss = outputs.loss\n",
-    "    accelerator.backward(loss)\n",
-    "    optimizer.step()\n",
-    "    lr_scheduler.step()\n",
-    "    optimizer.zero_grad()\n",
-    "    progress_bar.update(1)\n",
-    "\n",
-    "  metric= load_metric(\"glue\", \"sst2\")\n",
-    "  model.eval()\n",
-    "  for batch in eval_dataloader:\n",
-    "    with torch.no_grad():\n",
-    "        outputs = model(**batch)\n",
-    "    logits = outputs.logits\n",
-    "    predictions = torch.argmax(logits, dim=-1)\n",
-    "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
-    "  metrics.append(metric.compute())\n",
-    "\n",
-    "print(metrics)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "interpreter": {
-   "hash": "c23364dc34acf6d559b2ccbb804894040b11f1b7cd300b891de29d32dea3c2c2"
-  },
-  "kernelspec": {
-   "display_name": "Python 3.8.10 64-bit ('AI': conda)",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.10"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
+  "_name_or_path": "camembert-base",
+  "architectures": [
+    "CamembertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 5,
+  "eos_token_id": 6,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "camembert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 32005
 }

hello.txt

@@ -0,0 +1 @@
+Hello

part4.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 1,
    "id": "aa7a358a",
    "metadata": {},
    "outputs": [],
@@ -12,100 +12,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 2,
    "id": "c7e39f7f",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f07d3dc0c67842c5905d2a8d9bbc0ee8",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=810912.0, style=ProgressStyle(descripti…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "bafae4f91f7e490087300d6fcd12ad15",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1395301.0, style=ProgressStyle(descript…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "55b5654c906441d3bba3d48c72a373f2",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=508.0, style=ProgressStyle(description_…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "659a9e30adb94f24bd78d87fb4f7706d",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=445032417.0, style=ProgressStyle(descri…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
     "model = CamembertForMaskedLM.from_pretrained(\"camembert-base\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "30ca41f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.save_pretrained(\"./\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -317,10 +242,42 @@
     "repo.git_pull()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3442a913",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "To https://huggingface.co/SebastianS/dummy-model\n",
+      "   91d9c6c..bc13ef6  main -> main\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'https://huggingface.co/SebastianS/dummy-model/commit/bc13ef64436e852b999af0315b661eebf6fd6a42'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "repo.git_add()\n",
+    "repo.git_commit(\"added this file\")\n",
+    "repo.git_push()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "574529d9",
+   "id": "f036dfdd",
    "metadata": {},
    "outputs": [],
    "source": []

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e3c7b2ac552b676f86d627ac840b9138181091c281d047caa0fa638c5e562a
+size 442709831