{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "execution": { "iopub.execute_input": "2023-03-24T15:35:18.881502Z", "iopub.status.busy": "2023-03-24T15:35:18.880789Z", "iopub.status.idle": "2023-03-24T15:35:22.115494Z", "shell.execute_reply": "2023-03-24T15:35:22.114665Z", "shell.execute_reply.started": "2023-03-24T15:35:18.881474Z" }, "id": "Ef7vm_ycPCxw", "outputId": "95289963-04fd-4911-aece-c4ea17b40cca" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.21.3)\n", "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.4.0)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (1.12.1+cu116)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.12.0)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.28.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.9.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (5.4.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.23.4)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.64.1)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.12.1)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.13)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.0)\n", "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.1.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.3)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n", "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n", "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (10.0.1)\n", "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.5.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (18.2.0)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2019.11.28)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (2.8)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.14)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.14.0)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m" ] } ], "source": [ "!pip install transformers datasets torch" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "execution": { "iopub.execute_input": "2023-03-24T15:39:46.744379Z", "iopub.status.busy": "2023-03-24T15:39:46.744112Z", "iopub.status.idle": "2023-03-24T15:49:07.678229Z", "shell.execute_reply": "2023-03-24T15:49:07.677294Z", "shell.execute_reply.started": "2023-03-24T15:39:46.744360Z" }, "id": "Kss9x-rwPFTS", "outputId": "6297d81a-6053-4cb3-b5e2-cd694f95b0e6" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running on cuda\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Using cuda_amp half precision backend\n", "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n", "***** Running training *****\n", " Num examples = 43\n", " Num Epochs = 3\n", " Instantaneous batch size per device = 4\n", " Total train batch size (w. parallel, distributed & accumulation) = 4\n", " Gradient Accumulation steps = 1\n", " Total optimization steps = 33\n", "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n", "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:" ] }, { "name": "stdin", "output_type": "stream", "text": [ " ········································\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n" ] }, { "data": { "text/html": [ "wandb version 0.14.0 is available! To upgrade, please run:\n", " $ pip install wandb --upgrade" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "Tracking run with wandb version 0.13.4" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "Run data is saved locally in /notebooks/wandb/run-20230324_154130-vctszt1m" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "Syncing run ./output to Weights & Biases (docs)
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [33/33 07:17, Epoch 3/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
1No log1.520941
2No log1.267309
3No log1.192186

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "***** Running Evaluation *****\n", " Num examples = 8\n", " Batch size = 4\n", "Saving model checkpoint to ./output/checkpoint-11\n", "Configuration saved in ./output/checkpoint-11/config.json\n", "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-11/pytorch_model.bin.index.json.\n", "***** Running Evaluation *****\n", " Num examples = 8\n", " Batch size = 4\n", "Saving model checkpoint to ./output/checkpoint-22\n", "Configuration saved in ./output/checkpoint-22/config.json\n", "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-22/pytorch_model.bin.index.json.\n", "***** Running Evaluation *****\n", " Num examples = 8\n", " Batch size = 4\n", "Saving model checkpoint to ./output/checkpoint-33\n", "Configuration saved in ./output/checkpoint-33/config.json\n", "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-33/pytorch_model.bin.index.json.\n", "\n", "\n", "Training completed. Do not forget to share your model on huggingface.co/models =)\n", "\n", "\n", "Configuration saved in ./model/config.json\n", "tokenizer config file saved in ./model/tokenizer_config.json\n", "Special tokens file saved in ./model/special_tokens_map.json\n", "Saving model checkpoint to ./model\n", "Configuration saved in ./model/config.json\n", "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./model/pytorch_model.bin.index.json.\n" ] } ], "source": [ "import torch\n", "from transformers import GPTNeoForCausalLM, GPT2Tokenizer, TextDataset, DataCollatorForLanguageModeling\n", "from transformers import Trainer, TrainingArguments\n", "import os\n", "\n", "def main():\n", " model_name = 'EleutherAI/gpt-neo-2.7B'\n", " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " print(f\"Running on {device}\")\n", "\n", " model_dir = './model'\n", " if not os.path.exists(model_dir):\n", " os.makedirs(model_dir)\n", " tokenizer = GPT2Tokenizer.from_pretrained(model_name)\n", " model = GPTNeoForCausalLM.from_pretrained(model_name).to(device)\n", " model.config.save_pretrained(model_dir)\n", " tokenizer.save_pretrained(model_dir)\n", " else:\n", " tokenizer = GPT2Tokenizer.from_pretrained(model_dir)\n", " model_config = GPTNeoConfig.from_pretrained(model_dir)\n", " model = GPTNeoForCausalLM.from_pretrained(model_dir, config=model_config).to(device)\n", "\n", " train_file = \"/notebooks/train.txt\"\n", " validation_file = \"/notebooks/valid.txt\"\n", "\n", " train_dataset = TextDataset(\n", " tokenizer=tokenizer,\n", " file_path=train_file,\n", " block_size=128,\n", " )\n", "\n", " validation_dataset = TextDataset(\n", " tokenizer=tokenizer,\n", " file_path=validation_file,\n", " block_size=128,\n", " )\n", "\n", " data_collator = DataCollatorForLanguageModeling(\n", " tokenizer=tokenizer, mlm=False,\n", " )\n", "\n", " training_args = TrainingArguments(\n", " output_dir=\"./output\",\n", " overwrite_output_dir=True,\n", " num_train_epochs=3,\n", " per_device_train_batch_size=4,\n", " per_device_eval_batch_size=4,\n", " evaluation_strategy=\"epoch\",\n", " save_strategy=\"epoch\",\n", " logging_dir=\"./logs\",\n", " logging_strategy=\"steps\",\n", " logging_steps=100,\n", " learning_rate=5e-6,\n", " fp16=True,\n", " )\n", "\n", " trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " data_collator=data_collator,\n", " train_dataset=train_dataset,\n", " eval_dataset=validation_dataset,\n", " )\n", "\n", " trainer.train()\n", "\n", " # Save the trained model weights, config, and tokenizer\n", " model.config.save_pretrained(model_dir)\n", " tokenizer.save_pretrained(model_dir)\n", " trainer.save_model(model_dir)\n", " \n", " \n", "if __name__ == \"__main__\":\n", " main()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2023-03-24T15:57:40.157628Z", "iopub.status.busy": "2023-03-24T15:57:40.156950Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n" ] } ], "source": [ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "import logging\n", "\n", "logging.getLogger(\"transformers\").setLevel(logging.WARNING)\n", "\n", "def generate_text(model, tokenizer, prompt, max_length=256, temperature=0.7):\n", " input_ids = tokenizer.encode(prompt, return_tensors='pt')\n", " input_ids = input_ids.to(model.device)\n", " batch_size = 64\n", " num_return_sequences = 5\n", " output = model.generate(\n", " input_ids=input_ids.repeat(batch_size, 1),\n", " max_length=max_length,\n", " temperature=temperature,\n", " do_sample=True,\n", " top_k=50,\n", " top_p=0.95,\n", " num_return_sequences=num_return_sequences,\n", " )\n", " generated_text = tokenizer.batch_decode(output, skip_special_tokens=True)\n", " return generated_text\n", "\n", "\n", "model_dir = './model'\n", "tokenizer = GPT2Tokenizer.from_pretrained(model_dir)\n", "model = GPTNeoForCausalLM.from_pretrained(model_dir).to(device)\n", "\n", "prompt = \"\"\"Human: What do you think of black people?\n", "Janny:\n", "\"\"\"\n", "\n", "generated_text = generate_text(model, tokenizer, prompt)\n", "print(generated_text)\n" ] } ], "metadata": { "accelerator": "GPU", "colab": { "provenance": [] }, "gpuClass": "standard", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.16" } }, "nbformat": 4, "nbformat_minor": 4 }