bigram/bigram_testing.sync.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "864d6f55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)\n",
    "block_size = 8\n",
    "batch_size = 4\n",
    "max_iters = 100000\n",
    "learning_rate = 3e-4\n",
    "eval_every = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b056ad8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THE SONNETS\n",
      "\n",
      "                    1\n",
      "\n",
      "From fairest creatures we desire increase,\n",
      "That thereby beauty’s rose might never die,\n",
      "But as the riper should by time decease,\n",
      "His tender heir might bear his memory:\n",
      "But thou contracted to thine own bright eyes,\n",
      "Feed’st thy light’s flame with self-substantial fuel,\n",
      "Making a famine where abundance lies,\n",
      "Thyself thy foe, to thy sweet self too cruel:\n",
      "Thou that art now the world’s fresh ornament,\n",
      "And only herald to the gaudy spring,\n",
      "Within thine own bud buriest \n"
     ]
    }
   ],
   "source": [
    "with open(\"shakespeare.txt\") as f:\n",
    "    text = f.read()\n",
    "print(text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b539c516",
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(set(text))\n",
    "vocab_size = len(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1a6ecfab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 101\n",
      "Text length: 5357910\n"
     ]
    }
   ],
   "source": [
    "print(f\"Vocab size: {vocab_size}\")\n",
    "print(f\"Text length: {len(text)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4d12c432",
   "metadata": {},
   "outputs": [],
   "source": [
    "string_to_int = {ch: i for i, ch in enumerate(chars)}\n",
    "int_to_string = {i: ch for i, ch in enumerate(chars)}\n",
    "\n",
    "encode = lambda s: [string_to_int[ch] for ch in s]\n",
    "decode = lambda x: ''.join([int_to_string[i] for i in x])\n",
    "\n",
    "data = torch.tensor(encode(text), dtype=torch.long, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2ee10568",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = int(0.8 * len(data))\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f8d9963a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(split):\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    x, y = x.to(device), y.to(device)\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "31a09e9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = get_batch('train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "65b12427",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class BigramLanguageModel(nn.Module):\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "\n",
    "    def forward(self, index, targets=None):\n",
    "        logits = self.token_embedding_table(index)\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C) # reshape to what torch.cross_entropy expects\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets) \n",
    "        return logits, loss\n",
    "    def generate(self, index, max_new_tokens):\n",
    "        # index is (B, T) array of indices in the current context\n",
    "        for _ in range(max_new_tokens):\n",
    "            # get the predictions\n",
    "            logits, loss = self.forward(index)\n",
    "            # focus only on the last time step\n",
    "            logits = logits[:, -1, :] # becomes (B, C)\n",
    "            # apply softmax to get probabilities\n",
    "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
    "            # sample from the distribution\n",
    "            index_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "            # append sampled index to the running sequence\n",
    "            index = torch.cat((index, index_next), dim=1) # (B, T+1)\n",
    "        return index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "38c1fd2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\txSV……VF'3aFë-,7Xy*KV4RW6S7qêë]Kp:v—ç2G'…F*R—P612O‘dÀ5I7d:hMuaPB,AHAnSp3DhÇ8Æm?‘” -9IWÉi b,p925Lp2)z\n"
     ]
    }
   ],
   "source": [
    "model = BigramLanguageModel(vocab_size).to(device)\n",
    "\n",
    "context = torch.zeros((1,1), dtype=torch.long, device=device)\n",
    "generated_chars = decode(model.generate(context, max_new_tokens=100)[0].tolist())\n",
    "print(generated_chars)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a168e17",
   "metadata": {},
   "source": [
    "\n",
    "### Some common optimizers\n",
    "1. **Mean Squared Error (MSE)**: MSE is a common loss function used in regression problems, where the goal is to predict a continuous output. It measures the average squared difference between the predicted and actual values, and is often used to train neural networks for regression tasks.\n",
    "2. **Gradient Descent (GD):**  is an optimization algorithm used to minimize the loss function of a machine learning model. The loss function measures how well the model is able to predict the target variable based on the input features. The idea of GD is to iteratively adjust the model parameters in the direction of the steepest descent of the loss function\n",
    "3. **Momentum**: Momentum is an extension of SGD that adds a \\\"momentum\\\" term to the parameter updates. This term helps smooth out the updates and allows the optimizer to continue moving in the right direction, even if the gradient changes direction or varies in magnitude. Momentum is particularly useful for training deep neural networks.\n",
    "4. **RMSprop**: RMSprop is an optimization algorithm that uses a moving average of the squared gradient to adapt the learning rate of each parameter. This helps to avoid oscillations in the parameter updates and can improve convergence in some cases.\n",
    "5. **Adam**: Adam is a popular optimization algorithm that combines the ideas of momentum and RMSprop. It uses a moving average of both the gradient and its squared value to adapt the learning rate of each parameter. Adam is often used as a default optimizer for deep learning models.\n",
    "6. **AdamW**: AdamW is a modification of the Adam optimizer that adds weight decay to the parameter updates. This helps to regularize the model and can improve generalization performance. We will be using the AdamW optimizer as it best suits the properties of the model we will train in this video.\n",
    "\n",
    "find more optimizers and details at torch.optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "547c740d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 0:\n",
      "5.172138690948486\n",
      "Iter 5000:\n",
      "4.190489768981934\n",
      "Iter 10000:\n",
      "3.747365713119507\n",
      "Iter 15000:\n",
      "3.188748359680176\n",
      "Iter 20000:\n",
      "2.9176113605499268\n",
      "Iter 25000:\n",
      "2.900815963745117\n",
      "Iter 30000:\n",
      "2.2446277141571045\n",
      "Iter 35000:\n",
      "2.5964982509613037\n",
      "Iter 40000:\n",
      "2.225098133087158\n",
      "Iter 45000:\n",
      "2.517181158065796\n",
      "Iter 50000:\n",
      "2.586806535720825\n",
      "Iter 55000:\n",
      "2.4346859455108643\n",
      "Iter 60000:\n",
      "2.952392816543579\n",
      "Iter 65000:\n",
      "2.6527769565582275\n",
      "Iter 70000:\n",
      "2.369570016860962\n",
      "Iter 75000:\n",
      "2.434110641479492\n",
      "Iter 80000:\n",
      "2.4495511054992676\n",
      "Iter 85000:\n",
      "2.4248077869415283\n",
      "Iter 90000:\n",
      "2.6163671016693115\n",
      "Iter 95000:\n",
      "2.5076956748962402\n",
      "2.6161372661590576\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for iter in range(max_iters):\n",
    "    # sample a batch\n",
    "    xb, yb = get_batch('train')\n",
    "\n",
    "    # evaluate the loss\n",
    "    logits, loss = model.forward(xb, yb)\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if iter % eval_every == 0:\n",
    "        print(f\"Iter {iter}:\")\n",
    "        print(loss.item())\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "64e938b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\txsunt bly? fl ed s.\n",
      "Mulyowave br, chud nllyon wdiman nd st chapeed,\n",
      "Anoouscelllend mor yo hesspethe \n"
     ]
    }
   ],
   "source": [
    "\n",
    "context = torch.zeros((1,1), dtype=torch.long, device=device)\n",
    "generated_chars = decode(model.generate(context, max_new_tokens=100)[0].tolist())\n",
    "print(generated_chars)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}