code/cs224N_pytorch_tutorial_nlp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "\n",
    "import pprint \n",
    "pp = pprint.PrettyPrinter()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tensors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 2, 3], [4, 5, 6]]\n"
     ]
    }
   ],
   "source": [
    "list_of_lists = [[1 ,2, 3], [4,5,6]]\n",
    "print(list_of_lists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1],\n",
       "        [2, 3],\n",
       "        [4, 5]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#init a tensor\n",
    "data = torch.tensor([\n",
    "    [0,1],\n",
    "    [2,3],\n",
    "    [4,5]\n",
    "])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 1.],\n",
       "        [2., 3.],\n",
       "        [4., 5.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can cast them to different types -- usualloy float32 or int\n",
    "data = torch.tensor([\n",
    "    [0,1],\n",
    "    [2,3],\n",
    "    [4,5],\n",
    "] ,   dtype=torch.float32)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 2])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = torch.tensor([\n",
    "    [1, 2, 3],\n",
    "    [4, 5, 6]\n",
    "], dtype=torch.float32)\n",
    "\n",
    "# must be same type \n",
    "data3 = data @ data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rr = torch.arange(1, 10)\n",
    "rr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15])\n",
      "tensor([[ 1,  2,  3],\n",
      "        [ 4,  5,  6],\n",
      "        [ 7,  8,  9],\n",
      "        [10, 11, 12],\n",
      "        [13, 14, 15]])\n",
      "torch.Size([5, 3])\n"
     ]
    }
   ],
   "source": [
    "# can reshape a vector into a matrix!  \n",
    "# evidently this can make batch operations easier \n",
    "# e.x. take a 15 element row vector and turn it into a 5 x 3 matrix \n",
    "vec = torch.arange(1,16)\n",
    "print(vec)\n",
    "matrix = vec.view(5,3)\n",
    "print(matrix)\n",
    "print(matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one of the reason we use tensors is vectorized operations: \n",
    "# operations that be conducted in parallel over a paarticular dimension of a tensor \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([42., 57., 72.])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# things we can do like sum, mean, and std dev \n",
    "\n",
    "# https://stackoverflow.com/questions/55691819/why-does-dim-1-return-row-indices-in-torch-argmax\n",
    "\n",
    "# dim=1 is ROWS \n",
    "data3.mean(dim=1)\n",
    "data3.sum(dim=1)\n",
    "\n",
    "# dim=0 is COLUMNS\n",
    "data3.mean(dim=0)\n",
    "data3.sum(dim=0)\n",
    "\n",
    "# look at the official pytorch tutorial for a good example of all the way you can slice a tensor\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Autograd \n",
    "\n",
    "Pytorch is known for it's auto differentation feature.. We can call the `backward()` method to ask Pytorch to calculate the gradients which are then stored in the `grad` attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([2], dtype=torch.float32, requires_grad=True)\n",
    "\n",
    "pp.pprint(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.Tensor'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([84.])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = 3 * x * x \n",
    "# z = 3 * x * x\n",
    "print(type(y))\n",
    "y.backward()\n",
    "# if we just want to manually reset the grad instead of using an optimizer \n",
    "# we can just do it manually like this... \n",
    "# x.grad = torch.tensor([0.])\n",
    "# x.zero_grad()\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Module\n",
    "\n",
    "### Linear Layer \n",
    "\n",
    "we can use `nn.Linear(H_in, H_out)` to create a linear layer. This will take a matrix of (N, *, H_in) dimensions and ouptput a matrix of (N, *, H_out). the `*` denotes that there could be arbitrary number of dimensions in between. The linear layer performs the operation `Ax+b`., whewr e`A` and `b` are intialized randomly. If we don't wnat the linear layer to learn the bias parameters, we can intialize our layer with `bias=False`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1580, -0.3979],\n",
       "         [ 0.1580, -0.3979],\n",
       "         [ 0.1580, -0.3979]],\n",
       "\n",
       "        [[ 0.1580, -0.3979],\n",
       "         [ 0.1580, -0.3979],\n",
       "         [ 0.1580, -0.3979]]], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "# create the inputs \n",
    "# two, 3x4 matrices of ones. \n",
    "# 3 rows by 4 columns \n",
    "inputs = torch.ones(2,3,4)\n",
    "# print(inputs)\n",
    "\n",
    "# make a linear layers transforming N,*,H_in dimensional inputs to N,*,H_out dimensional outputs \n",
    "linear = nn.Linear(4,2)\n",
    "linear_output = linear(inputs)\n",
    "linear_output.shape\n",
    "linear_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get different values in the tensor each time we run it because `Linear()` randomly initializes the function `Ax + b` to different values each time!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([[-0.2625, -0.1494,  0.0883,  0.1954],\n",
       "         [ 0.0474,  0.1861,  0.1699, -0.3701]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([ 0.2862, -0.4312], requires_grad=True)]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(linear.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation Function Layer\n",
    "\n",
    "we can also use `nn` module to apply activations function to our tensors. Activation functions are used to add non-linearity to our network. Some examples of activations functions are `nn.ReLU()`, `nn.Sigmoid()` and `nn.LeakyReLU()`. Activation functions operate on each elementate seperatel, so sthe sahpe of the tensorwe get as an output ar ethe same as the ones we pass in."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.5394, 0.4018],\n",
       "         [0.5394, 0.4018],\n",
       "         [0.5394, 0.4018]],\n",
       "\n",
       "        [[0.5394, 0.4018],\n",
       "         [0.5394, 0.4018],\n",
       "         [0.5394, 0.4018]]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid = nn.Sigmoid()\n",
    "\n",
    "output = sigmoid(linear_output)\n",
    "output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Putting the Layers Together\n",
    "\n",
    "So far we have just created lyaers and pass the output of one as the input of the next. Instead of creating intermediate tensors and passing them around, we can use `nn.Sequeuntial` which puts layers together sequentially"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "block = nn.Sequential(\n",
    "    nn.Linear(4,2),\n",
    "    nn.Linear(2,1),\n",
    "    nn.Sigmoid(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.4056],\n",
       "         [0.4056],\n",
       "         [0.4056]],\n",
       "\n",
       "        [[0.4056],\n",
       "         [0.4056],\n",
       "         [0.4056]]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = torch.ones(2,3,4)\n",
    "output = block(input)\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Modules \n",
    "\n",
    "Instead of using the predefined modules, we can also build our own by extending the `nn.Module` class. For example , we can build the `nn.Linear` on our own using te tensor introduced earlier! We can also build new, more complex modules, such a s a custom neural network. \n",
    "\n",
    "To create a custom module, the first thing we have to do is to extend the `nn.Module` . We can then initialize our parameters in the `__init__` function, staring with a call to the `__init__` function of the super class. Al the class attributes we define which are `nn` module objects are treated as parameters , which can then learned during the training. Tensors are not parameters , but they can be turned into parameters if they are wrapp in `nn.Parameters` class \n",
    "\n",
    "All classes extending the `nn.Module` are also expected to implement a `forward(x)` function, where `x` is the tensor. this is the function that is called when a parameters is passed to our modul, for example `modelFooBarBaz(x)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiLayerPerceptron(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        # Call to the super() class \n",
    "        super(MultiLayerPerceptron, self).__init__()\n",
    "\n",
    "        # saving initialization params\n",
    "        self.input_size = input_size \n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        #define our model \n",
    "        # doesn't have to be named model \n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(self.input_size, self.hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(self.hidden_size, self.input_size),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        output = self.model(x)\n",
    "        return output\n",
    "\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3950, 0.4821, 0.3842, 0.5562, 0.4945],\n",
       "        [0.4746, 0.5936, 0.3286, 0.4435, 0.3822]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now lets instantiate it and see what it does. \n",
    "\n",
    "input = torch.randn(2,5)\n",
    "\n",
    "multi_layer_perceptron = MultiLayerPerceptron(5,3)\n",
    "\n",
    "#pass our input through our model \n",
    "multi_layer_perceptron(input)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('model.0.weight',\n",
       "  Parameter containing:\n",
       "  tensor([[-0.3665, -0.0611,  0.4206, -0.1927, -0.2036],\n",
       "          [-0.3317, -0.4113, -0.1653, -0.2854, -0.1773],\n",
       "          [-0.4202,  0.1413,  0.0547, -0.0287, -0.4163]], requires_grad=True)),\n",
       " ('model.0.bias',\n",
       "  Parameter containing:\n",
       "  tensor([ 0.4075, -0.2855, -0.0500], requires_grad=True)),\n",
       " ('model.2.weight',\n",
       "  Parameter containing:\n",
       "  tensor([[ 0.1899,  0.2779,  0.1600],\n",
       "          [ 0.1802,  0.2193,  0.5014],\n",
       "          [-0.2587,  0.1474,  0.2716],\n",
       "          [-0.3371,  0.4262,  0.0181],\n",
       "          [-0.4094,  0.2103,  0.2476]], requires_grad=True)),\n",
       " ('model.2.bias',\n",
       "  Parameter containing:\n",
       "  tensor([-0.4264, -0.0715, -0.4720,  0.2259, -0.0221], requires_grad=True))]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can inspect the parameters of our model with `named_parameters()` and `parameters()`\n",
    "list(multi_layer_perceptron.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([[-0.1681,  0.3615, -0.1085,  0.2136, -0.3068],\n",
       "         [ 0.1105, -0.0750, -0.4361,  0.4284,  0.3173],\n",
       "         [ 0.1914,  0.1469,  0.4325, -0.1647, -0.3230]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([-0.0533, -0.1809, -0.3992], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([[ 0.5454,  0.0090, -0.0701],\n",
       "         [ 0.4629, -0.2518, -0.4816],\n",
       "         [ 0.0641,  0.3194,  0.4202],\n",
       "         [ 0.5256,  0.1806, -0.4929],\n",
       "         [ 0.0383,  0.4789, -0.3110]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([-0.0959,  0.4256, -0.3175, -0.5408, -0.3457], requires_grad=True)]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(multi_layer_perceptron.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimization \n",
    "\n",
    "We have showed how gradients are calculated with the `backward()` function. Having the gradients isn't enough for our models to learn. We also need to know how to update the parameters of our models. this is where the optimizers comes in. `torch.optim` module contains several optimizers that we can use. Some popular examples are `optim.SGD` and `optim.Adam`. When initalizing optimizers, we pass our model parameters which can be accessed with `model.parameters()`, tellin g the optimizers which values it will optimizing. Optimizers also have a learning rate `lr` parameters, which determines how big of a n update will be made in every step. Different optimizers have different hypterparameters as well. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "after we have our optimization function, we can defien a loss that we want to optimize for. We can either define the loss ourselve, or use one of th epredefined loss function in `pyTorch`, such as `nn.BCELoss()`. Let's put everything together now! We will start by creating some dummy data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.6229, 1.6178, 1.8535, 1.6113, 1.0269],\n",
       "        [1.2863, 1.9466, 1.5752, 1.0483, 1.0441],\n",
       "        [1.9156, 1.2305, 1.4923, 1.9213, 1.9353],\n",
       "        [1.1242, 1.0729, 1.3030, 1.0789, 1.0008],\n",
       "        [1.9523, 1.0653, 1.9383, 1.7856, 1.3598],\n",
       "        [1.2001, 1.4937, 1.9633, 1.7623, 1.8388],\n",
       "        [1.9458, 1.6999, 1.5424, 1.1136, 1.2931],\n",
       "        [1.3253, 1.6970, 1.1702, 1.5542, 1.9958],\n",
       "        [1.4209, 1.7721, 1.8824, 1.5063, 1.6478],\n",
       "        [1.8896, 1.3686, 1.0754, 1.1028, 1.7788]])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.optim as optim \n",
    "\n",
    "# create label/y/output data \n",
    "y = torch.ones(10, 5)\n",
    "\n",
    "# add some noise to our y to geenrate our x \n",
    "x = y + torch.rand_like(y)\n",
    "x\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define our \n",
    "1. model \n",
    "2. optimizer \n",
    "3. loss function (to optimize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.optim.adam.Adam'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7805761098861694"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multi_layer_perceptron = MultiLayerPerceptron(5,3)\n",
    "\n",
    "# optimizer \n",
    "adam = optim.Adam(multi_layer_perceptron.parameters(), lr=1e-1)\n",
    "print(type(adam))\n",
    "\n",
    "# loss function \n",
    "loss_function = nn.BCELoss()\n",
    "\n",
    "# calculate how our model is doing now \n",
    "y_pred = multi_layer_perceptron(x)\n",
    "loss_function(y_pred, y).item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see if we can have our model achieve a smaller loss. Now that we have everything we need, we can setup our training loop. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: training loss: 0.7805761098861694\n",
      "Epoch 1: training loss: 0.465933233499527\n",
      "Epoch 2: training loss: 0.21142368018627167\n",
      "Epoch 3: training loss: 0.06858699023723602\n",
      "Epoch 4: training loss: 0.018102366477251053\n",
      "Epoch 5: training loss: 0.004522338043898344\n",
      "Epoch 6: training loss: 0.0011653534602373838\n",
      "Epoch 7: training loss: 0.00032139578252099454\n",
      "Epoch 8: training loss: 9.614464215701446e-05\n",
      "Epoch 9: training loss: 3.128984462819062e-05\n"
     ]
    }
   ],
   "source": [
    "# set number of epoch which determines the number of training iterations. \n",
    "n_epoch = 10 \n",
    "\n",
    "for epoch in range(n_epoch):\n",
    "    #set gradients to zero \n",
    "    adam.zero_grad()\n",
    "\n",
    "    # get model predictions \n",
    "    y_pred = multi_layer_perceptron(x)\n",
    "\n",
    "\n",
    "    # calculate the loss \n",
    "    loss = loss_function(y_pred, y)\n",
    "\n",
    "    # print stats\n",
    "    print(f\"Epoch {epoch}: training loss: {loss}\")\n",
    "\n",
    "    #computer the gradients \n",
    "    loss.backward() \n",
    "\n",
    "    # take a step to optimize the weights \n",
    "    adam.step()\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([[ 0.5215,  1.0621,  0.5877,  0.9016,  0.3710],\n",
       "         [ 0.8491,  0.6657,  0.3032,  1.1651,  1.0538],\n",
       "         [-0.2273, -0.2718,  0.0138, -0.5835, -0.7417]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([ 0.6412,  0.5603, -0.8180], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([[ 1.2475,  0.7545,  0.3486],\n",
       "         [ 1.2015,  0.5252, -0.0629],\n",
       "         [ 0.7851,  1.0803,  0.8389],\n",
       "         [ 1.2163,  0.9155, -0.0741],\n",
       "         [ 0.7432,  1.2263,  0.1078]], requires_grad=True),\n",
       " Parameter containing:\n",
       " tensor([0.5621, 1.1223, 0.3672, 0.1077, 0.3128], requires_grad=True)]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(multi_layer_perceptron.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we can see that our loss is decreasing. Let's check the predictions of our model now and see if they are close to our original y of all ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [0.9999, 0.9999, 0.9999, 1.0000, 0.9999],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000],\n",
       "        [1.0000, 1.0000, 1.0000, 1.0000, 1.0000]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = multi_layer_perceptron(x)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks our model almost perfectly learned to filter out the noise from the `x` that we passed in!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP - Word Window Classification \n",
    "\n",
    "In this section, our goal will be to train a model that will find the words in a sentence corresponding to a `LOCATION`, which will be always of span `1`. (Meaning tha t`San Fransisco` won't be recognized a s a `LOCATION`). our task is called `Word Window Classification` for a reason. Instead of letting our model to only take a look at one word in each forward pass, we would like tot be able to consider the context of the word in question. That i, for each word, we want our model to be aware of the surrounding words. Let's dive in! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data \n",
    "\n",
    "# Our raw corpus. \n",
    "corpus = [\n",
    "    \"We always come to Paris\", \n",
    "    \"The processor is from Australia\", \n",
    "    \"I live in Stanford\",\n",
    "    \"He comes from Taiwan\",\n",
    "    \"The capital of Turkey is Ankara\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing\n",
    "\n",
    "To make it easier for our models to learn, we usually apply a few preprocessing steps to our data. This is especially important when dealing with text data. Here are some examples of text preprocessing: \n",
    "\n",
    "* Tokenization: turning sentence in token representation \n",
    "* Lowercasing: convert all letters to lowercase \n",
    "* Noise Removal: removing special characters (such as punctuations)\n",
    "* Stop words removal: removing commonly used words \n",
    "\n",
    "Which preprocessing steps are necessary is determind by the task at hand. For example, although it is useful to remove special characers in some tasks, for other they may be important( for example, if we are dealing with ultiple langugaes). For our task, we will lowercase our words and tokenize. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['we', 'always', 'come', 'to', 'paris'],\n",
       " ['the', 'processor', 'is', 'from', 'australia'],\n",
       " ['i', 'live', 'in', 'stanford'],\n",
       " ['he', 'comes', 'from', 'taiwan'],\n",
       " ['the', 'capital', 'of', 'turkey', 'is', 'ankara']]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def preprocess_sentence(sentence):\n",
    "    return sentence.lower().split()\n",
    "\n",
    "# create our training set \n",
    "train_sentences = [preprocess_sentence(sentence) for sentence in corpus]\n",
    "train_sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for each training example we have, we should also have a corresponding label. Recall that the goal of our model was to determine which words correspond to a `LOCATION`. That is, we want our model to output `0` for all the words that are not `LOCATION` and `1` for the ones that are `LOCATION`s. But we need to create a data for our output "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0, 0, 0, 0, 1],\n",
       " [0, 0, 0, 0, 1],\n",
       " [0, 0, 0, 1],\n",
       " [0, 0, 0, 1],\n",
       " [0, 0, 0, 1, 0, 1]]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set of lcoations tha tappear in our corpus \n",
    "locations = set([\"australia\", \"ankara\", \"paris\", \"stanford\", \"taiwan\", \"turkey\"])\n",
    "\n",
    "# our train labels \n",
    "train_labels = [[1 if word in locations else 0 for word in sentence] for sentence in train_sentences]\n",
    "train_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting Words to Embeddings \n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b7e818f66e33c31ac0526ee7f8556503ff93918b8b22809241939dc19e90de0b"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit ('pytorch_m1': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:14) \n[Clang 12.0.1 ]"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}