Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+news.csv filter=lfs diff=lfs merge=lfs -text

GPT_Shakespeare.ipynb

@@ -0,0 +1,801 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "machine_shape": "hm",
+      "gpuType": "A100"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "G6BvseJ-0VwS",
+        "outputId": "72cafdf8-dd7b-4412-a9bc-2cfcebfb6949"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "--2023-11-03 11:10:34--  https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\n",
+            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
+            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
+            "HTTP request sent, awaiting response... 200 OK\n",
+            "Length: 1115394 (1.1M) [text/plain]\n",
+            "Saving to: ‘input.txt’\n",
+            "\n",
+            "input.txt           100%[===================>]   1.06M  --.-KB/s    in 0.02s   \n",
+            "\n",
+            "2023-11-03 11:10:35 (48.3 MB/s) - ‘input.txt’ saved [1115394/1115394]\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "!wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "with open('input.txt', 'r', encoding='utf-8') as f:\n",
+        "    text = f.read()"
+      ],
+      "metadata": {
+        "id": "pxZym4QU1mCq"
+      },
+      "execution_count": 11,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import torch\n",
+        "import torch.nn as nn\n",
+        "from torch.nn import functional as F\n",
+        "\n",
+        "# hyperparameters\n",
+        "batch_size = 16 # how many independent sequences will we process in parallel?\n",
+        "block_size = 32 # what is the maximum context length for predictions?\n",
+        "max_iters = 5000\n",
+        "eval_interval = 100\n",
+        "learning_rate = 1e-3\n",
+        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+        "eval_iters = 200\n",
+        "n_embd = 64\n",
+        "n_head = 4\n",
+        "n_layer = 4\n",
+        "dropout = 0.0\n",
+        "\n",
+        "torch.manual_seed(1337)\n",
+        "\n",
+        "\n",
+        "# here are all the unique characters that occur in this text\n",
+        "chars = sorted(list(set(text)))\n",
+        "vocab_size = len(chars)\n",
+        "# create a mapping from characters to integers\n",
+        "stoi = { ch:i for i,ch in enumerate(chars) }\n",
+        "itos = { i:ch for i,ch in enumerate(chars) }\n",
+        "encode = lambda s: [stoi[c] for c in s] # encoder: take a string, output a list of integers\n",
+        "decode = lambda l: ''.join([itos[i] for i in l]) # decoder: take a list of integers, output a string\n",
+        "\n",
+        "# Train and test splits\n",
+        "data = torch.tensor(encode(text), dtype=torch.long)\n",
+        "n = int(0.9*len(data)) # first 90% will be train, rest val\n",
+        "train_data = data[:n]\n",
+        "val_data = data[n:]\n",
+        "\n",
+        "# data loading\n",
+        "def get_batch(split):\n",
+        "    # generate a small batch of data of inputs x and targets y\n",
+        "    data = train_data if split == 'train' else val_data\n",
+        "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
+        "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
+        "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
+        "    x, y = x.to(device), y.to(device)\n",
+        "    return x, y\n",
+        "\n",
+        "@torch.no_grad()\n",
+        "def estimate_loss():\n",
+        "    out = {}\n",
+        "    model.eval()\n",
+        "    for split in ['train', 'val']:\n",
+        "        losses = torch.zeros(eval_iters)\n",
+        "        for k in range(eval_iters):\n",
+        "            X, Y = get_batch(split)\n",
+        "            logits, loss = model(X, Y)\n",
+        "            losses[k] = loss.item()\n",
+        "        out[split] = losses.mean()\n",
+        "    model.train()\n",
+        "    return out\n",
+        "\n",
+        "class Head(nn.Module):\n",
+        "    \"\"\" one head of self-attention \"\"\"\n",
+        "\n",
+        "    def __init__(self, head_size):\n",
+        "        super().__init__()\n",
+        "        self.key = nn.Linear(n_embd, head_size, bias=False)\n",
+        "        self.query = nn.Linear(n_embd, head_size, bias=False)\n",
+        "        self.value = nn.Linear(n_embd, head_size, bias=False)\n",
+        "        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n",
+        "\n",
+        "        self.dropout = nn.Dropout(dropout)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        B,T,C = x.shape\n",
+        "        k = self.key(x)   # (B,T,C)\n",
+        "        q = self.query(x) # (B,T,C)\n",
+        "        # compute attention scores (\"affinities\")\n",
+        "        wei = q @ k.transpose(-2,-1) * C**-0.5 # (B, T, C) @ (B, C, T) -> (B, T, T)\n",
+        "        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) # (B, T, T)\n",
+        "        wei = F.softmax(wei, dim=-1) # (B, T, T)\n",
+        "        wei = self.dropout(wei)\n",
+        "        # perform the weighted aggregation of the values\n",
+        "        v = self.value(x) # (B,T,C)\n",
+        "        out = wei @ v # (B, T, T) @ (B, T, C) -> (B, T, C)\n",
+        "        return out\n",
+        "\n",
+        "class MultiHeadAttention(nn.Module):\n",
+        "    \"\"\" multiple heads of self-attention in parallel \"\"\"\n",
+        "\n",
+        "    def __init__(self, num_heads, head_size):\n",
+        "        super().__init__()\n",
+        "        self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n",
+        "        self.proj = nn.Linear(n_embd, n_embd)\n",
+        "        self.dropout = nn.Dropout(dropout)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        out = torch.cat([h(x) for h in self.heads], dim=-1)\n",
+        "        out = self.dropout(self.proj(out))\n",
+        "        return out\n",
+        "\n",
+        "class FeedFoward(nn.Module):\n",
+        "    \"\"\" a simple linear layer followed by a non-linearity \"\"\"\n",
+        "\n",
+        "    def __init__(self, n_embd):\n",
+        "        super().__init__()\n",
+        "        self.net = nn.Sequential(\n",
+        "            nn.Linear(n_embd, 4 * n_embd),\n",
+        "            nn.ReLU(),\n",
+        "            nn.Linear(4 * n_embd, n_embd),\n",
+        "            nn.Dropout(dropout),\n",
+        "        )\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        return self.net(x)\n",
+        "\n",
+        "class Block(nn.Module):\n",
+        "    \"\"\" Transformer block: communication followed by computation \"\"\"\n",
+        "\n",
+        "    def __init__(self, n_embd, n_head):\n",
+        "        # n_embd: embedding dimension, n_head: the number of heads we'd like\n",
+        "        super().__init__()\n",
+        "        head_size = n_embd // n_head\n",
+        "        self.sa = MultiHeadAttention(n_head, head_size)\n",
+        "        self.ffwd = FeedFoward(n_embd)\n",
+        "        self.ln1 = nn.LayerNorm(n_embd)\n",
+        "        self.ln2 = nn.LayerNorm(n_embd)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        x = x + self.sa(self.ln1(x))\n",
+        "        x = x + self.ffwd(self.ln2(x))\n",
+        "        return x\n",
+        "\n",
+        "# super simple bigram model\n",
+        "class BigramLanguageModel(nn.Module):\n",
+        "\n",
+        "    def __init__(self):\n",
+        "        super().__init__()\n",
+        "        # each token directly reads off the logits for the next token from a lookup table\n",
+        "        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)\n",
+        "        self.position_embedding_table = nn.Embedding(block_size, n_embd)\n",
+        "        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])\n",
+        "        self.ln_f = nn.LayerNorm(n_embd) # final layer norm\n",
+        "        self.lm_head = nn.Linear(n_embd, vocab_size)\n",
+        "\n",
+        "    def forward(self, idx, targets=None):\n",
+        "        B, T = idx.shape\n",
+        "\n",
+        "        # idx and targets are both (B,T) tensor of integers\n",
+        "        tok_emb = self.token_embedding_table(idx) # (B,T,C)\n",
+        "        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (T,C)\n",
+        "        x = tok_emb + pos_emb # (B,T,C)\n",
+        "        x = self.blocks(x) # (B,T,C)\n",
+        "        x = self.ln_f(x) # (B,T,C)\n",
+        "        logits = self.lm_head(x) # (B,T,vocab_size)\n",
+        "\n",
+        "        if targets is None:\n",
+        "            loss = None\n",
+        "        else:\n",
+        "            B, T, C = logits.shape\n",
+        "            logits = logits.view(B*T, C)\n",
+        "            targets = targets.view(B*T)\n",
+        "            loss = F.cross_entropy(logits, targets)\n",
+        "\n",
+        "        return logits, loss\n",
+        "\n",
+        "    def generate(self, idx, max_new_tokens):\n",
+        "        # idx is (B, T) array of indices in the current context\n",
+        "        for _ in range(max_new_tokens):\n",
+        "            # crop idx to the last block_size tokens\n",
+        "            idx_cond = idx[:, -block_size:]\n",
+        "            # get the predictions\n",
+        "            logits, loss = self(idx_cond)\n",
+        "            # focus only on the last time step\n",
+        "            logits = logits[:, -1, :] # becomes (B, C)\n",
+        "            # apply softmax to get probabilities\n",
+        "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
+        "            # sample from the distribution\n",
+        "            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
+        "            # append sampled index to the running sequence\n",
+        "            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)\n",
+        "        return idx\n",
+        "\n",
+        "model = BigramLanguageModel()\n",
+        "m = model.to(device)\n",
+        "# print the number of parameters in the model\n",
+        "print(sum(p.numel() for p in m.parameters())/1e6, 'M parameters')\n",
+        "\n",
+        "# create a PyTorch optimizer\n",
+        "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
+        "\n",
+        "for iter in range(max_iters):\n",
+        "\n",
+        "    # every once in a while evaluate the loss on train and val sets\n",
+        "    if iter % eval_interval == 0 or iter == max_iters - 1:\n",
+        "        losses = estimate_loss()\n",
+        "        print(f\"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n",
+        "\n",
+        "    # sample a batch of data\n",
+        "    xb, yb = get_batch('train')\n",
+        "\n",
+        "    # evaluate the loss\n",
+        "    logits, loss = model(xb, yb)\n",
+        "    optimizer.zero_grad(set_to_none=True)\n",
+        "    loss.backward()\n",
+        "    optimizer.step()\n",
+        "\n",
+        "# generate from the model\n",
+        "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
+        "print(decode(m.generate(context, max_new_tokens=2000)[0].tolist()))\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "U_mrE9Vd10Ab",
+        "outputId": "f443c391-fd7f-4c2d-dd0d-72ef25849ef6"
+      },
+      "execution_count": 12,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "0.209729 M parameters\n",
+            "step 0: train loss 4.4116, val loss 4.4022\n",
+            "step 100: train loss 2.6568, val loss 2.6670\n",
+            "step 200: train loss 2.5091, val loss 2.5060\n",
+            "step 300: train loss 2.4199, val loss 2.4337\n",
+            "step 400: train loss 2.3500, val loss 2.3563\n",
+            "step 500: train loss 2.2961, val loss 2.3126\n",
+            "step 600: train loss 2.2408, val loss 2.2501\n",
+            "step 700: train loss 2.2053, val loss 2.2187\n",
+            "step 800: train loss 2.1636, val loss 2.1870\n",
+            "step 900: train loss 2.1226, val loss 2.1483\n",
+            "step 1000: train loss 2.1017, val loss 2.1283\n",
+            "step 1100: train loss 2.0683, val loss 2.1174\n",
+            "step 1200: train loss 2.0376, val loss 2.0798\n",
+            "step 1300: train loss 2.0256, val loss 2.0645\n",
+            "step 1400: train loss 1.9919, val loss 2.0362\n",
+            "step 1500: train loss 1.9696, val loss 2.0304\n",
+            "step 1600: train loss 1.9625, val loss 2.0470\n",
+            "step 1700: train loss 1.9402, val loss 2.0119\n",
+            "step 1800: train loss 1.9085, val loss 1.9957\n",
+            "step 1900: train loss 1.9080, val loss 1.9869\n",
+            "step 2000: train loss 1.8834, val loss 1.9941\n",
+            "step 2100: train loss 1.8727, val loss 1.9758\n",
+            "step 2200: train loss 1.8585, val loss 1.9622\n",
+            "step 2300: train loss 1.8537, val loss 1.9503\n",
+            "step 2400: train loss 1.8419, val loss 1.9424\n",
+            "step 2500: train loss 1.8153, val loss 1.9407\n",
+            "step 2600: train loss 1.8267, val loss 1.9374\n",
+            "step 2700: train loss 1.8126, val loss 1.9344\n",
+            "step 2800: train loss 1.8054, val loss 1.9230\n",
+            "step 2900: train loss 1.8045, val loss 1.9339\n",
+            "step 3000: train loss 1.7963, val loss 1.9243\n",
+            "step 3100: train loss 1.7691, val loss 1.9208\n",
+            "step 3200: train loss 1.7506, val loss 1.9092\n",
+            "step 3300: train loss 1.7548, val loss 1.9038\n",
+            "step 3400: train loss 1.7582, val loss 1.8960\n",
+            "step 3500: train loss 1.7376, val loss 1.8934\n",
+            "step 3600: train loss 1.7232, val loss 1.8888\n",
+            "step 3700: train loss 1.7280, val loss 1.8814\n",
+            "step 3800: train loss 1.7221, val loss 1.8951\n",
+            "step 3900: train loss 1.7228, val loss 1.8789\n",
+            "step 4000: train loss 1.7168, val loss 1.8635\n",
+            "step 4100: train loss 1.7168, val loss 1.8798\n",
+            "step 4200: train loss 1.7088, val loss 1.8672\n",
+            "step 4300: train loss 1.6995, val loss 1.8501\n",
+            "step 4400: train loss 1.7096, val loss 1.8686\n",
+            "step 4500: train loss 1.6907, val loss 1.8546\n",
+            "step 4600: train loss 1.6868, val loss 1.8348\n",
+            "step 4700: train loss 1.6786, val loss 1.8346\n",
+            "step 4800: train loss 1.6659, val loss 1.8445\n",
+            "step 4900: train loss 1.6711, val loss 1.8384\n",
+            "step 4999: train loss 1.6630, val loss 1.8230\n",
+            "\n",
+            "ROMEO:\n",
+            "But you far you\n",
+            "my swap with thus; come hath I uD\n",
+            "If sleemition of where's granded\n",
+            "Of their of tout the gortune upwon alond, liege man to is Iell this surpe\n",
+            "And than sleue thus mind, his by blow,\n",
+            "Virdty toward butied, Ditire spresiss with thou some not.\n",
+            "\n",
+            "LORIO:\n",
+            "I am part\n",
+            "But thou sging them but\n",
+            "shat secondes morry thou sovore.\n",
+            "\n",
+            "ISABUS:\n",
+            "What art sade but hither, thange e'en,\n",
+            "Protes as kingle me; an your tords whom are Ineal.\n",
+            "\n",
+            "MENENIUS:\n",
+            "But little sweet, hom, foust cerfort;\n",
+            "Winth hing diend enirs' tompy beds sick ways!\n",
+            "What curforself this grace. Won, passes us.\n",
+            "\n",
+            "BUCKINGHABY MARD:\n",
+            "Mether star: keep it any head which\n",
+            "He tall devioly that, out that confer old.\n",
+            "Our thy dears time.\n",
+            "Nay, the fragoly, pair, of new\n",
+            "my father, my lip Backnoward:\n",
+            "God therring for respide\n",
+            "What colvery, teminelyord, I mast,\n",
+            "While us that such differs I'll that confect I come,\n",
+            "But; man.\n",
+            "\n",
+            "VOLUMNIO:\n",
+            "Ontread confail with me. Humser dipporbried answeraw is codal one,\n",
+            "Onjestion, not or cheavess ensty with.\n",
+            "\n",
+            "GLOUCESTER:\n",
+            "\n",
+            "HENRY Mess to Lies?\n",
+            "Stand and these beguare youf stile that than war\n",
+            "offity are, I usquesch\n",
+            "Frown movhapty not duke with you addom\n",
+            "grack prowd--lost\n",
+            "But but they worse is senst my crunne undolier. But, beauts pruntaly; I stoll'ct my nor Murder, I sot, though who speak\n",
+            "Your bout told-man rathing if anyshal\n",
+            "epitence, tirre no the said he's,\n",
+            "Andis frultifs. what his lide? That mirdy this dudgetions?\n",
+            "\n",
+            "KING ARINIA:\n",
+            "I let holt not sucKether,\n",
+            "Whither, efore But lord: I, beget because at that his say\n",
+            "as to brought grave a donesmer all nobe.\n",
+            "\n",
+            "BUCKINGHUMBY:\n",
+            "Which forgeled! Came; nor thereforn's fiends strefet.\n",
+            "\n",
+            "PLORIA:\n",
+            "Yet to Capprohning, that brird\n",
+            "of say mover a desrick.\n",
+            "\n",
+            "MO\n",
+            "stompars, God the\n",
+            "citchard is high.\n",
+            "\n",
+            "Seth Second Methere:\n",
+            "Marrmat I unmale the bretcius unfoect that I would back where own thy lurges\n",
+            "And, iffillimorture:\n",
+            "As thou twand, York these that high praut.\n",
+            "Plafe merprates sure dread with her,\n",
+            "At not your must I suchon? too prant!\n",
+            "O 'hiles clight the bleave is graved before\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# generate from the model\n",
+        "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
+        "print(decode(m.generate(context, max_new_tokens=2000)[0].tolist()))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "M0qIA2GK2qzI",
+        "outputId": "86126a68-17b1-4171-920a-1d2df6fa3f1a"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\n",
+            "And thou to lesserve his his know'st broy by A towe than or fuch dight none worthy'st countinne, congess\n",
+            "our ire Iname's marriatate the entrity?\n",
+            "\n",
+            "COMIOLUS:\n",
+            "Yet there me your let thy by courtary, own but I cannot, to\n",
+            "you.\n",
+            "\n",
+            "MOth Osque, while and nett; pity, brow umput;\n",
+            "He betwered's prettedy if not you arter,\n",
+            "But woman furner his good me to ambled thy follows\n",
+            "Gents for you daying this distend and he but.\n",
+            "\n",
+            "COMINANUS:\n",
+            "But you know wish the wear? whoe not to breave maste gate?\n",
+            "Not, now you read own. Lo-honour shoes\n",
+            "honordore vilibert.\n",
+            "\n",
+            "ARTOS:\n",
+            "Nay, as Is theen, God\n",
+            "Were I saying cose\n",
+            "Will there's upon and tools.\n",
+            "\n",
+            "HORSIO:\n",
+            "Pomfort life?\n",
+            "Whereform make comps hersed, my what away,\n",
+            "Go'st Your haste entens, and succe?\n",
+            "\n",
+            "LORD RIARENCE:\n",
+            "Fies my like, wifch a my nobt.\n",
+            "And!\n",
+            "And ways. Whithing death.\n",
+            "\n",
+            "CORIOLUMNO:\n",
+            "It must I have grawits.-\n",
+            "Ris Gomisty yor then thin dot this no all-donged,\n",
+            "But quarry the latter: Have me the betime twooke steed to blood\n",
+            "That his rysour grower-foldds: bnot Plond,\n",
+            "By that all wittore old the malt our liight.\n",
+            "Would for not\n",
+            "And sabet I sout ofing more in must.\n",
+            "\n",
+            "MENENIUS:\n",
+            "Gor low your I standed\n",
+            "To heavy:\n",
+            "While to caid your inswoes!\n",
+            "Thrhing the princlusior lurmeng,\n",
+            "To Whie! entred mean the not, sare.\n",
+            "\n",
+            "BRUTUS:\n",
+            "Is my partend him, if Is verys be,\n",
+            "Whim you longs,\n",
+            "Say, his me. Murselets; not with is most.\n",
+            "\n",
+            "JOLINA:\n",
+            "That it where that thluse too the hath'd\n",
+            "unsomed of our heavis'd?\n",
+            "\n",
+            "So his were Clamind:\n",
+            "Ounly mistry's soul\n",
+            "To once myser flow\n",
+            "Which, then, whet must I as not drums as the ouch are\n",
+            "burnse contreased and in Comintity?\n",
+            "\n",
+            "Mistray is I curliented:\n",
+            "Thou herew bottust, How lad you blist a wear's art?\n",
+            "What the vave--batta thing with\n",
+            "that his my urtusaed and as mine, thus not,\n",
+            "May your pohed me mhalt livy very\n",
+            "But I sham I ham kitse, pean, for\n",
+            "was, ewith woll heave in thou art, dlignt,\n",
+            "Of fair Griward that remottes must;\n",
+            "Cadyfore the not lords not, I say's gener which of your rame? Istand my hearth\n",
+            "And thou alt nenget that shame\n",
+            "She with them kinderire it put this\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "prompt = \"Once upon a time\"\n",
+        "context = torch.tensor(encode(prompt), dtype=torch.long, device=device).view(1, -1)\n",
+        "print(decode(m.generate(context, max_new_tokens=200)[0].tolist()))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "Na0SThjv5-iz",
+        "outputId": "c649c8ad-42fe-4a77-a219-9dcb1857a9c0"
+      },
+      "execution_count": 14,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Once upon a times peacts mother saclaves is 'Then of my tonguen,\n",
+            "Thus are been\n",
+            "My my behot prilatte, what you brot,\n",
+            "Speeke there is my bud the be, 'smandion from me:\n",
+            "And the barttes, rechard, where capuse,\n",
+            "Rentent, I\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "\n",
+        "# Save the model\n",
+        "torch.save(m.state_dict(), 'GPT_Shakespeare_language_model.pth')"
+      ],
+      "metadata": {
+        "id": "sfmRYo9h6B24"
+      },
+      "execution_count": 15,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load the model\n",
+        "loaded_model = BigramLanguageModel()  # Initialize an instance of your model\n",
+        "loaded_model.load_state_dict(torch.load('GPT_Shakespeare_language_model.pth'))\n",
+        "loaded_model.to(device).eval()  # Set the model to evaluation mode"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "xO9JefxH6KHS",
+        "outputId": "d7f0191c-4e02-4ed7-ff4b-b6f25a538fe5"
+      },
+      "execution_count": 17,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "BigramLanguageModel(\n",
+              "  (token_embedding_table): Embedding(65, 64)\n",
+              "  (position_embedding_table): Embedding(32, 64)\n",
+              "  (blocks): Sequential(\n",
+              "    (0): Block(\n",
+              "      (sa): MultiHeadAttention(\n",
+              "        (heads): ModuleList(\n",
+              "          (0-3): 4 x Head(\n",
+              "            (key): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (query): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (value): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (dropout): Dropout(p=0.0, inplace=False)\n",
+              "          )\n",
+              "        )\n",
+              "        (proj): Linear(in_features=64, out_features=64, bias=True)\n",
+              "        (dropout): Dropout(p=0.0, inplace=False)\n",
+              "      )\n",
+              "      (ffwd): FeedFoward(\n",
+              "        (net): Sequential(\n",
+              "          (0): Linear(in_features=64, out_features=256, bias=True)\n",
+              "          (1): ReLU()\n",
+              "          (2): Linear(in_features=256, out_features=64, bias=True)\n",
+              "          (3): Dropout(p=0.0, inplace=False)\n",
+              "        )\n",
+              "      )\n",
+              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "    )\n",
+              "    (1): Block(\n",
+              "      (sa): MultiHeadAttention(\n",
+              "        (heads): ModuleList(\n",
+              "          (0-3): 4 x Head(\n",
+              "            (key): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (query): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (value): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (dropout): Dropout(p=0.0, inplace=False)\n",
+              "          )\n",
+              "        )\n",
+              "        (proj): Linear(in_features=64, out_features=64, bias=True)\n",
+              "        (dropout): Dropout(p=0.0, inplace=False)\n",
+              "      )\n",
+              "      (ffwd): FeedFoward(\n",
+              "        (net): Sequential(\n",
+              "          (0): Linear(in_features=64, out_features=256, bias=True)\n",
+              "          (1): ReLU()\n",
+              "          (2): Linear(in_features=256, out_features=64, bias=True)\n",
+              "          (3): Dropout(p=0.0, inplace=False)\n",
+              "        )\n",
+              "      )\n",
+              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "    )\n",
+              "    (2): Block(\n",
+              "      (sa): MultiHeadAttention(\n",
+              "        (heads): ModuleList(\n",
+              "          (0-3): 4 x Head(\n",
+              "            (key): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (query): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (value): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (dropout): Dropout(p=0.0, inplace=False)\n",
+              "          )\n",
+              "        )\n",
+              "        (proj): Linear(in_features=64, out_features=64, bias=True)\n",
+              "        (dropout): Dropout(p=0.0, inplace=False)\n",
+              "      )\n",
+              "      (ffwd): FeedFoward(\n",
+              "        (net): Sequential(\n",
+              "          (0): Linear(in_features=64, out_features=256, bias=True)\n",
+              "          (1): ReLU()\n",
+              "          (2): Linear(in_features=256, out_features=64, bias=True)\n",
+              "          (3): Dropout(p=0.0, inplace=False)\n",
+              "        )\n",
+              "      )\n",
+              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "    )\n",
+              "    (3): Block(\n",
+              "      (sa): MultiHeadAttention(\n",
+              "        (heads): ModuleList(\n",
+              "          (0-3): 4 x Head(\n",
+              "            (key): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (query): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (value): Linear(in_features=64, out_features=16, bias=False)\n",
+              "            (dropout): Dropout(p=0.0, inplace=False)\n",
+              "          )\n",
+              "        )\n",
+              "        (proj): Linear(in_features=64, out_features=64, bias=True)\n",
+              "        (dropout): Dropout(p=0.0, inplace=False)\n",
+              "      )\n",
+              "      (ffwd): FeedFoward(\n",
+              "        (net): Sequential(\n",
+              "          (0): Linear(in_features=64, out_features=256, bias=True)\n",
+              "          (1): ReLU()\n",
+              "          (2): Linear(in_features=256, out_features=64, bias=True)\n",
+              "          (3): Dropout(p=0.0, inplace=False)\n",
+              "        )\n",
+              "      )\n",
+              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "    )\n",
+              "  )\n",
+              "  (ln_f): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "  (lm_head): Linear(in_features=64, out_features=65, bias=True)\n",
+              ")"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 17
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# generate from the model\n",
+        "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
+        "print(decode(loaded_model.generate(context, max_new_tokens=2000)[0].tolist()))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "m46OnNXq6PAV",
+        "outputId": "e547525f-98c0-4355-92ef-559f6c2ba238"
+      },
+      "execution_count": 18,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\n",
+            "Forntlefires, love the done, or all love tears\n",
+            "That braud the strough.\n",
+            "\n",
+            "BUCHNIO:\n",
+            "Is\n",
+            "For that I hat deam throve? we parrlignos;\n",
+            "My bregain minousiner mile into the doth,\n",
+            "Warwien not his day hath;\n",
+            "Whose basy touther ploudde metornies'drey would be themseremes to have\n",
+            "You good accarm, menot wtoo cown:\n",
+            "Is have mostil\n",
+            "Before prunces.\n",
+            "\n",
+            "Speaking A-dught:\n",
+            "Whow 'sile her fry hath acvionce,\n",
+            "Your cange, side of-day; this I seep!\n",
+            "Aher approve; I\n",
+            "drumber, any till amberd, come it suffet nexwarrans\n",
+            "To hear you that what art thim for a dish! Whiler not some men;\n",
+            "Hareth, I am broth, thenese oof.\n",
+            "Croth before wortune's hande and if brote\n",
+            "Come andmitation it. Tentess I what\n",
+            "That ascess Weringmans, te us;\n",
+            "And your Servant-thy moime, that whose.\n",
+            "\n",
+            "CORIOLANUS:\n",
+            "Now, stay to the resmorn?\n",
+            "\n",
+            "CRANGE:\n",
+            "It have pleave to some, for soul;\n",
+            "He fatelly here that you, hesseliemes five oldince\n",
+            "Our confolle, too you stay'd my being to't,\n",
+            "My lord I am then most the knows doot hid-gress.\n",
+            "\n",
+            "KING RICHARD GDITH:\n",
+            "As beconsure! So youil heart fear; and whilook my arm verpast,\n",
+            "And staven to fathy down I vir all prace,\n",
+            "And be betcasion your balt, to draying and the bottchmy,\n",
+            "The griake must worse it. As I have owle well I who stray good.\n",
+            "\n",
+            "My anviusice: andress unthat fonds of oad;\n",
+            "ne's eye the notraing and timer cimmman:\n",
+            "Heth lain. What's is the castad,\n",
+            "And their speake fatwort off.\n",
+            "\n",
+            "Shy:\n",
+            "What marry; thysele, time onge,\n",
+            "And by bown, merpety to to of crive thou secam.\n",
+            "\n",
+            "QUEEN VINCENTIO:\n",
+            "How my bold; good poson\n",
+            "I finly torthus.\n",
+            "Our you if your aware watly sweet\n",
+            "On all fair livishts thee our then plast banoting\n",
+            "What have duckn, so\n",
+            "them the hostfeive.\n",
+            "\n",
+            "HIRDIO:\n",
+            "\n",
+            "GLOUCERDIO:\n",
+            "And all capure Toncant mack.\n",
+            "\n",
+            "CAPULET:\n",
+            "O mean bodams'd my tone thy wralf thee wilth\n",
+            "And rencrown prow my ear them lovery\n",
+            "Coringlike hath in recond:\n",
+            "You will you from of God their all and not mine:\n",
+            "With doess be Sives?\n",
+            "So regort it thy mart solued sgaft world of him,\n",
+            "What'st in else agged namfutiol.\n",
+            "\n",
+            "ANGENO:\n",
+            "For that it you briave lay to your unpalssi\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "2GpnegQc8A9R"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

GPT_Shakespeare_language_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d2cbe6d12d0d566b1bfb4aed76ee5ca713cfda0f68d75666cb01675edcf72a2
+size 946452

app.py

@@ -0,0 +1,233 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+import numpy as np
+import random
+import re
+import gradio as gr
+
+# hyperparameters
+batch_size = 16 # how many independent sequences will we process in parallel?
+block_size = 32 # what is the maximum context length for predictions?
+max_iters = 5000
+eval_interval = 100
+learning_rate = 1e-3
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+eval_iters = 200
+n_embd = 64
+n_head = 4
+n_layer = 4
+dropout = 0.0
+# ------------
+
+torch.manual_seed(1337)
+
+class Head(nn.Module):
+    """ one head of self-attention """
+
+    def __init__(self, head_size):
+        super().__init__()
+        self.key = nn.Linear(n_embd, head_size, bias=False)
+        self.query = nn.Linear(n_embd, head_size, bias=False)
+        self.value = nn.Linear(n_embd, head_size, bias=False)
+        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
+
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        B,T,C = x.shape
+        k = self.key(x)   # (B,T,C)
+        q = self.query(x) # (B,T,C)
+        # compute attention scores ("affinities")
+        wei = q @ k.transpose(-2,-1) * C**-0.5 # (B, T, C) @ (B, C, T) -> (B, T, T)
+        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) # (B, T, T)
+        wei = F.softmax(wei, dim=-1) # (B, T, T)
+        wei = self.dropout(wei)
+        # perform the weighted aggregation of the values
+        v = self.value(x) # (B,T,C)
+        out = wei @ v # (B, T, T) @ (B, T, C) -> (B, T, C)
+        return out
+
+class MultiHeadAttention(nn.Module):
+    """ multiple heads of self-attention in parallel """
+
+    def __init__(self, num_heads, head_size):
+        super().__init__()
+        self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)])
+        self.proj = nn.Linear(n_embd, n_embd)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        out = torch.cat([h(x) for h in self.heads], dim=-1)
+        out = self.dropout(self.proj(out))
+        return out
+
+class FeedFoward(nn.Module):
+    """ a simple linear layer followed by a non-linearity """
+
+    def __init__(self, n_embd):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(n_embd, 4 * n_embd),
+            nn.ReLU(),
+            nn.Linear(4 * n_embd, n_embd),
+            nn.Dropout(dropout),
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+class Block(nn.Module):
+    """ Transformer block: communication followed by computation """
+
+    def __init__(self, n_embd, n_head):
+        # n_embd: embedding dimension, n_head: the number of heads we'd like
+        super().__init__()
+        head_size = n_embd // n_head
+        self.sa = MultiHeadAttention(n_head, head_size)
+        self.ffwd = FeedFoward(n_embd)
+        self.ln1 = nn.LayerNorm(n_embd)
+        self.ln2 = nn.LayerNorm(n_embd)
+
+    def forward(self, x):
+        x = x + self.sa(self.ln1(x))
+        x = x + self.ffwd(self.ln2(x))
+        return x
+
+# super simple bigram model
+class BigramLanguageModel(nn.Module):
+    def __init__(self, dataset_text, n_embd):
+        super().__init__()
+
+        # Compute character-related parameters
+        self.chars = sorted(list(set(dataset_text)))
+        self.vocab_size = len(self.chars)
+        self.stoi = {ch: i for i, ch in enumerate(self.chars)}
+        self.itos = {i: ch for ch, i in self.stoi.items()}
+
+        self.token_embedding_table = nn.Embedding(self.vocab_size, n_embd)
+        self.position_embedding_table = nn.Embedding(block_size, n_embd)
+        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.lm_head = nn.Linear(n_embd, self.vocab_size)
+        self.encode = lambda s: [self.stoi[c] for c in s] # encoder: take a string, output a list of integers
+        self.decode = lambda l: ''.join([self.itos[i] for i in l]) # decoder: take a list of integers, output a string
+
+        
+    def forward(self, idx, targets=None):
+        B, T = idx.shape
+
+        # idx and targets are both (B,T) tensor of integers
+        tok_emb = self.token_embedding_table(idx) # (B,T,C)
+        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (T,C)
+        x = tok_emb + pos_emb # (B,T,C)
+        x = self.blocks(x) # (B,T,C)
+        x = self.ln_f(x) # (B,T,C)
+        logits = self.lm_head(x) # (B,T,vocab_size)
+
+        if targets is None:
+            loss = None
+        else:
+            B, T, C = logits.shape
+            logits = logits.view(B*T, C)
+            targets = targets.view(B*T)
+            loss = F.cross_entropy(logits, targets)
+
+        return logits, loss
+
+    def generate(self, idx, max_new_tokens):
+        # idx is (B, T) array of indices in the current context
+        for _ in range(max_new_tokens):
+            # crop idx to the last block_size tokens
+            idx_cond = idx[:, -block_size:]
+            # get the predictions
+            logits, loss = self(idx_cond)
+            # focus only on the last time step
+            logits = logits[:, -1, :] # becomes (B, C)
+            # apply softmax to get probabilities
+            probs = F.softmax(logits, dim=-1) # (B, C)
+            # sample from the distribution
+            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)
+            # append sampled index to the running sequence
+            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)
+        return idx
+
+# Reading shakespeare data
+with open('input.txt', 'r', encoding='utf-8') as f:
+    shakespeare_text = f.read()
+
+
+# Reading wikipedia data
+DATA_PATH = 'wikisent2.txt'
+# load wikipedia sentences
+with open(DATA_PATH, 'r') as f:
+    lines = f.read().splitlines()
+
+# Selecting 250k lines from the dataset.
+random.seed(42)
+texts = random.choices(lines, k=250000)
+del lines
+
+def preprocess(text):
+    text = re.sub('@.*?\s+', '', text)  # Remove mentions
+    text = re.sub('#.*?\s+', '', text)  # Remove hashtags
+    text = re.sub(r'https?:\/\/.*[\r\n]*', '', text)  # Remove URLs
+    text = re.sub(r'[^\w\s\'.]', '', text)  # Remove special characters except for single quotes and periods
+    text = re.sub('\s+', ' ', text)  # Replace multiple spaces with a single space
+    text = re.sub('^\d+\s*|^\d+\.\d+\s*|^\d+\.\d+\.\d+\s*', '', text)  # Remove digits at the start of sentences
+    text = text.strip()  # Remove leading and trailing whitespace
+    return text
+
+wiki_text = [preprocess(t) for t in texts]
+wiki_text = '\n'.join(wiki_text)
+
+# Load the shakespeaere model
+shakespeare_model = BigramLanguageModel(shakespeare_text, n_embd).to(device)  # Initialize an instance of your model
+shakespeare_model.load_state_dict(torch.load('shakespeaere_language_model.pth', map_location=torch.device('cpu')))
+shakespeare_model.eval()  # Set the model to evaluation mode
+
+# Load the wikipedia model
+wikipedia_model = BigramLanguageModel(wiki_text, n_embd).to(device)  # Initialize an instance of your model
+wikipedia_model.load_state_dict(torch.load('wikipedia_language_model.pth', map_location=torch.device('cpu')))
+wikipedia_model.eval()  # Set the model to evaluation mode
+
+
+def generate_shakespeare_outputs(prompt=None, max_new_tokens=2000):
+  if prompt:
+    context = torch.tensor(shakespeare_model.encode(prompt), dtype=torch.long, device=device).view(1, -1)
+  else:
+    context = torch.zeros((1, 1), dtype=torch.long, device=device)
+  text_output = shakespeare_model.decode(shakespeare_model.generate(context, max_new_tokens=max_new_tokens)[0].tolist())
+  return text_output
+
+
+def generate_wikipedia_outputs(prompt=None, max_new_tokens=2000):
+  if prompt:
+    context = torch.tensor(wikipedia_model.encode(prompt), dtype=torch.long, device=device).view(1, -1)
+  else:
+    context = torch.zeros((1, 1), dtype=torch.long, device=device)
+  text_output = wikipedia_model.decode(wikipedia_model.generate(context, max_new_tokens=max_new_tokens)[0].tolist())
+  return text_output
+    
+
+title = "Nano GPT"
+
+description1 = "Nano GPT trained on <a href='https://www.kaggle.com/datasets/mikeortman/wikipedia-sentences'>Shakespeare dataset</a>. It is trained on a very small amount of data to understand how GPT's are trained and built. The implementation can be found <a href='https://github.com/karpathy/nanoGPT'>here.</a>"
+
+shakespeare_interface = gr.Interface(generate_shakespeare_outputs,
+                    inputs=[gr.Textbox(label="Enter any prompt ", type="text", value="Once upon a time,"),
+                            gr.Slider(minimum=100, maximum=5000, step=100, value=2000, label="Max new tokens")],
+                    outputs=gr.Textbox(label="Output generated", type="text"), description=description1)
+
+description2 = "Nano GPT trained on <a href='https://github.com/karpathy/char-rnn/blob/6f9487a6fe5b420b7ca9afb0d7c078e37c1d1b4e/data/tinyshakespeare/input.txt'>Wikipedia dataset</a>. It is trained on a very small amount of data to understand how GPT's are trained and built. The implementation can be found <a href='https://github.com/karpathy/nanoGPT'>here.</a>"
+
+wiki_interface = gr.Interface(generate_wikipedia_outputs,
+                    inputs=[gr.Textbox(label="Enter any prompt ", type="text", value="James Bond"),
+                            gr.Slider(minimum=100, maximum=5000, step=100, value=2000, label="Max new tokens")],
+                    outputs=gr.Textbox(label="Output generated", type="text"), description=description2)
+
+demo = gr.TabbedInterface([shakespeare_interface, wiki_interface], tab_names=["Shakespeare Data", "Wikipedia Data"], 
+                          title=title)
+
+
+demo.launch()

news.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b3df2ebf6b3b72bed68f665853caf5ab68345ba2f67618d6dae52add20a850d
+size 63807429

requirements.txt

@@ -0,0 +1,2 @@
+torch
+gradio