to the moon bois

app.py

@@ -1,14 +1,18 @@
 import gradio as gr
 import gpt
 
-
-print(gpt.get_response("test"))
-
-
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
-demo = gr.Interface(fn=gpt.get_response, inputs="textbox", outputs="textbox")
+demo = gr.Interface(fn=gpt.get_response, inputs=["textbox",
+                                                gr.Slider(0, 100, value=50, step=1),
+                                                gr.Slider(0.1, 2.0, value=1.0),
+                                                 ], outputs="textbox", title="Mike Chat", article="""Mike is the greatest AI ever created. It was trained for about 8 hrs on my pc using fineweb-edu and open orca datasets. While it hallucinates a lot, it seems to be about on par with other lms of its size (about 160M params). Model details:
+                                                 block_size: 512
+                                                 n_layers: 12
+                                                 n_heads: 12
+                                                 d_model: 768
+                                                 (Same as gpt-2 but without weight tying)""")
 
 
 if __name__ == "__main__":

gpt.py

@@ -136,35 +136,21 @@ my_GPT.eval()
 
 eot = enc._special_tokens['<|endoftext|>']
 
-def get_response(in_text):
+def get_response(in_text, top_k=50, temperature=1):
   prompt = "USER: " + in_text + "\nASSISTANT: "
   input_tokens = enc.encode(prompt)
   output_tokens = enc.encode(prompt)
-  top_k = 50
-  top_p = 0
   for x in range(block_size):
     if len(input_tokens) > block_size:
       input_tokens = input_tokens[1:]
     context_tensor = torch.tensor(input_tokens).view(1, -1).to(device)
 
     logits, loss = my_GPT(context_tensor)
-    logits = logits[:, -1, :] 
+    logits = logits[:, -1, :] / temperature
     if top_k > 0:
           # Remove all tokens with a probability less than the last token of the top-k
           indices_to_remove = logits < torch.topk(logits, top_k, dim=1)[0][..., -1, None]
           logits[indices_to_remove] = float("-inf")
-    if top_p > 0.0:
-        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
-        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
-
-        # Remove tokens with cumulative probability above the threshold
-        sorted_indices_to_remove = cumulative_probs > top_p
-        # Shift the indices to the right to keep also the first token above the threshold
-        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
-        sorted_indices_to_remove[..., 0] = 0
-
-        indices_to_remove = sorted_indices[sorted_indices_to_remove]
-        logits[indices_to_remove] = float("-inf")
     probs = F.softmax(logits, dim=-1)
     result = torch.multinomial(probs, num_samples=1).item()
     if result == eot: