Update app.py

app.py

@@ -129,8 +129,13 @@ def talk(prompt, history):
       tokenizer.convert_tokens_to_ids("<|eot_id|>") # Converts a token strings in a single/ sequence of integer id using the vocabulary
     ]
 # indicates the end of a sequence
+    text = ""
+    stream = model(formatted_prompt, max_tokens=1000, stop=["</s>"], stream=True)
+    for output in stream:
+        text += output["choices"][0]["text"]
+        yield text
 
-    model_input = model.create_chat_completion(messages = messages)
+#    model_input = model.create_chat_completion(messages = messages)
     
  #   input_ids = tokenizer.apply_chat_template(
  #     messages,
@@ -166,9 +171,9 @@ def talk(prompt, history):
  #   t.start()
   #  print("check11")
     # start a thread
-    outputs = []  
-    outputs = model_input
-    return outputs
+#    outputs = []  
+#    outputs = model_input
+#    return outputs
 #    print(model.tokenize(messages))
 #    tokens = model.tokenize(messages)
 #    for token in model.generate(tokens):
@@ -190,20 +195,7 @@ def talk(prompt, history):
  #   print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
 
 
-    #outputtokens =  model.generate(input_ids)
-    print("check9")
-    #print(outputtokens)
-    
-    #outputs = model.detokenize(outputtokens, decode = True)
-    #print(outputs)
-#    for token in model.generate(input_ids):
-#      print(model.detokenize(token))
-  #      outputs.append(model.detokenize(token))
-  #     output = model.detokenize(token)
-  #     print(outputs)
-       # yield "".join(outputs)
-       # print("check12")
-
+ 
     pd.options.display.max_colwidth = 800
     print("check13")
 #    outputstring = ''.join(outputs)