Update app.py

app.py

@@ -133,28 +133,21 @@ def talk(prompt, history):
 
    # formatted_prompt_with_history = formatted_prompt_with_history[:600] # to avoid memory issue
 #    print(formatted_prompt_with_history)
-    messages = [{"role":"system","content":SYS_PROMPT},{"role":"user","content":formatted_prompt}]  
+#    messages = [{"role":"system","content":SYS_PROMPT},{"role":"user","content":formatted_prompt}]  
     # binding the system context and new prompt for LLM
     # the chat template structure should be based on text generation model format
     print("check6")    
-    print(messages)
-    print("check7")
-    streamer = TextIteratorStreamer(
-    tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True
-    )
-    # stores print-ready text in a queue, to be used by a downstream application as an iterator. removes special tokens in generated text. 
-    # timeout for text queue. tokenizer for decoding tokens
-    # called by generate_kwargs
 
     terminators = [
       tokenizer.eos_token_id, # End-of-Sequence Token that indicates where the model should consider the text sequence to be complete
       tokenizer.convert_tokens_to_ids("<|eot_id|>") # Converts a token strings in a single/ sequence of integer id using the vocabulary
     ]
 # indicates the end of a sequence
-    text = ""
-    stream = model.create_chat_completion(messages, max_tokens=1000, stop=["</s>"], stream=False)
-    return stream
-#    for output in stream:
+    
+    output = model.create_chat_completion(messages = [{"role":"system","content":SYS_PROMPT},{"role":"user","content":formatted_prompt}], max_tokens=1000, stop=["</s>"], stream=True)
+    print(output['choices'][0]['message']['content'])
+
+#   for output in stream:
 #        text += output["choices"][0]["text"]
 #        yield text
 
@@ -171,16 +164,6 @@ def talk(prompt, history):
 #    print("check7")  
 #    print(input_ids.dtype)
 
-#    generate_kwargs = dict(
-#        tokens= input_ids) #,
-#        streamer=streamer,
-#        do_sample=True,
-#        eos_token_id=terminators,
-#    )
-    
-  #  outputs = model.generate( 
-  #  )
-  #  print(outputs)
     # calling the model to generate response based on message/ input
     # do_sample if set to True uses strategies to select the next token from the probability distribution over the entire vocabulary
     # temperature controls randomness. more renadomness with higher temperature
@@ -202,6 +185,7 @@ def talk(prompt, history):
 #    for token in model.generate(tokens):
 #       print(model.detokenize([token]))
  #   input_ids = tokenizer(*messages)
+
 #    print(model.generate(tensor([[    1,   529, 29989,  5205, 29989]])))
 #    start = time.time()
 #    NUM_TOKENS=0
@@ -218,19 +202,7 @@ def talk(prompt, history):
  #   print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
 
 
- 
-    pd.options.display.max_colwidth = 800
-    print("check13")
-#    outputstring = ''.join(outputs)
 
-#    global historylog
-#    historynew = {
-#       "Prompt": prompt,
-#       "Output": outputstring
-#    }
-#    historylog.append(historynew)
-#    return historylog
-#    print(historylog)
     
     
 TITLE = "AI Copilot for Diabetes Patients"