Update app.py

app.py

@@ -5,6 +5,7 @@ import faiss
 import time
 #import torch
 import pandas as pd
+from llama_cpp import Llama
 
 from transformers import AutoTokenizer, GenerationConfig #, AutoModelForCausalLM
 #from transformers import AutoModelForCausalLM, AutoModel
@@ -64,8 +65,18 @@ generation_config = AutoConfig.from_pretrained(
 )
 # send additional parameters to model for generation
 
-model = AutoModelForCausalLM.from_pretrained(llm_model, model_file = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", model_type="llama", gpu_layers=0, config = generation_config)
-
+#model = llama_cpp.Llama(model_path = tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf AutoModelForCausalLM.from_pretrained(llm_model, model_file = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", model_type="llama", gpu_layers=0, config = generation_config)
+model = LlamaCpp(
+    model_path="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
+    chat_format="llama-2"
+    n_gpu_layers = 0,
+    temperature=0.75,
+    max_tokens=500,
+    top_p=0.95,
+    # callback_manager=callback_manager,
+    # verbose=True,  # Verbose is required to pass to the callback manager
+    )
+                        
 def search(query: str, k: int = 2 ):
     """a function that embeds a new query and returns the most probable results"""
     embedded_query = embedding_model.encode(query) # create embedding of a new query
@@ -117,11 +128,13 @@ def talk(prompt, history):
     ]
 # indicates the end of a sequence
 
-    input_ids = tokenizer.apply_chat_template(
-      messages,
-      add_generation_prompt=True,
-      return_tensors="pt"  
-     )
+    model_input = model.create_chat_completion(messages = messages)
+    
+ #   input_ids = tokenizer.apply_chat_template(
+ #     messages,
+ #     add_generation_prompt=True,
+ #     return_tensors="pt"  
+ #    )
     # preparing tokens for model input
     # add_generation_prompt argument tells the template to add tokens that indicate the start of a bot response
 #    print(input_ids)
@@ -152,24 +165,26 @@ def talk(prompt, history):
   #  print("check11")
     # start a thread
     outputs = []  
-    print(messages)
-    print(*messages)
+    print(model_input)
     print(model.tokenize(messages))
+    tokens = model.tokenize(messages)
+    for token in model.generate(tokens):
+       print(model.detokenize([token]))
  #   input_ids = tokenizer(*messages)
 #    print(model.generate(tensor([[    1,   529, 29989,  5205, 29989]])))
-    start = time.time()
-    NUM_TOKENS=0
-    print('-'*4+'Start Generation'+'-'*4)
-    for token in model.generate(input_ids):
-    	print(model.detokenize(input_ids), end='', flush=True)
-    	NUM_TOKENS+=1
-    time_generate = time.time() - start
-    print('\n')
-    print('-'*4+'End Generation'+'-'*4)
-    print(f'Num of generated tokens: {NUM_TOKENS}')
-    print(f'Time for complete generation: {time_generate}s')
-    print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
-    print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
+#    start = time.time()
+#    NUM_TOKENS=0
+#    print('-'*4+'Start Generation'+'-'*4)
+#    for token in model.generate(input_ids):
+#    	print(model.detokenize(input_ids), end='', flush=True)
+#    	NUM_TOKENS+=1
+#    time_generate = time.time() - start
+#    print('\n')
+#    print('-'*4+'End Generation'+'-'*4)
+#    print(f'Num of generated tokens: {NUM_TOKENS}')
+ #   print(f'Time for complete generation: {time_generate}s')
+ #   print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
+ #   print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
 
 
     #outputtokens =  model.generate(input_ids)