Update app.py

app.py

@@ -19,6 +19,28 @@ For more information on `huggingface_hub` Inference API support, please check th
 """
 client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
+model_id = 'mistralai/Mistral-7B-Instruct-v0.1'
+model_config = transformers.AutoConfig.from_pretrained(
+    model_id,
+    max_new_tokens=200
+)
+
+model = transformers.AutoModelForCausalLM.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    config=model_config,
+    quantization_config=bnb_config,
+    device_map='auto',
+)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+query_pipeline = transformers.pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        torch_dtype=torch.float16,
+        device_map="auto", max_new_tokens=200)
+
+
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     
     URL = "https://www.esmo.org/content/download/6594/114963/1/ES-Cancer-de-Mama-Guia-para-Pacientes.pdf"
@@ -36,6 +58,7 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
     
     vectordb = Chroma.from_documents(documents=all_splits, embedding=embeddings, persist_directory="chroma_db")
 
+    pipeline=query_pipeline
     query = message
     docs = vectordb.similarity_search_with_score(query)
     context = []