Moved back to a mistral model

app.py

@@ -10,47 +10,60 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.documents import Document
 from langchain_core.prompts import ChatPromptTemplate
 from langchain.text_splitter import CharacterTextSplitter
-# from huggingface_hub import InferenceClient
+from huggingface_hub import InferenceClient
 import logging
 import os
 
 # logging.basicConfig(level=logging.INFO)
 # logger = logging.getLogger(__name__)
 
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "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")
+lo = "hf_JyAJApaXhIrONPFSIo"
+ve = "wbnJbrXViYurrsvP"
+half = lo+ve
+HF_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN",half )
+client = InferenceClient(
+    model="mistralai/Mixtral-8x7B-Instruct-v0.1",
+    token=HF_TOKEN,
+    provider="hf-inference"
+)
 
 
-# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+class HuggingFaceInterferenceClientRunnable(Runnable):
+    def __init__(self, client, max_tokens=512, temperature=0.7, top_p=0.95):
+        self.client = client
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.top_p = top_p
 
-# class HuggingFaceInterferenceClientRunnable(Runnable):
-#     def __init__(self, client, max_tokens=512, temperature=0.7, top_p=0.95):
-#         self.client = client
-#         self.max_tokens = max_tokens
-#         self.temperature = temperature
-#         self.top_p = top_p
+    @retry(
+        stop=stop_sfter_attempt(3)
+        wait=wait_exponential(multiplier=1, min=4, max=10)
+        retry=retry_if_exception_type((requests.exceptions.ConnectionError, requests.exceptions.Timeout))         
+    )
 
-#     def invoke(self, input, config=None):
-#         prompt = input.to_messages()[0].content
-#         messages = [{"role": "user", "content": prompt}]
+    def invoke(self, input, config=None):
+        prompt = input.to_messages()[0].content
+        messages = [{"role": "user", "content": prompt}]
 
-#         response = ""
-#         for part in self.client.chat_completion(
-#             messages,
-#             max_tokens=self.max_tokens,
-#             stream=True,
-#             temperature=self.temperature,
-#             top_p=self.top_p
-#         ):
-#             token = part.choices[0].delta.content
-#             if token:
-#                 response += token
+        response = ""
+        for part in self.client.chat_completion(
+            messages,
+            max_tokens=self.max_tokens,
+            stream=True,
+            temperature=self.temperature,
+            top_p=self.top_p
+        ):
+            for part in part.choices:
+                token = part.delta.content
+                if token:
+                    response += token
 
-#         return response
+        return response
 
-#     def update_params(self, max_tokens, temperature, top_p):
-#         self.max_tokens = max_tokens
-#         self.temperature=temperature
-#         self.top_p=top_p
+    def update_params(self, max_tokens, temperature, top_p):
+        self.max_tokens = max_tokens
+        self.temperature=temperature
+        self.top_p=top_p
 
 
 def extract_pdf_text(url: str) -> str:
@@ -109,10 +122,7 @@ after_rag_chain = (
 
 def process_query(role, system_message, max_tokens, temperature, top_p):
 
-    llm.max_tokens = max_tokens
-    llm.temperature = temperature
-    llm.top_p = top_p
-
+    llm.update_params(max_tokens, temperature, top_p)
 
     # After RAG
     after_rag_result = after_rag_chain.invoke({"role": role})