Switching from zephyr to gpt turbo

app.py

@@ -5,47 +5,52 @@ from langchain_community.vectorstores import Chroma
 from langchain_huggingface import HuggingFaceEmbeddings, ChatHuggingFace
 from langchain_core.runnables import RunnablePassthrough, Runnable
 from io import BytesIO
+from langchain_openai import ChatOpenAI
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.documents import Document
 from langchain_core.prompts import ChatPromptTemplate
 from langchain.text_splitter import CharacterTextSplitter
-from huggingface_hub import InferenceClient
+# from huggingface_hub import InferenceClient
 import logging
+import os
 
 # logging.basicConfig(level=logging.INFO)
 # logger = logging.getLogger(__name__)
 
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "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")
 
-class HuggingFaceInterferenceClientRunnable(Runnable):
-    def __init__(self, client, max_tokens=512, temperature=0.7, top_p=0.95):
-        self.client = client
-        self.max_tokens = max_tokens
-        self.temperature = temperature
-        self.top_p = top_p
 
-    def invoke(self, input, config=None):
-        prompt = input.to_messages()[0].content
-        messages = [{"role": "user", "content": prompt}]
+# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
-        response = ""
-        for part in self.client.chat_completion(
-            messages,
-            max_tokens=self.max_tokens,
-            stream=True,
-            temperature=self.temperature,
-            top_p=self.top_p
-        ):
-            token = part.choices[0].delta.content
-            if token:
-                response += token
+# class HuggingFaceInterferenceClientRunnable(Runnable):
+#     def __init__(self, client, max_tokens=512, temperature=0.7, top_p=0.95):
+#         self.client = client
+#         self.max_tokens = max_tokens
+#         self.temperature = temperature
+#         self.top_p = top_p
 
-        return response
+#     def invoke(self, input, config=None):
+#         prompt = input.to_messages()[0].content
+#         messages = [{"role": "user", "content": prompt}]
 
-    def update_params(self, max_tokens, temperature, top_p):
-        self.max_tokens = max_tokens
-        self.temperature=temperature
-        self.top_p=top_p
+#         response = ""
+#         for part in self.client.chat_completion(
+#             messages,
+#             max_tokens=self.max_tokens,
+#             stream=True,
+#             temperature=self.temperature,
+#             top_p=self.top_p
+#         ):
+#             token = part.choices[0].delta.content
+#             if token:
+#                 response += token
+
+#         return response
+
+#     def update_params(self, max_tokens, temperature, top_p):
+#         self.max_tokens = max_tokens
+#         self.temperature=temperature
+#         self.top_p=top_p
 
 
 def extract_pdf_text(url: str) -> str:
@@ -70,7 +75,13 @@ vectorstore = Chroma.from_documents(
 )
 retriever = vectorstore.as_retriever()
 
-llm = HuggingFaceInterferenceClientRunnable(client)
+llm = ChatOpenAI(
+    model="gpt-3.5-turbo",
+    api_key=OPENAI_API_KEY,
+    max_tokens=512,
+    temperature=0.7,
+    top_p=0.95
+)
 
 # Before RAG chain
 before_rag_template = "What is {topic}"
@@ -103,7 +114,9 @@ after_rag_chain = (
 
 def process_query(role, system_message, max_tokens, temperature, top_p):
 
-    llm.update_params(max_tokens, temperature, top_p)
+    llm.max_tokens = max_tokens
+    llm.temperature = temperature
+    llm.top_p = top_p
 
     # Before RAG
     before_rag_result = before_rag_chain.invoke({"topic": "Hugging Face"})