Trying lmsys/fastchat-t5 model

app.py

@@ -5,8 +5,6 @@ from query_data import get_chain
 import os
 import time
 
-os.environ["OPENAI_API_KEY"] = "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"
-
 st.set_page_config(page_title="LangChain Local PDF Chat", page_icon=":robot:")
 
 footer="""<style>

query_data.py

@@ -1,20 +1,28 @@
-from langchain.llms import LlamaCpp
+#from langchain.llms import LlamaCpp
+from langchain import HuggingFacePipeline
 from langchain.chains import ConversationalRetrievalChain
-from huggingface_hub import hf_hub_download
+#from huggingface_hub import hf_hub_download
 
-import psutil
+#import psutil
 import os
 
-
-
+offload_path = "offload"
 
 def get_chain(vectorstore):
-    if not os.path.exists("ggml-vic7b-q5_1.bin"):
-        hf_hub_download(repo_id="eachadea/ggml-vicuna-7b-1.1", filename="ggml-vic7b-q5_1.bin", local_dir=".")
-    llm = LlamaCpp(model_path="ggml-vic7b-q5_1.bin", n_ctx=2048, n_threads=psutil.cpu_count(logical=False))
+    #if not os.path.exists("ggml-vic7b-q5_1.bin"):
+    #    hf_hub_download(repo_id="eachadea/ggml-vicuna-7b-1.1", filename="ggml-vic7b-q5_1.bin", local_dir=".")
+    #llm = LlamaCpp(model_path="ggml-vic7b-q5_1.bin", n_ctx=2048, n_threads=psutil.cpu_count(logical=False))
+    if not os.path.exists(offload_path):
+        os.makedirs(offload_path)
+    llm = HuggingFacePipeline.from_model_id(model_id="lmsys/fastchat-t5-3b-v1.0",
+                                           task="text2text-generation", 
+                                           model_kwargs={"max_length":512,
+                                                        "device_map":"auto",
+                                                        "offload_folder":"offload"
+                                                        }
+                                           )
     qa_chain = ConversationalRetrievalChain.from_llm(
-        llm,
-        vectorstore.as_retriever(),
-#        condense_question_prompt=CONDENSE_QUESTION_PROMPT,
-    )
+                llm,
+                vectorstore.as_retriever(),
+                )
     return qa_chain

requirements.txt

@@ -1,9 +1,9 @@
 langchain
 typing-extensions>=4.5.0
-llama-cpp-python
 streamlit_chat
 pypdf
 chromadb
 tensorflow_text
-psutil
-huggingface-hub
+huggingface-hub
+accelerate
+transformers