switching back to llama.cpp due to memory limits on huggingface free tier

query_data.py

@@ -1,26 +1,26 @@
-#from langchain.llms import LlamaCpp
-from langchain import HuggingFacePipeline
-from langchain.chains import ConversationalRetrievalChain
-#from huggingface_hub import hf_hub_download
+from langchain.llms import LlamaCpp
+#from langchain import HuggingFacePipeline
+#from langchain.chains import ConversationalRetrievalChain
+from huggingface_hub import hf_hub_download
 
-#import psutil
+import psutil
 import os
 
-offload_path = "offload"
+#offload_path = "offload"
 
 def get_chain(vectorstore):
-    #if not os.path.exists("ggml-vic7b-q5_1.bin"):
-    #    hf_hub_download(repo_id="eachadea/ggml-vicuna-7b-1.1", filename="ggml-vic7b-q5_1.bin", local_dir=".")
-    #llm = LlamaCpp(model_path="ggml-vic7b-q5_1.bin", n_ctx=2048, n_threads=psutil.cpu_count(logical=False))
-    if not os.path.exists(offload_path):
-        os.makedirs(offload_path)
-    llm = HuggingFacePipeline.from_model_id(model_id="lmsys/fastchat-t5-3b-v1.0",
-                                           task="text2text-generation", 
-                                           model_kwargs={"max_length":512,
-                                                        "device_map":"auto",
-                                                        "offload_folder":"offload"
-                                                        }
-                                           )
+    if not os.path.exists("ggml-vic7b-q5_1.bin"):
+        hf_hub_download(repo_id="eachadea/ggml-vicuna-7b-1.1", filename="ggml-vic7b-q5_1.bin", local_dir=".")
+    llm = LlamaCpp(model_path="ggml-vic7b-q5_1.bin", n_ctx=2048, n_threads=psutil.cpu_count(logical=False))
+    #if not os.path.exists(offload_path):
+    #    os.makedirs(offload_path)
+    #llm = HuggingFacePipeline.from_model_id(model_id="lmsys/fastchat-t5-3b-v1.0",
+    #                                       task="text2text-generation", 
+    #                                       model_kwargs={"max_length":512,
+    #                                                    "device_map":"auto",
+    #                                                    "offload_folder":"offload"
+    #                                                    }
+    #                                       )
     qa_chain = ConversationalRetrievalChain.from_llm(
                 llm,
                 vectorstore.as_retriever(),

requirements.txt

@@ -1,9 +1,8 @@
 langchain
-typing-extensions>=4.5.0
+typing-extensions
 streamlit_chat
 pypdf
+llama-cpp-python
 chromadb
-tensorflow_text
-huggingface-hub
 accelerate
 transformers