Use local inference.

app.py

@@ -1,24 +1,29 @@
 import gradio as gr
 from langchain import HuggingFaceHub
 from langchain.chains.question_answering import load_qa_chain
-from langchain.document_loaders import PyMuPDFLoader, TextLoader
+from langchain.document_loaders import TextLoader
 from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.llms import HuggingFacePipeline
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import FAISS
-from timeit import default_timer as timer
 
 
+print("Loading documents")
 loader = TextLoader("rdna3.txt")
 documents = loader.load()
 
+print("Creating chunks")
 splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=50)
 chunks = splitter.split_documents(documents)
 
+print("Creating database")
 embeddings = HuggingFaceEmbeddings()
 db = FAISS.from_documents(chunks, embeddings)
 
-llm = HuggingFaceHub(
-    repo_id='google/flan-t5-base',
+print("Loading model")
+llm = HuggingFacePipeline.from_model_id(
+    model_id="google/flan-t5-base",
+    task="text2text-generation",
     model_kwargs={"temperature": 0, "max_length": 128})
 chain = load_qa_chain(llm, chain_type="stuff")
 

requirements.txt

@@ -1,3 +1,4 @@
 langchain
 faiss-cpu
 sentence_transformers
+protobuf==3.20.1