Update app.py

app.py

@@ -8,17 +8,13 @@ from langchain_community.vectorstores import FAISS
 from langchain_core.prompts import PromptTemplate
 from langchain_community.document_loaders import PDFMinerLoader, CSVLoader, JSONLoader
 from langchain.text_splitter import SentenceTransformersTokenTextSplitter
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-from transformers import AutoConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, pipeline
 
-MODEL_NAME = "TheBloke/Llama-2-13B-chat-GPTQ"
+MODEL_NAME = "TheBloke/Llama-2-7B-GPTQ"
 
+# Initialize tokenizer and model
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-
-config = AutoConfig.from_pretrained(MODEL_NAME)
-config.quantization_config.disable_exllama = True
-
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, config=config, device_map="cpu")
+model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="cpu")
 
 text_pipeline = pipeline(
     "text-generation",
@@ -26,6 +22,7 @@ text_pipeline = pipeline(
     tokenizer=tokenizer
 )
 
+# Define prompt template
 template = """
 <s>[INST] <<SYS>>
 Use the following information to answer the question at the end.
@@ -75,7 +72,7 @@ async def process_files(file_paths):
 
         embeddings = HuggingFaceEmbeddings(
             model_name="thenlper/gte-large",
-            model_kwargs={"device": "cpu"},  # Use CPU instead of CUDA
+            model_kwargs={"device": "cpu"},
             encode_kwargs={"normalize_embeddings": True},
         )
 
@@ -110,6 +107,9 @@ async def query_files(files, question):
 
     return generated_text
 
+def process_and_query(files, question):
+    return asyncio.run(query_files(files, question))
+
 with gr.Blocks() as interface:
     gr.Markdown("### Retrieval Augmented Generation (RAG) for LLM Local Trial")
     gr.Markdown(
@@ -121,8 +121,7 @@ with gr.Blocks() as interface:
 
     submit_button = gr.Button("Submit")
     output_text = gr.Textbox(label="LLM Response", lines=8)
-    submit_button.click(lambda files, q: asyncio.run(query_files(files, q)), inputs=[files_input, question_input],
-                        outputs=output_text)
+    submit_button.click(process_and_query, inputs=[files_input, question_input], outputs=output_text)
 
 if __name__ == "__main__":
     interface.launch()