Update app.py

app.py

@@ -1,8 +1,45 @@
 
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
+from llama_index.llms.huggingface import HuggingFaceLLM
+import torch
+import gradio as gr
+from llama_index.core import Settings
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from llama_index.core import ServiceContext
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
 
+documents = SimpleDirectoryReader('/files').load_data()
 
+system_prompt="""
+You are a Q&A assistant. Your goal is to answer questions as
+accurately as possible based on the instructions and context provided.
+"""
 
+llm = HuggingFaceLLM(
+    context_window=4096,
+    max_new_tokens=256,
+    generate_kwargs={"temperature": 0.1, "do_sample": True},
+    system_prompt=system_prompt,
+    tokenizer_name="anasmkh/customized_llama3.1_8b",
+    model_name="anasmkh/customized_llama3.1_8b",
+    device_map="auto",
+    model_kwargs={"torch_dtype": torch.float16 }
+)
 
+embed_model= HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+
+
+Settings.llm = llm
+Settings.embed_model =embed_model
+# Settings.node_parser = SentenceSplitter(chunk_size=512, chunk_overlap=20)
+Settings.num_output = 250
+Settings.context_window = 3900
+
+index = VectorStoreIndex.from_documents(
+    documents, embed_model=embed_model
+)
+
+query_engine = index.as_query_engine(llm=llm)
 
 
 
@@ -10,7 +47,8 @@
 def chat(message, history):
     history = history or []
     history.append({"role": "user", "content": message})
-    response = generator(history)[-1]["generated_text"]
+    response=query_engine.query(message)
+    # response = generator(history)[-1]["generated_text"]
     history.append({"role": "assistant", "content": response})
     return history