Update app.py

app.py

@@ -3,6 +3,8 @@ from g4f import Provider, models
 from langchain.llms.base import LLM
 import asyncio
 import nest_asyncio
+from langchain.callbacks.manager import CallbackManager
+from langchain.llms import LlamaCpp
 from llama_index import ServiceContext, LLMPredictor, PromptHelper
 from llama_index.text_splitter import TokenTextSplitter
 from llama_index.node_parser import SimpleNodeParser
@@ -10,6 +12,16 @@ from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbed
 from llama_index import SimpleDirectoryReader, VectorStoreIndex
 from gradio import Interface
 nest_asyncio.apply()
+from huggingface_hub import hf_hub_download
+
+model_name_or_path = "hlhr202/llama-7B-ggml-int4"
+model_basename = "ggml-model-q4_0.bin" # the model is in bin format
+
+model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)
+
+n_gpu_layers = 40 # Change this value based on your model and your GPU VRAM pool.
+n_batch = 256 
+
 
 embed_model = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl",
                                                       model_kwargs={"device": "cpu"})
@@ -25,9 +37,8 @@ prompt_helper = PromptHelper(
 from langchain_g4f import G4FLLM
 
 async def main(question):
-    llm: LLM = G4FLLM(
-        model=models.gpt_35_turbo,
-        provider=Provider.DeepAi,
+    llm = LlamaCpp(
+    model_path=model_path, callbacks=[StreamingStdOutCallbackHandler()]
     )
     from llama_index.llms import LangChainLLM