Update app.py

CPU-compatible version of Mistral 7B

app.py

@@ -44,33 +44,30 @@ warnings.filterwarnings('ignore')
 #     return watsonx_llm
 
 def get_llm():
-    model_id = 'ibm-granite/granite-3.0-8b-instruct'
-    
-    # Configuration for 4-bit compression
-    quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.float16,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=True,
-    )
-    
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_id,
-        quantization_config=quantization_config,
-        device_map='cpu',
-        torch_dtype='float32',
+    mglobal LLM_INSTANCE
+    if LLM_INSTANCE is not None:
+        return LLM_INSTANCE
+
+    # Download quantized GGUF model (works on CPU)
+    model_path = hf_hub_download(
+        repo_id="TheBloke/Mistral-7B-Instruct-v0.3-GGUF",
+        filename="mistral-7b-instruct-v0.3.Q4_K_M.gguf",
+        local_dir=".",
+        local_dir_use_symlinks=False,
     )
 
-    pipe = pipeline(
-        model=model,
-        tokenizer=tokenizer,
-        max_new_tokens=256,
-        temperature= 0.3,
-        do_sample= True
+    llm = LlamaCpp(
+        model_path=model_path,
+        n_ctx=4096,
+        n_threads=4,
+        temperature=0.3,
+        top_p=0.9,
+        verbose=False,
     )
-    
-    return HuggingFacePipeline(pipeline=pipe)
+
+    LLM_INSTANCE = llm
+    return llm
+
 
 ## Document loader
 def document_loader(file):
@@ -156,4 +153,4 @@ rag_application = gr.Interface(
     description = 'Upload a PDF document and ask any question. The chatbot will try to answer using the provided document.'
 )
 # Launch the app
-rag_application.launch(server_name= '0.0.0.0', server_port= 7860)
+rag_application.launch()