Update app.py

app.py

@@ -1,62 +1,39 @@
 import gradio as gr
 
-from unsloth import FastLanguageModel
-import torch
-max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
-dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
-load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+from langchain_community.llms import LlamaCpp
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+from langchain_core.callbacks import StreamingStdOutCallbackHandler
 
 
+callbacks = [StreamingStdOutCallbackHandler()]
 
-model, tokenizer = FastLanguageModel.from_pretrained(
-    model_name = "unsloth/Meta-Llama-3.1-8B",
-    max_seq_length = max_seq_length,
-    dtype = dtype,
-    load_in_4bit = load_in_4bit,
-    # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
+llm = LlamaCpp(
+    model_path="/content/drive/MyDrive/models/demo1/unsloth.Q5_K_M.gguf",
+    n_gpu_layers=40,
+    n_batch=512,
+    callbacks=callbacks,
+    verbose=True,
 )
 
-model = FastLanguageModel.get_peft_model(
-    model,
-    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
-                      "gate_proj", "up_proj", "down_proj",],
-    lora_alpha = 16,
-    lora_dropout = 0, # Supports any, but = 0 is optimized
-    bias = "none",    # Supports any, but = "none" is optimized
-    # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
-    use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
-    random_state = 3407,
-    use_rslora = False,  # We support rank stabilized LoRA
-    loftq_config = None, # And LoftQ
-)
-
-alpaca_prompt = """You are the Finiantial expert: 
+template = """You are the Finiantial expert:
 
 ### Instruction:
-{}
+{question}
 
 ### Input:
-{}
+
 
 ### Response:
-{}"""
-
-
-def greet(name):
-    # alpaca_prompt = Copied from above
-    FastLanguageModel.for_inference(model) # Enable native 2x faster inference
-    inputs = tokenizer(
-    [
-        alpaca_prompt.format(
-            f"{name}", # instruction
-            "", # input
-            "", # output - leave this blank for generation!
-        )
-    ], return_tensors = "pt").to("cuda")
-    
-    outputs = model.generate(**inputs, max_new_tokens = 128, use_cache = True)
-    out_gen = tokenizer.batch_decode(outputs)
+"""
+
+prompt = PromptTemplate(template=template, input_variables=["question"])
+
+llm_chain_model = LLMChain(prompt=prompt, llm=llm)
+
+
+def greet(question):
+    out_gen = llm_chain_model.run(question)    
     return out_gen
 
 demo = gr.Interface(fn=greet, inputs="text", outputs="text")