use automodel instead

app.py

@@ -1,20 +1,18 @@
 import gradio as gr
-from llama_cpp import Llama
-from huggingface_hub import hf_hub_download
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from huggingface_hub import login
 
-# Load your local .gguf model
-repo_id = "Selinaliu1030/lora_model"
-filename = "llama-3.2-3b-finetuned-q8_0.gguf"
+# Hugging Face model repo ID (must contain HF model weights, NOT .gguf)
+MODEL_ID = "Selinaliu1030/lora_model"
 
-model_path = hf_hub_download(
-    repo_id=repo_id,
-    filename=filename,
-    local_dir=".",            # where to download in the Space
-)
-llm = Llama(
-    model_path=model_path,  # <-- modify path if needed
-    n_ctx=4096,
-    n_threads=4,
+# Load tokenizer + model
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    device_map="auto",            # uses GPU if available
+    torch_dtype="auto",           # automatically picks fp16/bf16
+    low_cpu_mem_usage=True,
 )
 
 def respond(
@@ -24,35 +22,37 @@ def respond(
     max_tokens,
     temperature,
     top_p,
-    hf_token,  # unused now, but keep for interface compatibility
+    hf_token,   # still required by UI signature; unused
 ):
-    # Combine system + history + new user message
+    # Build prompt
     messages = [{"role": "system", "content": system_message}]
     messages.extend(history)
     messages.append({"role": "user", "content": message})
 
-    # Convert to llama.cpp style input
     prompt = ""
     for msg in messages:
-        role = msg["role"]
-        content = msg["content"]
-        prompt += f"<{role}>: {content}\n"
+        prompt += f"<{msg['role']}>: {msg['content']}\n"
     prompt += "<assistant>: "
 
-    # Stream tokens
-    stream = llm(
-        prompt,
-        max_tokens=max_tokens,
+    # Tokenize
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+    # Generate
+    output = model.generate(
+        **inputs,
+        max_new_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
-        stream=True,
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id,
     )
 
-    response = ""
-    for chunk in stream:
-        token = chunk["choices"][0]["text"]
-        response += token
-        yield response
+    # Decode
+    result = tokenizer.decode(output[0], skip_special_tokens=True)
+    # Extract only the assistant's response
+    assistant_reply = result.split("<assistant>:")[-1].strip()
+
+    yield assistant_reply
 
 
 # Gradio UI
@@ -61,8 +61,8 @@ chatbot = gr.ChatInterface(
     type="messages",
     additional_inputs=[
         gr.Textbox(value="You are a helpful assistant.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=1, maximum=2048, value=256, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,
             maximum=1.0,

requirements.txt

@@ -1,3 +1,2 @@
-llama-cpp-python==0.2.79
 huggingface_hub
 gradio