app.py

# ============================================
# 1️⃣ Libraries import koro
# ============================================
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# ============================================
# 2️⃣ HF merged model load koro
# ============================================
MODEL_NAME = "jsshanto001/llama3-merged-qLoRA"  # ekhane tumar repo name boshabe

tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    device_map="auto",  # GPU thakle use hobe
    torch_dtype=torch.float16
)

# ============================================
# 3️⃣ Chat function define koro
# ============================================
def chat_with_model(user_input, chat_history=[]):
    # User input add koro history te
    chat_history.append(f"You: {user_input}")
    
    # Input prepare koro
    input_text = "\n".join(chat_history) + tokenizer.eos_token
    input_ids = tokenizer.encode(input_text, return_tensors="pt").to(model.device)
    
    # Response generate koro
    output_ids = model.generate(
        input_ids,
        max_length=512,
        do_sample=True,
        temperature=0.7,
        top_p=0.9,
        pad_token_id=tokenizer.eos_token_id
    )
    response = tokenizer.decode(output_ids[0], skip_special_tokens=True)
    
    # Bot er reply extract koro
    bot_reply = response[len("\n".join(chat_history)):]
    chat_history.append(f"Bot: {bot_reply.strip()}")
    
    return "", chat_history  # input box clear koro

# ============================================
# 4️⃣ Gradio interface setup
# ============================================
with gr.Blocks() as demo:
    gr.Markdown("## 🤖 Chat with Your HF Merged Model")
    
    chatbot = gr.Chatbot()
    msg = gr.Textbox(label="Your prompt here...")
    clear = gr.Button("Clear Chat")
    
    msg.submit(chat_with_model, inputs=[msg, chatbot], outputs=[msg, chatbot])
    clear.click(lambda: None, None, chatbot, queue=False)

# ============================================
# 5️⃣ Launch
# ============================================
demo.launch()