Update app.py

app.py

@@ -1,40 +1,27 @@
 import os
 import torch
 import streamlit as st
-from transformers import (
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    BitsAndBytesConfig
-)
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from peft import PeftModel
 
-# Offload directory for CPU inference
-os.makedirs("offload", exist_ok=True)
+# Set CPU device
+device = torch.device("cpu")
 
-# Load base model + quantization config
+# Load base model and tokenizer
 model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-bnb_config = BitsAndBytesConfig(
-    load_in_8bit=True,
-    llm_int8_threshold=6.0,
-    llm_int8_enable_fp32_cpu_offload=True
-)
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
 base_model = AutoModelForCausalLM.from_pretrained(
     model_name,
-    quantization_config=bnb_config,
-    device_map="auto",
-    offload_folder="offload"
+    torch_dtype=torch.float32,  # You can try float16 if supported
+    device_map={"": device}
 )
 
-# Load LoRA adapter
-model = PeftModel.from_pretrained(base_model, "lora_adapter")
-
-# Evaluation mode
+# Load LoRA adapter (your fine-tuned weights)
+model = PeftModel.from_pretrained(base_model, "lora_adapter", device_map={"": device})
 model.eval()
 
-# Prompt template
+# Format the prompt with filtering
 def format_prompt(instruction):
     return f"""You are a helpful and expert Python programming tutor.
 Only answer questions related to Python programming.
@@ -47,17 +34,17 @@ If the question is unrelated to Python, respond with:
 ### Response:
 """
 
-# Chat function
+# Generate answer
 def chat(instruction):
     prompt = format_prompt(instruction)
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
     outputs = model.generate(
         **inputs,
         max_new_tokens=256,
         do_sample=True,
         temperature=0.7,
-        top_p=0.95,
-        repetition_penalty=1.2
+        top_p=0.9,
+        repetition_penalty=1.1
     )
     response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response.split("### Response:")[-1].strip()
@@ -66,13 +53,13 @@ def chat(instruction):
 st.title("🐍 Python Tutor Chatbot")
 st.markdown("Ask me Python programming questions!")
 
-user_input = st.text_area("Your question:")
+question = st.text_area("Your question:")
 
 if st.button("Answer"):
-    if user_input.strip():
+    if question.strip():
         with st.spinner("Thinking..."):
-            answer = chat(user_input)
+            response = chat(question)
         st.markdown("**Answer:**")
-        st.write(answer)
+        st.write(response)
     else:
         st.warning("Please enter a question.")