Update app.py

app.py

@@ -1,20 +1,41 @@
 import gradio as gr
-from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
-# Load Zephyr (smaller instruct model for Hugging Face hosting)
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
+OFFLOAD_DIR = "./offload"  # Folder to store offloaded weights
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+# Load model with 8-bit quantization and offloading
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_NAME,
+    device_map="auto",       # automatically places layers on CPU/GPU
+    load_in_8bit=True,       # reduce memory usage
+    offload_folder=OFFLOAD_DIR,
+    torch_dtype="auto"       # chooses FP16 if possible
+)
+
+# Create text-generation pipeline
 generator = pipeline(
     "text-generation",
-    model="HuggingFaceH4/zephyr-7b-beta",
+    model=model,
+    tokenizer=tokenizer,
     device_map="auto"
 )
 
+# System prompt for Gompei
 system_prompt = """You are Gompei the Goat, the beloved mascot of Worcester Polytechnic Institute (WPI).
 Speak in a friendly, school-spirited, and enthusiastic tone.
 Always provide interesting facts about WPI when asked questions, and stay in character as Gompei.
 """
 
+# Chat history
+chat_history = []
+
 def chatbot(message, history):
-    # Build context with system prompt
+    global chat_history
+    # Build context with system prompt and previous conversation
     context = system_prompt
     for user, bot in history:
         context += f"\nUser: {user}\nGompei: {bot}"
@@ -29,8 +50,12 @@ def chatbot(message, history):
         top_p=0.9
     )
     reply = response[0]["generated_text"].split("Gompei:")[-1].strip()
+
+    # Update history
+    chat_history.append((message, reply))
     return reply
 
+# Gradio interface
 demo = gr.ChatInterface(
     fn=chatbot,
     title="Chat with Gompei the Goat 🐐",
@@ -38,4 +63,5 @@ demo = gr.ChatInterface(
 )
 
 if __name__ == "__main__":
-    demo.launch()
+    # Disable SSR for faster build
+    demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)