Upload folder using huggingface_hub

README.md

@@ -4,7 +4,7 @@ emoji: 🤖
 colorFrom: blue
 colorTo: purple
 sdk: gradio
-sdk_version: 4.44.1
+sdk_version: 5.23.3
 app_file: app.py
 pinned: true
 ---

app.py

@@ -1,24 +1,41 @@
-import torch
+import os
 import threading
+import torch
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
 MODEL_ID = "Splashdude/reasoning-chat-model-7b"
 SYSTEM_PROMPT = "You are a helpful, friendly AI assistant. You give clear, accurate, and concise answers."
 
-print("Loading model and tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    torch_dtype=torch.float16,
-    device_map="auto",
-    trust_remote_code=True,
-)
-model.eval()
-print("Model loaded successfully!")
+model = None
+tokenizer = None
+
+
+def load_model():
+    global model, tokenizer
+    if model is not None:
+        return
+    print("Loading model and tokenizer...")
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        torch_dtype=torch.float16,
+        device_map="auto",
+        trust_remote_code=True,
+        low_cpu_mem_usage=True,
+    )
+    model.eval()
+    print("Model loaded successfully!")
 
 
 def generate_response(message, history):
+    if model is None or tokenizer is None:
+        try:
+            load_model()
+        except Exception as e:
+            yield f"Error loading model: {e}"
+            return
+
     messages = [{"role": "system", "content": SYSTEM_PROMPT}]
     for user_msg, bot_msg in history:
         messages.append({"role": "user", "content": user_msg})
@@ -31,7 +48,9 @@ def generate_response(message, history):
     )
     inputs = tokenizer(text, return_tensors="pt").to(model.device)
 
-    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(
+        tokenizer, skip_prompt=True, skip_special_tokens=True
+    )
 
     generation_kwargs = {
         **inputs,