Create app.py

app.py

@@ -0,0 +1,61 @@
+import gradio as gr
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
+
+model_id = "deepgrove/maple-preview"
+
+print("🔄 Loading tokenizer...")
+tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+
+print("🔄 Loading Maple-Preview (this may take a few minutes to download weights and compile kernels)...")
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    device_map="auto",
+    torch_dtype=torch.bfloat16,
+    trust_remote_code=True
+)
+
+def predict(message, history):
+    # Gradio type="messages" format: [{"role": "user", "content": "..."}, ...]
+    messages = history + [{"role": "user", "content": message}]
+    
+    # Apply the custom Jinja chat template included in the repo
+    prompt = tokenizer.apply_chat_template(
+        messages, 
+        tokenize=False, 
+        add_generation_prompt=True
+    )
+    
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    
+    generation_kwargs = dict(
+        **inputs,
+        streamer=streamer,
+        max_new_tokens=2048,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.9,
+    )
+    
+    # Generate in a separate thread to allow streaming to the UI
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+    
+    generated_text = ""
+    for new_text in streamer:
+        generated_text += new_text
+        yield generated_text
+
+demo = gr.ChatInterface(
+    fn=predict,
+    type="messages",
+    title="🍁 Maple-Preview",
+    description="A 20B-A1B ternary-weight reasoning LLM by DeepGrove. Powered by ZeroGPU.",
+    theme="soft",
+)
+
+if __name__ == "__main__":
+    demo.queue().launch()