Update app.py

app.py

@@ -1,65 +1,54 @@
 import gradio as gr
-from transformers import pipeline
+from threading import Thread
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+import torch
 
-# Load open-source model for code generation (no token required)
-code_generator = pipeline("text-generation", model="bigcode/starcoder")
+# Load non-gated model and tokenizer
+model_id = "Qwen/Qwen2-7B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto")
 
-def generate_code_and_response(prompt):
-    # Generate code using the model
-    code_prompt = f"Generate Python code for: {prompt}\n```python\n"
-    code_output = code_generator(code_prompt, max_length=500, num_return_sequences=1, do_sample=True, temperature=0.7)[0]['generated_text']
+def chat(message, history):
+    # Build message history
+    messages = [
+        {"role": "system", "content": "You are a helpful and friendly assistant."}
+    ] + history + [{"role": "user", "content": message}]
     
-    # Extract the code part (between ```python and ```)
-    code_start = code_output.find("```python") + 9
-    code_end = code_output.find("```", code_start)
-    code = code_output[code_start:code_end].strip() if code_end != -1 else code_output.strip()
-
-    # Generate conversational response in Grok-like style
-    response_text = (
-        f"Alright, I’ve spun up some Python code for your request: '{prompt}'. Here it comes:\n\n"
-        f"```python\n{code}\n```\n\n"
-        f"Pretty cool, right? I’m running on an open-source model, zipping along on Hugging Face’s free tier. "
-        f"Want me to tweak this code, explain it, or whip up something else? I’m all cosmic curiosity!"
-    )
+    # Prepare inputs
+    inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").to(model.device)
+    
+    # Set up streamer for live typing
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    
+    # Generation kwargs
+    generation_kwargs = {
+        "inputs": inputs,
+        "streamer": streamer,
+        "max_new_tokens": 256,
+        "do_sample": True,
+        "top_p": 0.95,
+        "temperature": 0.7,
+    }
+    
+    # Run generation in a separate thread
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
     
-    # Escape special characters for JavaScript
-    escaped_response = response_text.replace("`", "\\`").replace("\n", "\\n")
+    # Yield tokens for live streaming
+    generated_text = ""
+    for new_text in streamer:
+        generated_text += new_text
+        yield generated_text
     
-    # Wrap response in HTML with JavaScript for typing effect
-    html_response = """
-    <div id="typing-output" style="font-family: Arial, sans-serif; font-size: 16px; white-space: pre-wrap;"></div>
-    <script>
-        const text = '""" + escaped_response + """';
-        let index = 0;
-        const speed = 30; // Typing speed in milliseconds
-        const outputElement = document.getElementById('typing-output');
-        
-        function typeWriter() {
-            if (index < text.length) {
-                outputElement.innerHTML += text.charAt(index);
-                index++;
-                setTimeout(typeWriter, speed);
-            }
-        }
-        typeWriter();
-    </script>
-    """
-    return html_response
+    thread.join()
 
-# Create Gradio interface
-demo = gr.Interface(
-    fn=generate_code_and_response,
-    inputs=gr.Textbox(label="Enter your coding request", placeholder="e.g., Create a function to calculate factorial"),
-    outputs=gr.HTML(label="Generated Code and Response"),
-    title="Live CodeGen Bot",
-    description="Generate Python code using open-source models with a Grok-like conversational style and live typing effect. Hosted on Hugging Face Spaces free tier.",
-    css="""
-        body { background-color: #f0f2f5; }
-        .gr-box { border-radius: 8px; padding: 20px; }
-        #typing-output { background: #fff; padding: 15px; border: 1px solid #ddd; border-radius: 5px; }
-    """
+# Create Gradio chat interface
+demo = gr.ChatInterface(
+    fn=chat,
+    type="messages",
+    title="Qwen2-7B Chatbot",
+    description="Chat with a non-gated Qwen2-7B-Instruct model. Responses stream live.",
+    examples=[["Tell me a fun fact."], ["Explain neural networks in simple terms."]],
 )
 
-# Launch the app
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()