Add Gradio interface for ZeroGPU testing

app/gradio_interface.py

@@ -0,0 +1,121 @@
+"""
+Gradio interface for ORA - Fast GPU testing
+This runs alongside the FastAPI server for quick testing with ZeroGPU
+"""
+
+import gradio as gr
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+import os
+
+# Use ZeroGPU decorator for free GPU access
+try:
+    import spaces
+    USE_GPU = True
+except ImportError:
+    USE_GPU = False
+    print("ZeroGPU not available, running on CPU")
+
+# Settings
+BASE_MODEL = "unsloth/Llama-3.2-1B-Instruct"
+ADAPTER_PATH = "important/finetuning/models/ora_adapter"
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Load model once
+print(f"Loading ORA Model on {device}...")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    torch_dtype=torch.float16 if device == "cuda" else torch.float32,
+    device_map=device,
+    low_cpu_mem_usage=True
+)
+
+if os.path.exists(ADAPTER_PATH):
+    print(f"Loading adapter from {ADAPTER_PATH}...")
+    model = PeftModel.from_pretrained(base_model, ADAPTER_PATH)
+else:
+    print("Adapter not found, using base model.")
+    model = base_model
+
+print("ORA Model Ready!")
+
+# Apply ZeroGPU decorator if available
+if USE_GPU:
+    @spaces.GPU
+    def generate_response(message, history):
+        return _generate_response(message, history)
+else:
+    def generate_response(message, history):
+        return _generate_response(message, history)
+
+def _generate_response(message, history):
+    """Generate ORA response"""
+    
+    # Build conversation history
+    messages = [{
+        "role": "system",
+        "content": """You are ORA, a wise and compassionate spiritual guide.
+
+Your role:
+- Provide biblically-grounded wisdom
+- Speak with warmth, empathy, and pastoral care
+- Keep responses concise but meaningful (2-3 sentences)
+- Always cite scripture when relevant
+
+Respond with compassion and wisdom."""
+    }]
+    
+    # Add conversation history
+    for user_msg, assistant_msg in history:
+        messages.append({"role": "user", "content": user_msg})
+        messages.append({"role": "assistant", "content": assistant_msg})
+    
+    messages.append({"role": "user", "content": message})
+    
+    # Generate response
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(device)
+    
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+    
+    outputs = model.generate(
+        input_ids,
+        max_new_tokens=128,
+        eos_token_id=terminators,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.9,
+    )
+    
+    response_tokens = outputs[0][input_ids.shape[-1]:]
+    response = tokenizer.decode(response_tokens, skip_special_tokens=True)
+    
+    return response
+
+# Create Gradio interface
+demo = gr.ChatInterface(
+    fn=generate_response,
+    title="🙏 ORA - Spiritual AI Assistant (GPU Testing)",
+    description="Fast GPU-powered testing interface for ORA. Ask theological questions, seek spiritual guidance, or discuss scripture.",
+    examples=[
+        "What does the Bible say about finding peace in difficult times?",
+        "How can I grow closer to God?",
+        "Explain the parable of the prodigal son",
+        "What is the meaning of faith?",
+    ],
+    theme=gr.themes.Soft(),
+    retry_btn=None,
+    undo_btn=None,
+    clear_btn="Clear",
+)
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860, share=False)

requirements.txt

@@ -1,8 +1,11 @@
+gradio>=5.9.1
+torch>=2.0.0
+transformers>=4.40.0
+peft>=0.10.0
+accelerate>=0.27.0
+huggingface-hub>=0.20.0
+spaces
+lancedb
 fastapi
 uvicorn
 pydantic
-pydantic-settings
-openai
-python-dotenv
-pytest
-pytest-asyncio