Storyteller-API / app.py
UH32's picture
Update app.py
81ec766 verified
Raw History Blame Contribute Delete
2.02 kB
import spaces
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
print("📥 Downloading model weights...")
model_path = hf_hub_download(
repo_id="UH32/Storyteller-Llama3-GGUF",
filename="llama-3.1-8b.Q4_K_M.gguf"
)
# Global variable to hold the model instance
llm = None
@spaces.GPU(duration=120)
def generate_urdu_story(prompt, system_instruction):
global llm
# 1. Initialize the model INSIDE the @spaces.GPU context when the GPU is actually assigned
if llm is None:
print("🧠 Loading Llama model into GPU memory...")
llm = Llama(
model_path=model_path,
n_ctx=2048,
n_threads=4,
n_gpu_layers=-1 # Offloads layers to the ZeroGPU instance
)
# 2. Format the prompt
formatted_prompt = (
f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
f"{system_instruction}<|eot_id|>"
f"<|start_header_id|>user<|end_header_id|>\n\n"
f"{prompt}<|eot_id|>"
f"<|start_header_id|>assistant<|end_header_id|>\n\n"
)
print("🎬 Generating response...")
response = llm(
formatted_prompt,
max_tokens=512,
temperature=0.7,
top_p=0.9,
stop=["<|eot_id|>", "<|end_of_text|>"]
)
return response["choices"][0]["text"]
default_system = "You are a professional Urdu creative writer. Write an engaging, immersive, and culturally rich story based on the user's prompt."
demo = gr.Interface(
fn=generate_urdu_story,
inputs=[
gr.Textbox(label="Story Prompt / کہانی کا عنوان", placeholder="ایک دفعہ کا ذکر ہے...", lines=3),
gr.Textbox(label="System Instructions / ہدایات", value=default_system, lines=2)
],
outputs=gr.Textbox(label="Generated Story / لکھی گئی کہانی", lines=12),
title="✍️ Urdu AI Storyteller Portal",
theme="soft"
)
if __name__ == "__main__":
demo.launch()