import spaces import gradio as gr from huggingface_hub import hf_hub_download from llama_cpp import Llama print("📥 Downloading model weights...") model_path = hf_hub_download( repo_id="UH32/Storyteller-Llama3-GGUF", filename="llama-3.1-8b.Q4_K_M.gguf" ) # Global variable to hold the model instance llm = None @spaces.GPU(duration=120) def generate_urdu_story(prompt, system_instruction): global llm # 1. Initialize the model INSIDE the @spaces.GPU context when the GPU is actually assigned if llm is None: print("🧠 Loading Llama model into GPU memory...") llm = Llama( model_path=model_path, n_ctx=2048, n_threads=4, n_gpu_layers=-1 # Offloads layers to the ZeroGPU instance ) # 2. Format the prompt formatted_prompt = ( f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" f"{system_instruction}<|eot_id|>" f"<|start_header_id|>user<|end_header_id|>\n\n" f"{prompt}<|eot_id|>" f"<|start_header_id|>assistant<|end_header_id|>\n\n" ) print("🎬 Generating response...") response = llm( formatted_prompt, max_tokens=512, temperature=0.7, top_p=0.9, stop=["<|eot_id|>", "<|end_of_text|>"] ) return response["choices"][0]["text"] default_system = "You are a professional Urdu creative writer. Write an engaging, immersive, and culturally rich story based on the user's prompt." demo = gr.Interface( fn=generate_urdu_story, inputs=[ gr.Textbox(label="Story Prompt / کہانی کا عنوان", placeholder="ایک دفعہ کا ذکر ہے...", lines=3), gr.Textbox(label="System Instructions / ہدایات", value=default_system, lines=2) ], outputs=gr.Textbox(label="Generated Story / لکھی گئی کہانی", lines=12), title="✍️ Urdu AI Storyteller Portal", theme="soft" ) if __name__ == "__main__": demo.launch()