File size: 2,015 Bytes
81ec766
7efd3e9
adb2197
 
 
81ec766
adb2197
 
e636ab5
7efd3e9
 
81ec766
 
7efd3e9
81ec766
e636ab5
81ec766
 
 
 
 
 
 
 
 
 
 
 
 
e636ab5
 
 
 
 
 
 
7efd3e9
81ec766
e636ab5
adb2197
e636ab5
 
 
 
7efd3e9
 
e636ab5
 
 
7efd3e9
 
e636ab5
7efd3e9
81ec766
e636ab5
7efd3e9
e636ab5
 
 
7efd3e9
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
import spaces
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama

print("📥 Downloading model weights...")
model_path = hf_hub_download(
    repo_id="UH32/Storyteller-Llama3-GGUF",
    filename="llama-3.1-8b.Q4_K_M.gguf"
)

# Global variable to hold the model instance
llm = None

@spaces.GPU(duration=120)
def generate_urdu_story(prompt, system_instruction):
    global llm
    
    # 1. Initialize the model INSIDE the @spaces.GPU context when the GPU is actually assigned
    if llm is None:
        print("🧠 Loading Llama model into GPU memory...")
        llm = Llama(
            model_path=model_path,
            n_ctx=2048,
            n_threads=4,
            n_gpu_layers=-1  # Offloads layers to the ZeroGPU instance
        )
    
    # 2. Format the prompt
    formatted_prompt = (
        f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
        f"{system_instruction}<|eot_id|>"
        f"<|start_header_id|>user<|end_header_id|>\n\n"
        f"{prompt}<|eot_id|>"
        f"<|start_header_id|>assistant<|end_header_id|>\n\n"
    )
    
    print("🎬 Generating response...")
    response = llm(
        formatted_prompt,
        max_tokens=512,
        temperature=0.7,
        top_p=0.9,
        stop=["<|eot_id|>", "<|end_of_text|>"]
    )
    
    return response["choices"][0]["text"]

default_system = "You are a professional Urdu creative writer. Write an engaging, immersive, and culturally rich story based on the user's prompt."

demo = gr.Interface(
    fn=generate_urdu_story,
    inputs=[
        gr.Textbox(label="Story Prompt / کہانی کا عنوان", placeholder="ایک دفعہ کا ذکر ہے...", lines=3),
        gr.Textbox(label="System Instructions / ہدایات", value=default_system, lines=2)
    ],
    outputs=gr.Textbox(label="Generated Story / لکھی گئی کہانی", lines=12),
    title="✍️ Urdu AI Storyteller Portal",
    theme="soft"
)

if __name__ == "__main__":
    demo.launch()