UH32 commited on
Commit
e636ab5
·
verified ·
1 Parent(s): 537ea1c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +39 -42
app.py CHANGED
@@ -1,66 +1,63 @@
 
 
1
  import gradio as gr
2
  from huggingface_hub import hf_hub_download
3
  from llama_cpp import Llama
4
 
5
- # # 1. Download the high-performance CPU file from your new repository
6
- # print("📥 Fetching optimized GGUF weights...")
7
- # model_path = hf_hub_download(
8
- # repo_id="UH32/Storyteller-Llama3-GGUF",
9
- # filename="unsloth.Q4_K_M.gguf" # This is the standard output name generated by Unsloth
10
- # )
11
-
12
- # 1. Download the high-performance CPU file from your new repository
13
  print("📥 Fetching optimized GGUF weights...")
14
  model_path = hf_hub_download(
15
  repo_id="UH32/Storyteller-Llama3-GGUF",
16
- filename="llama-3.1-8b.Q4_K_M.gguf" # <-- CHANGED THIS LINE
17
  )
18
 
19
- # 2. Initialize the model inside the CPU memory footprint
20
- print("🧠 Initializing Llama Engine on CPU...")
21
  llm = Llama(
22
  model_path=model_path,
23
- n_ctx=2048, # Context window length
24
- n_threads=2, # Optimized for free Hugging Face CPU cores
 
25
  )
26
 
27
- alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
28
-
29
- ### Instruction:
30
- {}
31
-
32
- ### Input:
33
- {}
34
-
35
- ### Response:
36
- """
37
-
38
- def predict(instruction, manual_input):
39
- # Format according to your Alpaca training structure
40
- formatted_prompt = alpaca_prompt.format(instruction, manual_input)
41
 
42
- print("⚡ Generating story...")
43
- output = llm(
 
 
44
  formatted_prompt,
45
- max_tokens=256,
46
- temperature=0.68,
47
- repeat_penalty=1.12,
48
- stop=["<|end_of_text|>", "### Instruction:"] # Prevents the model from rambling
49
  )
50
 
51
- # Extract text cleanly from the GGUF response structure
52
- generated_text = output["choices"][0]["text"]
53
- return generated_text.strip()
 
54
 
55
- # Create clean UI
56
  demo = gr.Interface(
57
- fn=predict,
58
  inputs=[
59
- gr.Textbox(label="Instruction", value="ایک مختصر کہانی لکھیں۔"),
60
- gr.Textbox(label="Input Context", value="چاند پر ایک عجیب خط ملا۔")
61
  ],
62
- outputs=gr.Textbox(label="Generated Output"),
63
- title="Urdu Storyteller API (CPU-Optimized GGUF)"
 
 
64
  )
65
 
66
  if __name__ == "__main__":
 
1
+ import os
2
+ import spaces # <--- Crucial Hugging Face ZeroGPU management library
3
  import gradio as gr
4
  from huggingface_hub import hf_hub_download
5
  from llama_cpp import Llama
6
 
7
+ # 1. Download the correct GGUF weights safely from your repository
 
 
 
 
 
 
 
8
  print("📥 Fetching optimized GGUF weights...")
9
  model_path = hf_hub_download(
10
  repo_id="UH32/Storyteller-Llama3-GGUF",
11
+ filename="llama-3.1-8b.Q4_K_M.gguf"
12
  )
13
 
14
+ # 2. Initialize the Llama engine with GPU support active
15
+ print("🧠 Initializing Llama Engine with GPU layers...")
16
  llm = Llama(
17
  model_path=model_path,
18
+ n_ctx=2048, # Sequence context window
19
+ n_threads=4, # CPU fallback thread configuration
20
+ n_gpu_layers=-1 # -1 means offload ALL model layers to the GPU
21
  )
22
 
23
+ # 3. Use the @spaces.GPU decorator so Hugging Face detects it on startup
24
+ @spaces.GPU(duration=90)
25
+ def generate_urdu_story(prompt, system_instruction):
26
+ # Constructing a structured prompt layout for Llama 3 style chats
27
+ formatted_prompt = (
28
+ f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
29
+ f"{system_instruction}<|eot_id|>"
30
+ f"<|start_header_id|>user<|end_header_id|>\n\n"
31
+ f"{prompt}<|eot_id|>"
32
+ f"<|start_header_id|>assistant<|end_header_id|>\n\n"
33
+ )
 
 
 
34
 
35
+ print(f"🎬 Running inference on GPU for prompt: {prompt[:30]}...")
36
+
37
+ # Run text generation
38
+ response = llm(
39
  formatted_prompt,
40
+ max_tokens=512,
41
+ temperature=0.7,
42
+ top_p=0.9,
43
+ stop=["<|eot_id|>", "<|end_of_text|>"]
44
  )
45
 
46
+ return response["choices"][0]["text"]
47
+
48
+ # 4. Build a sleek Gradio Interface for the user
49
+ default_system = "You are a professional Urdu creative writer. Write an engaging, immersive, and culturally rich story based on the user's prompt."
50
 
 
51
  demo = gr.Interface(
52
+ fn=generate_urdu_story,
53
  inputs=[
54
+ gr.Textbox(label="Story Prompt / کہانی کا عنوان", placeholder="ایک دفعہ کا ذکر ہے کہ ایک جادوئی جنگل میں...", lines=3),
55
+ gr.Textbox(label="System Instructions / ہدایات", value=default_system, lines=2)
56
  ],
57
+ outputs=gr.Textbox(label="Generated Story / لکھی گئی کہانی", lines=12),
58
+ title="✍️ Urdu AI Storyteller Portal",
59
+ description="Provide a prompt concept or beginning to generate beautiful stories using your fine-tuned Llama-3 model running directly on Hugging Face ZeroGPU.",
60
+ theme="soft"
61
  )
62
 
63
  if __name__ == "__main__":