UH32 commited on
Commit
adb2197
·
verified ·
1 Parent(s): 195bbd6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +32 -60
app.py CHANGED
@@ -1,25 +1,20 @@
1
  import gradio as gr
2
- import torch
3
- import httpx
4
- from bs4 import BeautifulSoup
5
- from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
6
-
7
- # Load model using free CPU/Low memory setup
8
- HF_REPO = "UH32/Storyteller-Llama3"
9
-
10
- quantization_config = BitsAndBytesConfig(
11
- load_in_4bit=True,
12
- bnb_4bit_quant_type="nf4",
13
- bnb_4bit_compute_dtype=torch.float16,
14
- bnb_4bit_use_double_quant=True
15
  )
16
 
17
- print("Loading model weights safely...")
18
- tokenizer = AutoTokenizer.from_pretrained(HF_REPO)
19
- model = AutoModelForCausalLM.from_pretrained(
20
- HF_REPO,
21
- quantization_config=quantization_config,
22
- device_map="auto"
23
  )
24
 
25
  alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
@@ -33,55 +28,32 @@ alpaca_prompt = """Below is an instruction that describes a task, paired with an
33
  ### Response:
34
  """
35
 
36
- def fetch_live_context(url):
37
- if not url or not url.startswith("http"):
38
- return "تازہ ترین علاقائی اور عالمی حالات کا پس منظر۔"
39
- try:
40
- headers = {"User-Agent": "Mozilla/5.0"}
41
- with httpx.Client(headers=headers, timeout=5.0) as client:
42
- res = client.get(url)
43
- soup = BeautifulSoup(res.text, 'html.parser')
44
- for e in soup(["script", "style", "nav", "footer", "header"]): e.decompose()
45
- raw_text = " ".join([p.get_text().strip() for p in soup.find_all('p') if len(p.get_text().strip()) > 30])
46
- return raw_text[:800] + "..."
47
- except:
48
- return "تازہ ترین بین الاقوامی خبریں اور افسانوی پس منظر۔"
49
-
50
- def predict(instruction, live_url, manual_input):
51
- # Determine context
52
- context = fetch_live_context(live_url) if live_url else manual_input
53
-
54
- # Intercept Roman Urdu
55
- is_urdu = any(w in instruction.lower() for w in ["urdu", "kahani", "likho", "likhein"]) or any(char >= '\u0600' and char <= '\u06FF' for char in instruction)
56
- if is_urdu and not any(char >= '\u0600' and char <= '\u06FF' for char in instruction):
57
- instruction = "ایک خوبصورت اور تفصیلی کہانی اردو رسم الخط میں لکھیں۔"
58
-
59
- formatted_input = alpaca_prompt.format(instruction, context)
60
- inputs = tokenizer([formatted_input], return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
61
 
62
- outputs = model.generate(
63
- **inputs,
64
- max_new_tokens=512,
65
- temperature=0.68 if is_urdu else 0.75,
66
- repetition_penalty=1.12 if is_urdu else 1.18,
67
- do_sample=True
 
68
  )
69
 
70
- decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
71
- if "### Response:" in decoded:
72
- return decoded.split("### Response:")[-1].strip()
73
- return decoded.strip()
74
 
75
- # Create intuitive interface
76
  demo = gr.Interface(
77
  fn=predict,
78
  inputs=[
79
- gr.Textbox(label="Instruction (English, Urdu, or Roman Urdu)", value="ایک مختصر کہانی لکھیں۔"),
80
- gr.Textbox(label="Optional Real-time Data URL (e.g., BBC Urdu)", value=""),
81
- gr.Textbox(label="Manual Context Input (If URL is blank)", value="چاند پر ایک عجیب خط ملا۔")
82
  ],
83
- outputs=gr.Textbox(label="Generated Story"),
84
- title="Urdu Storyteller API & UI Dashboard"
85
  )
86
 
87
  if __name__ == "__main__":
 
1
  import gradio as gr
2
+ from huggingface_hub import hf_hub_download
3
+ from llama_cpp import Llama
4
+
5
+ # 1. Download the high-performance CPU file from your new repository
6
+ print("📥 Fetching optimized GGUF weights...")
7
+ model_path = hf_hub_download(
8
+ repo_id="UH32/Storyteller-Llama3-GGUF",
9
+ filename="unsloth.Q4_K_M.gguf" # This is the standard output name generated by Unsloth
 
 
 
 
 
10
  )
11
 
12
+ # 2. Initialize the model inside the CPU memory footprint
13
+ print("🧠 Initializing Llama Engine on CPU...")
14
+ llm = Llama(
15
+ model_path=model_path,
16
+ n_ctx=2048, # Context window length
17
+ n_threads=2, # Optimized for free Hugging Face CPU cores
18
  )
19
 
20
  alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
 
28
  ### Response:
29
  """
30
 
31
+ def predict(instruction, manual_input):
32
+ # Format according to your Alpaca training structure
33
+ formatted_prompt = alpaca_prompt.format(instruction, manual_input)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
 
35
+ print("⚡ Generating story...")
36
+ output = llm(
37
+ formatted_prompt,
38
+ max_tokens=256,
39
+ temperature=0.68,
40
+ repeat_penalty=1.12,
41
+ stop=["<|end_of_text|>", "### Instruction:"] # Prevents the model from rambling
42
  )
43
 
44
+ # Extract text cleanly from the GGUF response structure
45
+ generated_text = output["choices"][0]["text"]
46
+ return generated_text.strip()
 
47
 
48
+ # Create clean UI
49
  demo = gr.Interface(
50
  fn=predict,
51
  inputs=[
52
+ gr.Textbox(label="Instruction", value="ایک مختصر کہانی لکھیں۔"),
53
+ gr.Textbox(label="Input Context", value="چاند پر ایک عجیب خط ملا۔")
 
54
  ],
55
+ outputs=gr.Textbox(label="Generated Output"),
56
+ title="Urdu Storyteller API (CPU-Optimized GGUF)"
57
  )
58
 
59
  if __name__ == "__main__":