identityvoid commited on
Commit
fc7e212
·
1 Parent(s): 3fd1be3

Convert model app to Gradio ZeroGPU

Browse files
Files changed (1) hide show
  1. app.py.cuda-original +98 -560
app.py.cuda-original CHANGED
@@ -1,592 +1,130 @@
1
- """
2
- Run qwen 7b chat.
3
-
4
- transformers 4.31.0
5
-
6
- import torch
7
- torch.cuda.empty_cache()
8
-
9
- model.chat(
10
- tokenizer: transformers.tokenization_utils.PreTrainedTokenizer,
11
- query: str,
12
- history: Optional[List[Tuple[str, str]]],
13
- system: str = 'You are a helpful assistant.',
14
- append_history: bool = True,
15
- stream: Optional[bool] = <object object at 0x7f905797ec20>,
16
- stop_words_ids: Optional[List[List[int]]] = None,
17
- **kwargs) -> Tuple[str, List[Tuple[str, str]]]
18
- )
19
-
20
- model.generation_config
21
- GenerationConfig {
22
- "chat_format": "chatml",
23
- "do_sample": true,
24
- "eos_token_id": 151643,
25
- "max_new_tokens": 512,
26
- "max_window_size": 6144,
27
- "pad_token_id": 151643,
28
- "top_k": 0,
29
- "top_p": 0.5,
30
- "transformers_version": "4.31.0",
31
- "trust_remote_code": true
32
- }
33
- """
34
- # pylint: disable=line-too-long, invalid-name, no-member, redefined-outer-name, missing-function-docstring, missing-class-docstring, broad-except,
35
- from run_cmd import run_cmd # noqa
36
-
37
- # autodl with cuda12 NVIDIA-SMI appears
38
- # 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0
39
- # no fix needed
40
-
41
- # clumsy fix for hf overwrite libbitsandbytes_cpu.so with libbitsandbytes_cuda118.so
42
- run_cmd(
43
- "cd /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes; cp libbitsandbytes_cuda118.so libbitsandbytes_cpu.so"
44
- ) # noqa
45
-
46
- import gc
47
  import os
48
- import subprocess as sp
49
- import sys
50
- import time
51
- from collections import deque
52
- from dataclasses import asdict, dataclass
53
- from textwrap import dedent
54
- from types import SimpleNamespace
55
- from typing import List, Optional
56
-
57
  import gradio as gr
58
- import rich
59
- import torch
60
- from loguru import logger
61
- from transformers import AutoModelForCausalLM, AutoTokenizer
62
- from transformers.generation import GenerationConfig
63
-
64
- from example_list import css, example_list
65
-
66
- os.environ["TZ"] = "Asia/Shanghai"
67
- try:
68
- time.tzset() # type: ignore # pylint: disable=no-member
69
- except Exception:
70
- # Windows
71
- logger.warning("Windows, cant run time.tzset()")
72
-
73
- if True:
74
- run_cmd(
75
- "ls -rtl /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes"
76
- )
77
-
78
- logger.info("lsb_release -a")
79
- ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding="utf8")
80
- if ret.stdout:
81
- rich.print(ret.stdout)
82
- if ret.stderr:
83
- rich.print("[red bold]" + ret.stdout)
84
-
85
- logger.info("nvidia-smi")
86
- ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding="utf8")
87
- if ret.stdout:
88
- rich.print(ret.stdout)
89
- if ret.stderr:
90
- rich.print("[red bold]" + ret.stdout)
91
- # raise SystemExit("Interrupt by intentioin")
92
-
93
-
94
- if not torch.cuda.is_available():
95
- raise gr.Error("torch.cuda.is_available() is False, cant continue...")
96
-
97
- model_name = "tangger/Qwen-7B-Chat" # try
98
- model_name = "Qwen/Qwen-7B-Chat" # gone!
99
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
100
-
101
- n_gpus = torch.cuda.device_count()
102
- try:
103
- _ = f"{int(torch.cuda.mem_get_info()[0]/1024**3)-2}GB"
104
- except AssertionError:
105
- _ = 0
106
- max_memory = {i: _ for i in range(n_gpus)}
107
-
108
- del sys
109
- # logger.remove() # to turn on trace
110
- # logger.add(sys.stderr, level="TRACE")
111
- # logger.trace(f"{chat_history=}")
112
-
113
-
114
- def gen_model(model_name: str):
115
- model = AutoModelForCausalLM.from_pretrained(
116
- model_name,
117
- trust_remote_code=True,
118
- # device_map="auto",
119
- device_map={"": 0},
120
- # load_in_4bit=True,
121
- load_in_8bit=True,
122
- max_memory=max_memory,
123
- fp16=True,
124
- torch_dtype=torch.float16,
125
- bnb_4bit_quant_type="nf4",
126
- bnb_4bit_compute_dtype=torch.bfloat16,
127
- )
128
- model = model.eval()
129
- model.generation_config = GenerationConfig.from_pretrained(
130
- model_name,
131
- trust_remote_code=True,
132
- )
133
- return model
134
-
135
-
136
- def user_clear(message, chat_history):
137
- """Gen a response, clear message in user textbox."""
138
- logger.debug(f"{message=}")
139
-
140
- try:
141
- chat_history.append([message, ""])
142
- except Exception:
143
- chat_history = deque([message, ""], maxlen=5)
144
-
145
- logger.trace(f"{chat_history=}")
146
- return "", chat_history
147
-
148
-
149
- def user(message, chat_history):
150
- """Gen a response."""
151
- logger.debug(f"{message=}")
152
- logger.trace(f"{chat_history=}")
153
-
154
- try:
155
- chat_history.append([message, ""])
156
- except Exception:
157
- chat_history = deque([message, ""], maxlen=5)
158
- return message, chat_history
159
 
 
 
 
160
 
161
- # for rerun in tests
162
- model = None
163
- gc.collect()
164
- torch.cuda.empty_cache()
165
 
166
- if not torch.cuda.is_available():
167
- # raise gr.Error("GPU not available, cant run. Turn on GPU and retry")
168
- raise SystemExit("GPU not available, cant run. Turn on GPU and retry")
169
 
170
- model = gen_model(model_name)
171
 
 
 
172
 
173
- def bot(chat_history, **kwargs):
174
- try:
175
- message = chat_history[-1][0]
176
- except Exception as exc:
177
- logger.error(f"{chat_history=}: {exc}")
178
- return chat_history
179
- logger.debug(f"{chat_history=}")
180
- try:
181
- _ = """
182
- response, chat_history = model.chat(
183
- tokenizer,
184
- message,
185
- history=chat_history,
186
- temperature=0.7,
187
- repetition_penalty=1.2,
188
- # max_length=128,
189
  )
190
- """
191
- logger.debug("run model.chat...")
192
- model.generation_config.update(**kwargs)
193
- response, chat_history = model.chat(
194
- tokenizer,
195
- message,
196
- chat_history[:-1],
197
- # **kwargs,
198
- )
199
- del response
200
- return chat_history
201
- except Exception as exc:
202
- logger.error(exc)
203
- chat_history[:-1].append(["message", str(exc)])
204
- return chat_history
205
-
206
-
207
- def bot_stream(chat_history, **kwargs):
208
- logger.trace(f"{kwargs=}")
209
-
210
- # somehow, empty chat_history
211
- if chat_history is None or not chat_history:
212
- logger.trace(f" *** {chat_history=}")
213
- chat_history.append(["", ""])
214
-
215
- try:
216
- message = chat_history[-1][0]
217
- except Exception as exc:
218
- logger.error(f"{chat_history=}: {exc}")
219
- raise gr.Error(f"{chat_history=}")
220
- # yield chat_history
221
-
222
- # for elm in model.chat_stream(tokenizer, message, chat_history):
223
- model.generation_config.update(**kwargs)
224
- response = ""
225
- for elm in model.chat_stream(tokenizer, message, chat_history):
226
- chat_history[-1] = [message, elm]
227
- response = elm
228
- yield chat_history
229
- logger.debug(f"{response=}")
230
- logger.debug(f"{model.generation_config=}")
231
-
232
-
233
- SYSTEM_PROMPT = "You are a helpful assistant."
234
- MAX_MAX_NEW_TOKENS = 2048 # sequence length 2048
235
- MAX_NEW_TOKENS = 256
236
-
237
-
238
- @dataclass
239
- class Config:
240
- max_new_tokens: int = MAX_NEW_TOKENS
241
- repetition_penalty: float = 1.1
242
- temperature: float = 1.0
243
- top_k: int = 0
244
- top_p: float = 0.9
245
 
246
-
247
- # stats_default = SimpleNamespace(llm=model, system_prompt=SYSTEM_PROMPT, config=Config())
248
- stats_default = SimpleNamespace(llm=None, system_prompt=SYSTEM_PROMPT, config=Config())
249
-
250
-
251
- # input max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
252
- def api_fn( # pylint: disable=too-many-arguments
253
- input_text: Optional[str],
254
- # max_length: int = 256,
255
- max_new_tokens: int = stats_default.config.max_new_tokens,
256
- temperature: float = stats_default.config.temperature,
257
- repetition_penalty: float = stats_default.config.repetition_penalty,
258
- top_k: int = stats_default.config.top_k,
259
- top_p: int = stats_default.config.top_p,
260
- system_prompt: Optional[str] = None,
261
- history: Optional[List[str]] = None,
262
- ):
263
- if input_text is None:
264
- input_text = ""
265
- try:
266
- input_text = str(input_text).strip()
267
- except Exception as exc:
268
- logger.error(exc)
269
- input_text = ""
270
- if not input_text:
271
- return ""
272
- if history is None:
273
- history = []
274
- try:
275
- temperature = float(temperature)
276
- except Exception:
277
- temperature = stats_default.config.temperature
278
-
279
- if system_prompt is None:
280
- system_prompt = stats_default.system_prompt
281
- # if max_length < 10: max_length = 4096
282
- if max_new_tokens < 10:
283
- max_new_tokens = stats_default.config.max_new_tokens
284
- if top_p < 0.1 or top_p > 1:
285
- top_p = stats_default.config.top_p
286
- if temperature <= 0.5:
287
- temperature = stats_default.config.temperature
288
-
289
- _ = {
290
- "max_new_tokens": max_new_tokens,
291
- "temperature": temperature,
292
- "repetition_penalty": repetition_penalty,
293
- "top_k": top_k,
294
- "top_p": top_p,
295
- }
296
- model.generation_config.update(**_)
297
- try:
298
- res, _ = model.chat(
299
- tokenizer,
300
- input_text,
301
- history=history,
302
- # max_length=max_length,
303
- # append_history=False,
304
  )
305
- # logger.debug(f"{res=} \n{_=}")
306
- except Exception as exc:
307
- logger.error(f"{exc=}")
308
- res = str(exc)
309
 
310
- logger.debug(f"api {res=}")
311
- logger.debug(f"api {model.generation_config=}")
312
 
313
- return res
314
-
315
-
316
- theme = gr.themes.Soft(text_size="sm")
317
- with gr.Blocks(
318
- theme=theme,
319
- title=model_name.lower(),
320
- css=css,
321
- ) as block:
322
- stats = gr.State(stats_default)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
 
324
- # would this reset model?
325
- model.generation_config = GenerationConfig.from_pretrained(
326
- model_name,
327
- trust_remote_code=True,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
328
  )
329
- config = asdict(stats.value.config)
330
-
331
- def bot_stream_state(chat_history):
332
- logger.trace(f"{chat_history=}")
333
- yield from bot_stream(chat_history, **config)
334
-
335
- with gr.Accordion("🎈 Info", open=False):
336
- gr.Markdown(
337
- dedent(
338
- f"""
339
- ## {model_name.lower()}
340
 
341
- * temperature range: .51 and up; higher temperature implies more randomness. Suggested temperature for chatting and creative writing is around 1.1 while it should be set to 0.51-1.0 for summarizing and translation.
342
- * Set `repetition_penalty` to 2.1 or higher for a chatty conversation (more unpredictable and undesirable output). Lower it to 1.1 or smaller if more focused anwsers are desired (for example for translations or fact-oriented queries).
343
- * Smaller `top_k` probably will result in smoothier sentences.
344
- (`top_k=0` is equivalent to `top_k` equal to very very big though.) Consult `transformers` documentation for more details.
345
- * An API is available at https://mikeee-qwen-7b-chat.hf.space/ that can be queried, e.g., in python
346
- ```python
347
- from gradio_client import Client
348
 
349
- client = Client("https://mikeee-qwen-7b-chat.hf.space/")
350
 
351
- result = client.predict(
352
- "你好!", # user prompt
353
- 256, # max_new_tokens
354
- 1.2, # temperature
355
- 1.1, # repetition_penalty
356
- 0, # top_k
357
- 0.9, # top_p
358
- "You are a helpful assistant.", # system_prompt
359
- None, # history
360
- api_name="/api"
361
- )
362
- print(result)
363
- ```
364
- or in javascript
365
- ```js
366
- import {{ client }} from "@gradio/client";
367
 
368
- const app = await client("https://mikeee-qwen-7b-chat.hf.space/");
369
- const result = await app.predict("api", [...]);
370
- console.log(result.data);
371
- ```
372
- Check documentation and examples by clicking `Use via API` at the very bottom of [https://huggingface.co/spaces/mikeee/qwen-7b-chat](https://huggingface.co/spaces/mikeee/qwen-7b-chat).
373
-
374
- <p></p>
375
- Most examples are meant for another model.
376
- You probably should try to test
377
- some related prompts. System prompt can be changed in Advaned Options as well."""
378
- ),
379
- elem_classes="xsmall",
380
- )
381
-
382
- chatbot = gr.Chatbot(height=500, value=deque([], maxlen=5)) # type: ignore
383
-
384
- with gr.Row():
385
- with gr.Column(scale=5):
386
- msg = gr.Textbox(
387
- label="Chat Message Box",
388
- placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
389
- show_label=False,
390
- # container=False,
391
- lines=4,
392
- max_lines=30,
393
- show_copy_button=True,
394
- # ).style(container=False)
395
- )
396
- with gr.Column(scale=1, min_width=50):
397
- with gr.Row():
398
- submit = gr.Button("Submit", elem_classes="xsmall")
399
- stop = gr.Button("Stop", visible=True)
400
- clear = gr.Button("Clear History", visible=True)
401
-
402
- msg_submit_event = msg.submit(
403
- # fn=conversation.user_turn,
404
- fn=user,
405
- inputs=[msg, chatbot],
406
- outputs=[msg, chatbot],
407
- queue=True,
408
- show_progress="full",
409
- # api_name=None,
410
- ).then(bot_stream_state, chatbot, chatbot, queue=True)
411
- submit_click_event = submit.click(
412
- # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
413
- fn=user_clear, # clear msg
414
- inputs=[msg, chatbot],
415
- outputs=[msg, chatbot],
416
- queue=True,
417
- show_progress="full",
418
- # api_name=None,
419
- ).then(bot_stream_state, chatbot, chatbot, queue=True)
420
- stop.click(
421
- fn=None,
422
- inputs=None,
423
- outputs=None,
424
- cancels=[msg_submit_event, submit_click_event],
425
- queue=False,
426
  )
427
- clear.click(lambda: None, None, chatbot, queue=False)
428
 
429
- with gr.Accordion(label="Advanced Options", open=False):
430
- system_prompt = gr.Textbox(
431
- label="System prompt",
432
- value=stats_default.system_prompt,
433
- lines=3,
434
- visible=True,
435
- )
436
- max_new_tokens = gr.Slider(
437
- label="Max new tokens",
438
- minimum=1,
439
- maximum=MAX_MAX_NEW_TOKENS,
440
- step=1,
441
- value=stats_default.config.max_new_tokens,
442
- )
443
- repetition_penalty = gr.Slider(
444
- label="Repetition penalty",
445
- minimum=0.1,
446
- maximum=40.0,
447
- step=0.1,
448
- value=stats_default.config.repetition_penalty,
449
  )
 
450
  temperature = gr.Slider(
451
- label="Temperature",
452
- minimum=0.51,
453
- maximum=40.0,
454
- step=0.1,
455
- value=stats_default.config.temperature,
456
- )
457
- top_p = gr.Slider(
458
- label="Top-p (nucleus sampling)",
459
- minimum=0.05,
460
- maximum=1.0,
461
  step=0.05,
462
- value=stats_default.config.top_p,
463
- )
464
- top_k = gr.Slider(
465
- label="Top-k",
466
- minimum=0,
467
- maximum=1000,
468
- step=1,
469
- value=stats_default.config.top_k,
470
- )
471
-
472
- def system_prompt_fn(system_prompt):
473
- stats.value.system_prompt = system_prompt
474
- logger.debug(f"{stats.value.system_prompt=}")
475
-
476
- def max_new_tokens_fn(max_new_tokens):
477
- stats.value.config.max_new_tokens = max_new_tokens
478
- logger.debug(f"{stats.value.config.max_new_tokens=}")
479
-
480
- def repetition_penalty_fn(repetition_penalty):
481
- stats.value.config.repetition_penalty = repetition_penalty
482
- logger.debug(f"{stats.value=}")
483
-
484
- def temperature_fn(temperature):
485
- stats.value.config.temperature = temperature
486
- logger.debug(f"{stats.value=}")
487
-
488
- def top_p_fn(top_p):
489
- stats.value.config.top_p = top_p
490
- logger.debug(f"{stats.value=}")
491
-
492
- def top_k_fn(top_k):
493
- stats.value.config.top_k = top_k
494
- logger.debug(f"{stats.value=}")
495
-
496
- system_prompt.change(system_prompt_fn, system_prompt)
497
- max_new_tokens.change(max_new_tokens_fn, max_new_tokens)
498
- repetition_penalty.change(repetition_penalty_fn, repetition_penalty)
499
- temperature.change(temperature_fn, temperature)
500
- top_p.change(top_p_fn, top_p)
501
- top_k.change(top_k_fn, top_k)
502
-
503
- def reset_fn(stats_):
504
- logger.debug("reset_fn")
505
- stats_ = gr.State(stats_default)
506
- logger.debug(f"{stats_.value=}")
507
- return (
508
- stats_,
509
- stats_default.system_prompt,
510
- stats_default.config.max_new_tokens,
511
- stats_default.config.repetition_penalty,
512
- stats_default.config.temperature,
513
- stats_default.config.top_p,
514
- stats_default.config.top_k,
515
- )
516
-
517
- reset_btn = gr.Button("Reset")
518
- reset_btn.click(
519
- reset_fn,
520
- stats,
521
- [
522
- stats,
523
- system_prompt,
524
- max_new_tokens,
525
- repetition_penalty,
526
- temperature,
527
- top_p,
528
- top_k,
529
- ],
530
  )
531
 
532
- with gr.Accordion("Example inputs", open=True):
533
- etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
534
- examples = gr.Examples(
535
- examples=example_list,
536
- inputs=[msg],
537
- examples_per_page=60,
538
- )
539
- with gr.Accordion("Disclaimer", open=False):
540
- _ = model_name.lower()
541
- gr.Markdown(
542
- f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
543
- f"factually accurate information. {_} was trained on various public datasets; while great efforts "
544
- "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
545
- "biased, or otherwise offensive outputs.",
546
- elem_classes=["disclaimer"],
547
- )
548
 
549
- with gr.Accordion("For Chat/Translation API", open=False, visible=False):
550
- input_text = gr.Text()
551
- api_history = gr.Chatbot(value=[])
552
- api_btn = gr.Button("Go", variant="primary")
553
- out_text = gr.Text()
554
 
555
- # api_fn args order
556
- # input_text max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
557
- api_btn.click(
558
- api_fn,
559
- [
560
- input_text,
561
- max_new_tokens,
562
- temperature,
563
- repetition_penalty,
564
- top_k,
565
- top_p,
566
- system_prompt,
567
- api_history, # dont know how to pass this in gradio_client.Client calls
568
- ],
569
- out_text,
570
- api_name="api",
571
  )
572
 
573
 
574
  if __name__ == "__main__":
575
- logger.info("Just record start time")
576
-
577
- _ = """
578
- ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding='utf8')
579
- if ret.stdout:
580
- rich.print(ret.stdout)
581
- if ret.stderr:
582
- rich.print("[red bold]" + ret.stdout)
583
-
584
- ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding='utf8')
585
- if ret.stdout:
586
- rich.print(ret.stdout)
587
- if ret.stderr:
588
- rich.print("[red bold]" + ret.stdout)
589
- raise SystemExit("Interrupt by intentioin")
590
- # """
591
-
592
- block.queue(max_size=8).launch(debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ import subprocess
 
 
 
 
 
 
 
 
3
  import gradio as gr
4
+ import spaces
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
 
6
+ MODEL_REPO = "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive"
7
+ MODEL_FILE = "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf"
8
+ MMPROJ_FILE = "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf"
9
 
10
+ MODEL_PATH = os.path.join("/tmp", MODEL_FILE)
11
+ MMPROJ_PATH = os.path.join("/tmp", MMPROJ_FILE)
 
 
12
 
13
+ server_process = None
 
 
14
 
 
15
 
16
+ def download_models():
17
+ from huggingface_hub import hf_hub_download
18
 
19
+ if not os.path.exists(MODEL_PATH):
20
+ hf_hub_download(
21
+ repo_id=MODEL_REPO,
22
+ filename=MODEL_FILE,
23
+ local_dir="/tmp",
 
 
 
 
 
 
 
 
 
 
 
24
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
+ if not os.path.exists(MMPROJ_PATH):
27
+ hf_hub_download(
28
+ repo_id=MODEL_REPO,
29
+ filename=MMPROJ_FILE,
30
+ local_dir="/tmp",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  )
 
 
 
 
32
 
 
 
33
 
34
+ @spaces.GPU
35
+ def generate(prompt, max_tokens, temperature):
36
+ global server_process
37
+
38
+ download_models()
39
+
40
+ if server_process is None or server_process.poll() is not None:
41
+ cmd = [
42
+ "llama-server",
43
+ "-m", MODEL_PATH,
44
+ "--mmproj", MMPROJ_PATH,
45
+ "--host", "127.0.0.1",
46
+ "--port", "8080",
47
+ "-c", "32768",
48
+ "-n", "4096",
49
+ "--n-gpu-layers", "99",
50
+ "-t", "2",
51
+ ]
52
+
53
+ server_process = subprocess.Popen(
54
+ cmd,
55
+ stdout=subprocess.PIPE,
56
+ stderr=subprocess.STDOUT,
57
+ text=True,
58
+ )
59
 
60
+ import time
61
+ import requests
62
+
63
+ for _ in range(120):
64
+ try:
65
+ r = requests.get("http://127.0.0.1:8080/health", timeout=1)
66
+ if r.ok:
67
+ break
68
+ except Exception:
69
+ pass
70
+ time.sleep(1)
71
+ else:
72
+ raise RuntimeError("llama-server did not become ready.")
73
+
74
+ response = requests.post(
75
+ "http://127.0.0.1:8080/completion",
76
+ json={
77
+ "prompt": prompt,
78
+ "n_predict": int(max_tokens),
79
+ "temperature": float(temperature),
80
+ "stream": False,
81
+ },
82
+ timeout=600,
83
  )
 
 
 
 
 
 
 
 
 
 
 
84
 
85
+ response.raise_for_status()
86
+ return response.json().get("content", "")
 
 
 
 
 
87
 
 
88
 
89
+ with gr.Blocks(title="Gemma-4-E4B Uncensored") as demo:
90
+ gr.Markdown("# Gemma-4-E4B-Uncensored-HauhauCS-Aggressive")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
 
92
+ prompt = gr.Textbox(
93
+ label="Prompt",
94
+ placeholder="Enter your prompt...",
95
+ lines=8,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  )
 
97
 
98
+ with gr.Row():
99
+ max_tokens = gr.Slider(
100
+ minimum=64,
101
+ maximum=4096,
102
+ value=1024,
103
+ step=64,
104
+ label="Max tokens",
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  )
106
+
107
  temperature = gr.Slider(
108
+ minimum=0.0,
109
+ maximum=2.0,
110
+ value=0.7,
 
 
 
 
 
 
 
111
  step=0.05,
112
+ label="Temperature",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
  )
114
 
115
+ output = gr.Textbox(
116
+ label="Response",
117
+ lines=20,
118
+ )
 
 
 
 
 
 
 
 
 
 
 
 
119
 
120
+ generate_button = gr.Button("Generate", variant="primary")
 
 
 
 
121
 
122
+ generate_button.click(
123
+ generate,
124
+ inputs=[prompt, max_tokens, temperature],
125
+ outputs=output,
 
 
 
 
 
 
 
 
 
 
 
 
126
  )
127
 
128
 
129
  if __name__ == "__main__":
130
+ demo.launch()