identityvoid commited on
Commit
3fd1be3
·
1 Parent(s): af519e0

Convert model app to Gradio ZeroGPU

Browse files
Files changed (3) hide show
  1. app.py +98 -560
  2. app.py.cuda-original +592 -0
  3. requirements.txt +2 -12
app.py CHANGED
@@ -1,592 +1,130 @@
1
- """
2
- Run qwen 7b chat.
3
-
4
- transformers 4.31.0
5
-
6
- import torch
7
- torch.cuda.empty_cache()
8
-
9
- model.chat(
10
- tokenizer: transformers.tokenization_utils.PreTrainedTokenizer,
11
- query: str,
12
- history: Optional[List[Tuple[str, str]]],
13
- system: str = 'You are a helpful assistant.',
14
- append_history: bool = True,
15
- stream: Optional[bool] = <object object at 0x7f905797ec20>,
16
- stop_words_ids: Optional[List[List[int]]] = None,
17
- **kwargs) -> Tuple[str, List[Tuple[str, str]]]
18
- )
19
-
20
- model.generation_config
21
- GenerationConfig {
22
- "chat_format": "chatml",
23
- "do_sample": true,
24
- "eos_token_id": 151643,
25
- "max_new_tokens": 512,
26
- "max_window_size": 6144,
27
- "pad_token_id": 151643,
28
- "top_k": 0,
29
- "top_p": 0.5,
30
- "transformers_version": "4.31.0",
31
- "trust_remote_code": true
32
- }
33
- """
34
- # pylint: disable=line-too-long, invalid-name, no-member, redefined-outer-name, missing-function-docstring, missing-class-docstring, broad-except,
35
- from run_cmd import run_cmd # noqa
36
-
37
- # autodl with cuda12 NVIDIA-SMI appears
38
- # 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0
39
- # no fix needed
40
-
41
- # clumsy fix for hf overwrite libbitsandbytes_cpu.so with libbitsandbytes_cuda118.so
42
- run_cmd(
43
- "cd /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes; cp libbitsandbytes_cuda118.so libbitsandbytes_cpu.so"
44
- ) # noqa
45
-
46
- import gc
47
  import os
48
- import subprocess as sp
49
- import sys
50
- import time
51
- from collections import deque
52
- from dataclasses import asdict, dataclass
53
- from textwrap import dedent
54
- from types import SimpleNamespace
55
- from typing import List, Optional
56
-
57
  import gradio as gr
58
- import rich
59
- import torch
60
- from loguru import logger
61
- from transformers import AutoModelForCausalLM, AutoTokenizer
62
- from transformers.generation import GenerationConfig
63
-
64
- from example_list import css, example_list
65
-
66
- os.environ["TZ"] = "Asia/Shanghai"
67
- try:
68
- time.tzset() # type: ignore # pylint: disable=no-member
69
- except Exception:
70
- # Windows
71
- logger.warning("Windows, cant run time.tzset()")
72
-
73
- if True:
74
- run_cmd(
75
- "ls -rtl /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes"
76
- )
77
-
78
- logger.info("lsb_release -a")
79
- ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding="utf8")
80
- if ret.stdout:
81
- rich.print(ret.stdout)
82
- if ret.stderr:
83
- rich.print("[red bold]" + ret.stdout)
84
-
85
- logger.info("nvidia-smi")
86
- ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding="utf8")
87
- if ret.stdout:
88
- rich.print(ret.stdout)
89
- if ret.stderr:
90
- rich.print("[red bold]" + ret.stdout)
91
- # raise SystemExit("Interrupt by intentioin")
92
-
93
-
94
- if not torch.cuda.is_available():
95
- raise gr.Error("torch.cuda.is_available() is False, cant continue...")
96
-
97
- model_name = "tangger/Qwen-7B-Chat" # try
98
- model_name = "Qwen/Qwen-7B-Chat" # gone!
99
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
100
-
101
- n_gpus = torch.cuda.device_count()
102
- try:
103
- _ = f"{int(torch.cuda.mem_get_info()[0]/1024**3)-2}GB"
104
- except AssertionError:
105
- _ = 0
106
- max_memory = {i: _ for i in range(n_gpus)}
107
-
108
- del sys
109
- # logger.remove() # to turn on trace
110
- # logger.add(sys.stderr, level="TRACE")
111
- # logger.trace(f"{chat_history=}")
112
-
113
-
114
- def gen_model(model_name: str):
115
- model = AutoModelForCausalLM.from_pretrained(
116
- model_name,
117
- trust_remote_code=True,
118
- # device_map="auto",
119
- device_map={"": 0},
120
- # load_in_4bit=True,
121
- load_in_8bit=True,
122
- max_memory=max_memory,
123
- fp16=True,
124
- torch_dtype=torch.float16,
125
- bnb_4bit_quant_type="nf4",
126
- bnb_4bit_compute_dtype=torch.bfloat16,
127
- )
128
- model = model.eval()
129
- model.generation_config = GenerationConfig.from_pretrained(
130
- model_name,
131
- trust_remote_code=True,
132
- )
133
- return model
134
-
135
-
136
- def user_clear(message, chat_history):
137
- """Gen a response, clear message in user textbox."""
138
- logger.debug(f"{message=}")
139
-
140
- try:
141
- chat_history.append([message, ""])
142
- except Exception:
143
- chat_history = deque([message, ""], maxlen=5)
144
-
145
- logger.trace(f"{chat_history=}")
146
- return "", chat_history
147
-
148
-
149
- def user(message, chat_history):
150
- """Gen a response."""
151
- logger.debug(f"{message=}")
152
- logger.trace(f"{chat_history=}")
153
-
154
- try:
155
- chat_history.append([message, ""])
156
- except Exception:
157
- chat_history = deque([message, ""], maxlen=5)
158
- return message, chat_history
159
 
 
 
 
160
 
161
- # for rerun in tests
162
- model = None
163
- gc.collect()
164
- torch.cuda.empty_cache()
165
 
166
- if not torch.cuda.is_available():
167
- # raise gr.Error("GPU not available, cant run. Turn on GPU and retry")
168
- raise SystemExit("GPU not available, cant run. Turn on GPU and retry")
169
 
170
- model = gen_model(model_name)
171
 
 
 
172
 
173
- def bot(chat_history, **kwargs):
174
- try:
175
- message = chat_history[-1][0]
176
- except Exception as exc:
177
- logger.error(f"{chat_history=}: {exc}")
178
- return chat_history
179
- logger.debug(f"{chat_history=}")
180
- try:
181
- _ = """
182
- response, chat_history = model.chat(
183
- tokenizer,
184
- message,
185
- history=chat_history,
186
- temperature=0.7,
187
- repetition_penalty=1.2,
188
- # max_length=128,
189
  )
190
- """
191
- logger.debug("run model.chat...")
192
- model.generation_config.update(**kwargs)
193
- response, chat_history = model.chat(
194
- tokenizer,
195
- message,
196
- chat_history[:-1],
197
- # **kwargs,
198
- )
199
- del response
200
- return chat_history
201
- except Exception as exc:
202
- logger.error(exc)
203
- chat_history[:-1].append(["message", str(exc)])
204
- return chat_history
205
-
206
-
207
- def bot_stream(chat_history, **kwargs):
208
- logger.trace(f"{kwargs=}")
209
-
210
- # somehow, empty chat_history
211
- if chat_history is None or not chat_history:
212
- logger.trace(f" *** {chat_history=}")
213
- chat_history.append(["", ""])
214
-
215
- try:
216
- message = chat_history[-1][0]
217
- except Exception as exc:
218
- logger.error(f"{chat_history=}: {exc}")
219
- raise gr.Error(f"{chat_history=}")
220
- # yield chat_history
221
-
222
- # for elm in model.chat_stream(tokenizer, message, chat_history):
223
- model.generation_config.update(**kwargs)
224
- response = ""
225
- for elm in model.chat_stream(tokenizer, message, chat_history):
226
- chat_history[-1] = [message, elm]
227
- response = elm
228
- yield chat_history
229
- logger.debug(f"{response=}")
230
- logger.debug(f"{model.generation_config=}")
231
-
232
-
233
- SYSTEM_PROMPT = "You are a helpful assistant."
234
- MAX_MAX_NEW_TOKENS = 2048 # sequence length 2048
235
- MAX_NEW_TOKENS = 256
236
-
237
-
238
- @dataclass
239
- class Config:
240
- max_new_tokens: int = MAX_NEW_TOKENS
241
- repetition_penalty: float = 1.1
242
- temperature: float = 1.0
243
- top_k: int = 0
244
- top_p: float = 0.9
245
 
246
-
247
- # stats_default = SimpleNamespace(llm=model, system_prompt=SYSTEM_PROMPT, config=Config())
248
- stats_default = SimpleNamespace(llm=None, system_prompt=SYSTEM_PROMPT, config=Config())
249
-
250
-
251
- # input max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
252
- def api_fn( # pylint: disable=too-many-arguments
253
- input_text: Optional[str],
254
- # max_length: int = 256,
255
- max_new_tokens: int = stats_default.config.max_new_tokens,
256
- temperature: float = stats_default.config.temperature,
257
- repetition_penalty: float = stats_default.config.repetition_penalty,
258
- top_k: int = stats_default.config.top_k,
259
- top_p: int = stats_default.config.top_p,
260
- system_prompt: Optional[str] = None,
261
- history: Optional[List[str]] = None,
262
- ):
263
- if input_text is None:
264
- input_text = ""
265
- try:
266
- input_text = str(input_text).strip()
267
- except Exception as exc:
268
- logger.error(exc)
269
- input_text = ""
270
- if not input_text:
271
- return ""
272
- if history is None:
273
- history = []
274
- try:
275
- temperature = float(temperature)
276
- except Exception:
277
- temperature = stats_default.config.temperature
278
-
279
- if system_prompt is None:
280
- system_prompt = stats_default.system_prompt
281
- # if max_length < 10: max_length = 4096
282
- if max_new_tokens < 10:
283
- max_new_tokens = stats_default.config.max_new_tokens
284
- if top_p < 0.1 or top_p > 1:
285
- top_p = stats_default.config.top_p
286
- if temperature <= 0.5:
287
- temperature = stats_default.config.temperature
288
-
289
- _ = {
290
- "max_new_tokens": max_new_tokens,
291
- "temperature": temperature,
292
- "repetition_penalty": repetition_penalty,
293
- "top_k": top_k,
294
- "top_p": top_p,
295
- }
296
- model.generation_config.update(**_)
297
- try:
298
- res, _ = model.chat(
299
- tokenizer,
300
- input_text,
301
- history=history,
302
- # max_length=max_length,
303
- # append_history=False,
304
  )
305
- # logger.debug(f"{res=} \n{_=}")
306
- except Exception as exc:
307
- logger.error(f"{exc=}")
308
- res = str(exc)
309
 
310
- logger.debug(f"api {res=}")
311
- logger.debug(f"api {model.generation_config=}")
312
 
313
- return res
314
-
315
-
316
- theme = gr.themes.Soft(text_size="sm")
317
- with gr.Blocks(
318
- theme=theme,
319
- title=model_name.lower(),
320
- css=css,
321
- ) as block:
322
- stats = gr.State(stats_default)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
 
324
- # would this reset model?
325
- model.generation_config = GenerationConfig.from_pretrained(
326
- model_name,
327
- trust_remote_code=True,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
328
  )
329
- config = asdict(stats.value.config)
330
-
331
- def bot_stream_state(chat_history):
332
- logger.trace(f"{chat_history=}")
333
- yield from bot_stream(chat_history, **config)
334
-
335
- with gr.Accordion("🎈 Info", open=False):
336
- gr.Markdown(
337
- dedent(
338
- f"""
339
- ## {model_name.lower()}
340
 
341
- * temperature range: .51 and up; higher temperature implies more randomness. Suggested temperature for chatting and creative writing is around 1.1 while it should be set to 0.51-1.0 for summarizing and translation.
342
- * Set `repetition_penalty` to 2.1 or higher for a chatty conversation (more unpredictable and undesirable output). Lower it to 1.1 or smaller if more focused anwsers are desired (for example for translations or fact-oriented queries).
343
- * Smaller `top_k` probably will result in smoothier sentences.
344
- (`top_k=0` is equivalent to `top_k` equal to very very big though.) Consult `transformers` documentation for more details.
345
- * An API is available at https://mikeee-qwen-7b-chat.hf.space/ that can be queried, e.g., in python
346
- ```python
347
- from gradio_client import Client
348
 
349
- client = Client("https://mikeee-qwen-7b-chat.hf.space/")
350
 
351
- result = client.predict(
352
- "你好!", # user prompt
353
- 256, # max_new_tokens
354
- 1.2, # temperature
355
- 1.1, # repetition_penalty
356
- 0, # top_k
357
- 0.9, # top_p
358
- "You are a helpful assistant.", # system_prompt
359
- None, # history
360
- api_name="/api"
361
- )
362
- print(result)
363
- ```
364
- or in javascript
365
- ```js
366
- import {{ client }} from "@gradio/client";
367
 
368
- const app = await client("https://mikeee-qwen-7b-chat.hf.space/");
369
- const result = await app.predict("api", [...]);
370
- console.log(result.data);
371
- ```
372
- Check documentation and examples by clicking `Use via API` at the very bottom of [https://huggingface.co/spaces/mikeee/qwen-7b-chat](https://huggingface.co/spaces/mikeee/qwen-7b-chat).
373
-
374
- <p></p>
375
- Most examples are meant for another model.
376
- You probably should try to test
377
- some related prompts. System prompt can be changed in Advaned Options as well."""
378
- ),
379
- elem_classes="xsmall",
380
- )
381
-
382
- chatbot = gr.Chatbot(height=500, value=deque([], maxlen=5)) # type: ignore
383
-
384
- with gr.Row():
385
- with gr.Column(scale=5):
386
- msg = gr.Textbox(
387
- label="Chat Message Box",
388
- placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
389
- show_label=False,
390
- # container=False,
391
- lines=4,
392
- max_lines=30,
393
- show_copy_button=True,
394
- # ).style(container=False)
395
- )
396
- with gr.Column(scale=1, min_width=50):
397
- with gr.Row():
398
- submit = gr.Button("Submit", elem_classes="xsmall")
399
- stop = gr.Button("Stop", visible=True)
400
- clear = gr.Button("Clear History", visible=True)
401
-
402
- msg_submit_event = msg.submit(
403
- # fn=conversation.user_turn,
404
- fn=user,
405
- inputs=[msg, chatbot],
406
- outputs=[msg, chatbot],
407
- queue=True,
408
- show_progress="full",
409
- # api_name=None,
410
- ).then(bot_stream_state, chatbot, chatbot, queue=True)
411
- submit_click_event = submit.click(
412
- # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
413
- fn=user_clear, # clear msg
414
- inputs=[msg, chatbot],
415
- outputs=[msg, chatbot],
416
- queue=True,
417
- show_progress="full",
418
- # api_name=None,
419
- ).then(bot_stream_state, chatbot, chatbot, queue=True)
420
- stop.click(
421
- fn=None,
422
- inputs=None,
423
- outputs=None,
424
- cancels=[msg_submit_event, submit_click_event],
425
- queue=False,
426
  )
427
- clear.click(lambda: None, None, chatbot, queue=False)
428
 
429
- with gr.Accordion(label="Advanced Options", open=False):
430
- system_prompt = gr.Textbox(
431
- label="System prompt",
432
- value=stats_default.system_prompt,
433
- lines=3,
434
- visible=True,
435
- )
436
- max_new_tokens = gr.Slider(
437
- label="Max new tokens",
438
- minimum=1,
439
- maximum=MAX_MAX_NEW_TOKENS,
440
- step=1,
441
- value=stats_default.config.max_new_tokens,
442
- )
443
- repetition_penalty = gr.Slider(
444
- label="Repetition penalty",
445
- minimum=0.1,
446
- maximum=40.0,
447
- step=0.1,
448
- value=stats_default.config.repetition_penalty,
449
  )
 
450
  temperature = gr.Slider(
451
- label="Temperature",
452
- minimum=0.51,
453
- maximum=40.0,
454
- step=0.1,
455
- value=stats_default.config.temperature,
456
- )
457
- top_p = gr.Slider(
458
- label="Top-p (nucleus sampling)",
459
- minimum=0.05,
460
- maximum=1.0,
461
  step=0.05,
462
- value=stats_default.config.top_p,
463
- )
464
- top_k = gr.Slider(
465
- label="Top-k",
466
- minimum=0,
467
- maximum=1000,
468
- step=1,
469
- value=stats_default.config.top_k,
470
- )
471
-
472
- def system_prompt_fn(system_prompt):
473
- stats.value.system_prompt = system_prompt
474
- logger.debug(f"{stats.value.system_prompt=}")
475
-
476
- def max_new_tokens_fn(max_new_tokens):
477
- stats.value.config.max_new_tokens = max_new_tokens
478
- logger.debug(f"{stats.value.config.max_new_tokens=}")
479
-
480
- def repetition_penalty_fn(repetition_penalty):
481
- stats.value.config.repetition_penalty = repetition_penalty
482
- logger.debug(f"{stats.value=}")
483
-
484
- def temperature_fn(temperature):
485
- stats.value.config.temperature = temperature
486
- logger.debug(f"{stats.value=}")
487
-
488
- def top_p_fn(top_p):
489
- stats.value.config.top_p = top_p
490
- logger.debug(f"{stats.value=}")
491
-
492
- def top_k_fn(top_k):
493
- stats.value.config.top_k = top_k
494
- logger.debug(f"{stats.value=}")
495
-
496
- system_prompt.change(system_prompt_fn, system_prompt)
497
- max_new_tokens.change(max_new_tokens_fn, max_new_tokens)
498
- repetition_penalty.change(repetition_penalty_fn, repetition_penalty)
499
- temperature.change(temperature_fn, temperature)
500
- top_p.change(top_p_fn, top_p)
501
- top_k.change(top_k_fn, top_k)
502
-
503
- def reset_fn(stats_):
504
- logger.debug("reset_fn")
505
- stats_ = gr.State(stats_default)
506
- logger.debug(f"{stats_.value=}")
507
- return (
508
- stats_,
509
- stats_default.system_prompt,
510
- stats_default.config.max_new_tokens,
511
- stats_default.config.repetition_penalty,
512
- stats_default.config.temperature,
513
- stats_default.config.top_p,
514
- stats_default.config.top_k,
515
- )
516
-
517
- reset_btn = gr.Button("Reset")
518
- reset_btn.click(
519
- reset_fn,
520
- stats,
521
- [
522
- stats,
523
- system_prompt,
524
- max_new_tokens,
525
- repetition_penalty,
526
- temperature,
527
- top_p,
528
- top_k,
529
- ],
530
  )
531
 
532
- with gr.Accordion("Example inputs", open=True):
533
- etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
534
- examples = gr.Examples(
535
- examples=example_list,
536
- inputs=[msg],
537
- examples_per_page=60,
538
- )
539
- with gr.Accordion("Disclaimer", open=False):
540
- _ = model_name.lower()
541
- gr.Markdown(
542
- f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
543
- f"factually accurate information. {_} was trained on various public datasets; while great efforts "
544
- "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
545
- "biased, or otherwise offensive outputs.",
546
- elem_classes=["disclaimer"],
547
- )
548
 
549
- with gr.Accordion("For Chat/Translation API", open=False, visible=False):
550
- input_text = gr.Text()
551
- api_history = gr.Chatbot(value=[])
552
- api_btn = gr.Button("Go", variant="primary")
553
- out_text = gr.Text()
554
 
555
- # api_fn args order
556
- # input_text max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
557
- api_btn.click(
558
- api_fn,
559
- [
560
- input_text,
561
- max_new_tokens,
562
- temperature,
563
- repetition_penalty,
564
- top_k,
565
- top_p,
566
- system_prompt,
567
- api_history, # dont know how to pass this in gradio_client.Client calls
568
- ],
569
- out_text,
570
- api_name="api",
571
  )
572
 
573
 
574
  if __name__ == "__main__":
575
- logger.info("Just record start time")
576
-
577
- _ = """
578
- ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding='utf8')
579
- if ret.stdout:
580
- rich.print(ret.stdout)
581
- if ret.stderr:
582
- rich.print("[red bold]" + ret.stdout)
583
-
584
- ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding='utf8')
585
- if ret.stdout:
586
- rich.print(ret.stdout)
587
- if ret.stderr:
588
- rich.print("[red bold]" + ret.stdout)
589
- raise SystemExit("Interrupt by intentioin")
590
- # """
591
-
592
- block.queue(max_size=8).launch(debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ import subprocess
 
 
 
 
 
 
 
 
3
  import gradio as gr
4
+ import spaces
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
 
6
+ MODEL_REPO = "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive"
7
+ MODEL_FILE = "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf"
8
+ MMPROJ_FILE = "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf"
9
 
10
+ MODEL_PATH = os.path.join("/tmp", MODEL_FILE)
11
+ MMPROJ_PATH = os.path.join("/tmp", MMPROJ_FILE)
 
 
12
 
13
+ server_process = None
 
 
14
 
 
15
 
16
+ def download_models():
17
+ from huggingface_hub import hf_hub_download
18
 
19
+ if not os.path.exists(MODEL_PATH):
20
+ hf_hub_download(
21
+ repo_id=MODEL_REPO,
22
+ filename=MODEL_FILE,
23
+ local_dir="/tmp",
 
 
 
 
 
 
 
 
 
 
 
24
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
+ if not os.path.exists(MMPROJ_PATH):
27
+ hf_hub_download(
28
+ repo_id=MODEL_REPO,
29
+ filename=MMPROJ_FILE,
30
+ local_dir="/tmp",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  )
 
 
 
 
32
 
 
 
33
 
34
+ @spaces.GPU
35
+ def generate(prompt, max_tokens, temperature):
36
+ global server_process
37
+
38
+ download_models()
39
+
40
+ if server_process is None or server_process.poll() is not None:
41
+ cmd = [
42
+ "llama-server",
43
+ "-m", MODEL_PATH,
44
+ "--mmproj", MMPROJ_PATH,
45
+ "--host", "127.0.0.1",
46
+ "--port", "8080",
47
+ "-c", "32768",
48
+ "-n", "4096",
49
+ "--n-gpu-layers", "99",
50
+ "-t", "2",
51
+ ]
52
+
53
+ server_process = subprocess.Popen(
54
+ cmd,
55
+ stdout=subprocess.PIPE,
56
+ stderr=subprocess.STDOUT,
57
+ text=True,
58
+ )
59
 
60
+ import time
61
+ import requests
62
+
63
+ for _ in range(120):
64
+ try:
65
+ r = requests.get("http://127.0.0.1:8080/health", timeout=1)
66
+ if r.ok:
67
+ break
68
+ except Exception:
69
+ pass
70
+ time.sleep(1)
71
+ else:
72
+ raise RuntimeError("llama-server did not become ready.")
73
+
74
+ response = requests.post(
75
+ "http://127.0.0.1:8080/completion",
76
+ json={
77
+ "prompt": prompt,
78
+ "n_predict": int(max_tokens),
79
+ "temperature": float(temperature),
80
+ "stream": False,
81
+ },
82
+ timeout=600,
83
  )
 
 
 
 
 
 
 
 
 
 
 
84
 
85
+ response.raise_for_status()
86
+ return response.json().get("content", "")
 
 
 
 
 
87
 
 
88
 
89
+ with gr.Blocks(title="Gemma-4-E4B Uncensored") as demo:
90
+ gr.Markdown("# Gemma-4-E4B-Uncensored-HauhauCS-Aggressive")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
 
92
+ prompt = gr.Textbox(
93
+ label="Prompt",
94
+ placeholder="Enter your prompt...",
95
+ lines=8,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  )
 
97
 
98
+ with gr.Row():
99
+ max_tokens = gr.Slider(
100
+ minimum=64,
101
+ maximum=4096,
102
+ value=1024,
103
+ step=64,
104
+ label="Max tokens",
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  )
106
+
107
  temperature = gr.Slider(
108
+ minimum=0.0,
109
+ maximum=2.0,
110
+ value=0.7,
 
 
 
 
 
 
 
111
  step=0.05,
112
+ label="Temperature",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
  )
114
 
115
+ output = gr.Textbox(
116
+ label="Response",
117
+ lines=20,
118
+ )
 
 
 
 
 
 
 
 
 
 
 
 
119
 
120
+ generate_button = gr.Button("Generate", variant="primary")
 
 
 
 
121
 
122
+ generate_button.click(
123
+ generate,
124
+ inputs=[prompt, max_tokens, temperature],
125
+ outputs=output,
 
 
 
 
 
 
 
 
 
 
 
 
126
  )
127
 
128
 
129
  if __name__ == "__main__":
130
+ demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app.py.cuda-original ADDED
@@ -0,0 +1,592 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Run qwen 7b chat.
3
+
4
+ transformers 4.31.0
5
+
6
+ import torch
7
+ torch.cuda.empty_cache()
8
+
9
+ model.chat(
10
+ tokenizer: transformers.tokenization_utils.PreTrainedTokenizer,
11
+ query: str,
12
+ history: Optional[List[Tuple[str, str]]],
13
+ system: str = 'You are a helpful assistant.',
14
+ append_history: bool = True,
15
+ stream: Optional[bool] = <object object at 0x7f905797ec20>,
16
+ stop_words_ids: Optional[List[List[int]]] = None,
17
+ **kwargs) -> Tuple[str, List[Tuple[str, str]]]
18
+ )
19
+
20
+ model.generation_config
21
+ GenerationConfig {
22
+ "chat_format": "chatml",
23
+ "do_sample": true,
24
+ "eos_token_id": 151643,
25
+ "max_new_tokens": 512,
26
+ "max_window_size": 6144,
27
+ "pad_token_id": 151643,
28
+ "top_k": 0,
29
+ "top_p": 0.5,
30
+ "transformers_version": "4.31.0",
31
+ "trust_remote_code": true
32
+ }
33
+ """
34
+ # pylint: disable=line-too-long, invalid-name, no-member, redefined-outer-name, missing-function-docstring, missing-class-docstring, broad-except,
35
+ from run_cmd import run_cmd # noqa
36
+
37
+ # autodl with cuda12 NVIDIA-SMI appears
38
+ # 525.105.17 Driver Version: 525.105.17 CUDA Version: 12.0
39
+ # no fix needed
40
+
41
+ # clumsy fix for hf overwrite libbitsandbytes_cpu.so with libbitsandbytes_cuda118.so
42
+ run_cmd(
43
+ "cd /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes; cp libbitsandbytes_cuda118.so libbitsandbytes_cpu.so"
44
+ ) # noqa
45
+
46
+ import gc
47
+ import os
48
+ import subprocess as sp
49
+ import sys
50
+ import time
51
+ from collections import deque
52
+ from dataclasses import asdict, dataclass
53
+ from textwrap import dedent
54
+ from types import SimpleNamespace
55
+ from typing import List, Optional
56
+
57
+ import gradio as gr
58
+ import rich
59
+ import torch
60
+ from loguru import logger
61
+ from transformers import AutoModelForCausalLM, AutoTokenizer
62
+ from transformers.generation import GenerationConfig
63
+
64
+ from example_list import css, example_list
65
+
66
+ os.environ["TZ"] = "Asia/Shanghai"
67
+ try:
68
+ time.tzset() # type: ignore # pylint: disable=no-member
69
+ except Exception:
70
+ # Windows
71
+ logger.warning("Windows, cant run time.tzset()")
72
+
73
+ if True:
74
+ run_cmd(
75
+ "ls -rtl /home/user/.pyenv/versions/3.10.13/lib/python3.10/site-packages/bitsandbytes"
76
+ )
77
+
78
+ logger.info("lsb_release -a")
79
+ ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding="utf8")
80
+ if ret.stdout:
81
+ rich.print(ret.stdout)
82
+ if ret.stderr:
83
+ rich.print("[red bold]" + ret.stdout)
84
+
85
+ logger.info("nvidia-smi")
86
+ ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding="utf8")
87
+ if ret.stdout:
88
+ rich.print(ret.stdout)
89
+ if ret.stderr:
90
+ rich.print("[red bold]" + ret.stdout)
91
+ # raise SystemExit("Interrupt by intentioin")
92
+
93
+
94
+ if not torch.cuda.is_available():
95
+ raise gr.Error("torch.cuda.is_available() is False, cant continue...")
96
+
97
+ model_name = "tangger/Qwen-7B-Chat" # try
98
+ model_name = "Qwen/Qwen-7B-Chat" # gone!
99
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
100
+
101
+ n_gpus = torch.cuda.device_count()
102
+ try:
103
+ _ = f"{int(torch.cuda.mem_get_info()[0]/1024**3)-2}GB"
104
+ except AssertionError:
105
+ _ = 0
106
+ max_memory = {i: _ for i in range(n_gpus)}
107
+
108
+ del sys
109
+ # logger.remove() # to turn on trace
110
+ # logger.add(sys.stderr, level="TRACE")
111
+ # logger.trace(f"{chat_history=}")
112
+
113
+
114
+ def gen_model(model_name: str):
115
+ model = AutoModelForCausalLM.from_pretrained(
116
+ model_name,
117
+ trust_remote_code=True,
118
+ # device_map="auto",
119
+ device_map={"": 0},
120
+ # load_in_4bit=True,
121
+ load_in_8bit=True,
122
+ max_memory=max_memory,
123
+ fp16=True,
124
+ torch_dtype=torch.float16,
125
+ bnb_4bit_quant_type="nf4",
126
+ bnb_4bit_compute_dtype=torch.bfloat16,
127
+ )
128
+ model = model.eval()
129
+ model.generation_config = GenerationConfig.from_pretrained(
130
+ model_name,
131
+ trust_remote_code=True,
132
+ )
133
+ return model
134
+
135
+
136
+ def user_clear(message, chat_history):
137
+ """Gen a response, clear message in user textbox."""
138
+ logger.debug(f"{message=}")
139
+
140
+ try:
141
+ chat_history.append([message, ""])
142
+ except Exception:
143
+ chat_history = deque([message, ""], maxlen=5)
144
+
145
+ logger.trace(f"{chat_history=}")
146
+ return "", chat_history
147
+
148
+
149
+ def user(message, chat_history):
150
+ """Gen a response."""
151
+ logger.debug(f"{message=}")
152
+ logger.trace(f"{chat_history=}")
153
+
154
+ try:
155
+ chat_history.append([message, ""])
156
+ except Exception:
157
+ chat_history = deque([message, ""], maxlen=5)
158
+ return message, chat_history
159
+
160
+
161
+ # for rerun in tests
162
+ model = None
163
+ gc.collect()
164
+ torch.cuda.empty_cache()
165
+
166
+ if not torch.cuda.is_available():
167
+ # raise gr.Error("GPU not available, cant run. Turn on GPU and retry")
168
+ raise SystemExit("GPU not available, cant run. Turn on GPU and retry")
169
+
170
+ model = gen_model(model_name)
171
+
172
+
173
+ def bot(chat_history, **kwargs):
174
+ try:
175
+ message = chat_history[-1][0]
176
+ except Exception as exc:
177
+ logger.error(f"{chat_history=}: {exc}")
178
+ return chat_history
179
+ logger.debug(f"{chat_history=}")
180
+ try:
181
+ _ = """
182
+ response, chat_history = model.chat(
183
+ tokenizer,
184
+ message,
185
+ history=chat_history,
186
+ temperature=0.7,
187
+ repetition_penalty=1.2,
188
+ # max_length=128,
189
+ )
190
+ """
191
+ logger.debug("run model.chat...")
192
+ model.generation_config.update(**kwargs)
193
+ response, chat_history = model.chat(
194
+ tokenizer,
195
+ message,
196
+ chat_history[:-1],
197
+ # **kwargs,
198
+ )
199
+ del response
200
+ return chat_history
201
+ except Exception as exc:
202
+ logger.error(exc)
203
+ chat_history[:-1].append(["message", str(exc)])
204
+ return chat_history
205
+
206
+
207
+ def bot_stream(chat_history, **kwargs):
208
+ logger.trace(f"{kwargs=}")
209
+
210
+ # somehow, empty chat_history
211
+ if chat_history is None or not chat_history:
212
+ logger.trace(f" *** {chat_history=}")
213
+ chat_history.append(["", ""])
214
+
215
+ try:
216
+ message = chat_history[-1][0]
217
+ except Exception as exc:
218
+ logger.error(f"{chat_history=}: {exc}")
219
+ raise gr.Error(f"{chat_history=}")
220
+ # yield chat_history
221
+
222
+ # for elm in model.chat_stream(tokenizer, message, chat_history):
223
+ model.generation_config.update(**kwargs)
224
+ response = ""
225
+ for elm in model.chat_stream(tokenizer, message, chat_history):
226
+ chat_history[-1] = [message, elm]
227
+ response = elm
228
+ yield chat_history
229
+ logger.debug(f"{response=}")
230
+ logger.debug(f"{model.generation_config=}")
231
+
232
+
233
+ SYSTEM_PROMPT = "You are a helpful assistant."
234
+ MAX_MAX_NEW_TOKENS = 2048 # sequence length 2048
235
+ MAX_NEW_TOKENS = 256
236
+
237
+
238
+ @dataclass
239
+ class Config:
240
+ max_new_tokens: int = MAX_NEW_TOKENS
241
+ repetition_penalty: float = 1.1
242
+ temperature: float = 1.0
243
+ top_k: int = 0
244
+ top_p: float = 0.9
245
+
246
+
247
+ # stats_default = SimpleNamespace(llm=model, system_prompt=SYSTEM_PROMPT, config=Config())
248
+ stats_default = SimpleNamespace(llm=None, system_prompt=SYSTEM_PROMPT, config=Config())
249
+
250
+
251
+ # input max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
252
+ def api_fn( # pylint: disable=too-many-arguments
253
+ input_text: Optional[str],
254
+ # max_length: int = 256,
255
+ max_new_tokens: int = stats_default.config.max_new_tokens,
256
+ temperature: float = stats_default.config.temperature,
257
+ repetition_penalty: float = stats_default.config.repetition_penalty,
258
+ top_k: int = stats_default.config.top_k,
259
+ top_p: int = stats_default.config.top_p,
260
+ system_prompt: Optional[str] = None,
261
+ history: Optional[List[str]] = None,
262
+ ):
263
+ if input_text is None:
264
+ input_text = ""
265
+ try:
266
+ input_text = str(input_text).strip()
267
+ except Exception as exc:
268
+ logger.error(exc)
269
+ input_text = ""
270
+ if not input_text:
271
+ return ""
272
+ if history is None:
273
+ history = []
274
+ try:
275
+ temperature = float(temperature)
276
+ except Exception:
277
+ temperature = stats_default.config.temperature
278
+
279
+ if system_prompt is None:
280
+ system_prompt = stats_default.system_prompt
281
+ # if max_length < 10: max_length = 4096
282
+ if max_new_tokens < 10:
283
+ max_new_tokens = stats_default.config.max_new_tokens
284
+ if top_p < 0.1 or top_p > 1:
285
+ top_p = stats_default.config.top_p
286
+ if temperature <= 0.5:
287
+ temperature = stats_default.config.temperature
288
+
289
+ _ = {
290
+ "max_new_tokens": max_new_tokens,
291
+ "temperature": temperature,
292
+ "repetition_penalty": repetition_penalty,
293
+ "top_k": top_k,
294
+ "top_p": top_p,
295
+ }
296
+ model.generation_config.update(**_)
297
+ try:
298
+ res, _ = model.chat(
299
+ tokenizer,
300
+ input_text,
301
+ history=history,
302
+ # max_length=max_length,
303
+ # append_history=False,
304
+ )
305
+ # logger.debug(f"{res=} \n{_=}")
306
+ except Exception as exc:
307
+ logger.error(f"{exc=}")
308
+ res = str(exc)
309
+
310
+ logger.debug(f"api {res=}")
311
+ logger.debug(f"api {model.generation_config=}")
312
+
313
+ return res
314
+
315
+
316
+ theme = gr.themes.Soft(text_size="sm")
317
+ with gr.Blocks(
318
+ theme=theme,
319
+ title=model_name.lower(),
320
+ css=css,
321
+ ) as block:
322
+ stats = gr.State(stats_default)
323
+
324
+ # would this reset model?
325
+ model.generation_config = GenerationConfig.from_pretrained(
326
+ model_name,
327
+ trust_remote_code=True,
328
+ )
329
+ config = asdict(stats.value.config)
330
+
331
+ def bot_stream_state(chat_history):
332
+ logger.trace(f"{chat_history=}")
333
+ yield from bot_stream(chat_history, **config)
334
+
335
+ with gr.Accordion("🎈 Info", open=False):
336
+ gr.Markdown(
337
+ dedent(
338
+ f"""
339
+ ## {model_name.lower()}
340
+
341
+ * temperature range: .51 and up; higher temperature implies more randomness. Suggested temperature for chatting and creative writing is around 1.1 while it should be set to 0.51-1.0 for summarizing and translation.
342
+ * Set `repetition_penalty` to 2.1 or higher for a chatty conversation (more unpredictable and undesirable output). Lower it to 1.1 or smaller if more focused anwsers are desired (for example for translations or fact-oriented queries).
343
+ * Smaller `top_k` probably will result in smoothier sentences.
344
+ (`top_k=0` is equivalent to `top_k` equal to very very big though.) Consult `transformers` documentation for more details.
345
+ * An API is available at https://mikeee-qwen-7b-chat.hf.space/ that can be queried, e.g., in python
346
+ ```python
347
+ from gradio_client import Client
348
+
349
+ client = Client("https://mikeee-qwen-7b-chat.hf.space/")
350
+
351
+ result = client.predict(
352
+ "你好!", # user prompt
353
+ 256, # max_new_tokens
354
+ 1.2, # temperature
355
+ 1.1, # repetition_penalty
356
+ 0, # top_k
357
+ 0.9, # top_p
358
+ "You are a helpful assistant.", # system_prompt
359
+ None, # history
360
+ api_name="/api"
361
+ )
362
+ print(result)
363
+ ```
364
+ or in javascript
365
+ ```js
366
+ import {{ client }} from "@gradio/client";
367
+
368
+ const app = await client("https://mikeee-qwen-7b-chat.hf.space/");
369
+ const result = await app.predict("api", [...]);
370
+ console.log(result.data);
371
+ ```
372
+ Check documentation and examples by clicking `Use via API` at the very bottom of [https://huggingface.co/spaces/mikeee/qwen-7b-chat](https://huggingface.co/spaces/mikeee/qwen-7b-chat).
373
+
374
+ <p></p>
375
+ Most examples are meant for another model.
376
+ You probably should try to test
377
+ some related prompts. System prompt can be changed in Advaned Options as well."""
378
+ ),
379
+ elem_classes="xsmall",
380
+ )
381
+
382
+ chatbot = gr.Chatbot(height=500, value=deque([], maxlen=5)) # type: ignore
383
+
384
+ with gr.Row():
385
+ with gr.Column(scale=5):
386
+ msg = gr.Textbox(
387
+ label="Chat Message Box",
388
+ placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
389
+ show_label=False,
390
+ # container=False,
391
+ lines=4,
392
+ max_lines=30,
393
+ show_copy_button=True,
394
+ # ).style(container=False)
395
+ )
396
+ with gr.Column(scale=1, min_width=50):
397
+ with gr.Row():
398
+ submit = gr.Button("Submit", elem_classes="xsmall")
399
+ stop = gr.Button("Stop", visible=True)
400
+ clear = gr.Button("Clear History", visible=True)
401
+
402
+ msg_submit_event = msg.submit(
403
+ # fn=conversation.user_turn,
404
+ fn=user,
405
+ inputs=[msg, chatbot],
406
+ outputs=[msg, chatbot],
407
+ queue=True,
408
+ show_progress="full",
409
+ # api_name=None,
410
+ ).then(bot_stream_state, chatbot, chatbot, queue=True)
411
+ submit_click_event = submit.click(
412
+ # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
413
+ fn=user_clear, # clear msg
414
+ inputs=[msg, chatbot],
415
+ outputs=[msg, chatbot],
416
+ queue=True,
417
+ show_progress="full",
418
+ # api_name=None,
419
+ ).then(bot_stream_state, chatbot, chatbot, queue=True)
420
+ stop.click(
421
+ fn=None,
422
+ inputs=None,
423
+ outputs=None,
424
+ cancels=[msg_submit_event, submit_click_event],
425
+ queue=False,
426
+ )
427
+ clear.click(lambda: None, None, chatbot, queue=False)
428
+
429
+ with gr.Accordion(label="Advanced Options", open=False):
430
+ system_prompt = gr.Textbox(
431
+ label="System prompt",
432
+ value=stats_default.system_prompt,
433
+ lines=3,
434
+ visible=True,
435
+ )
436
+ max_new_tokens = gr.Slider(
437
+ label="Max new tokens",
438
+ minimum=1,
439
+ maximum=MAX_MAX_NEW_TOKENS,
440
+ step=1,
441
+ value=stats_default.config.max_new_tokens,
442
+ )
443
+ repetition_penalty = gr.Slider(
444
+ label="Repetition penalty",
445
+ minimum=0.1,
446
+ maximum=40.0,
447
+ step=0.1,
448
+ value=stats_default.config.repetition_penalty,
449
+ )
450
+ temperature = gr.Slider(
451
+ label="Temperature",
452
+ minimum=0.51,
453
+ maximum=40.0,
454
+ step=0.1,
455
+ value=stats_default.config.temperature,
456
+ )
457
+ top_p = gr.Slider(
458
+ label="Top-p (nucleus sampling)",
459
+ minimum=0.05,
460
+ maximum=1.0,
461
+ step=0.05,
462
+ value=stats_default.config.top_p,
463
+ )
464
+ top_k = gr.Slider(
465
+ label="Top-k",
466
+ minimum=0,
467
+ maximum=1000,
468
+ step=1,
469
+ value=stats_default.config.top_k,
470
+ )
471
+
472
+ def system_prompt_fn(system_prompt):
473
+ stats.value.system_prompt = system_prompt
474
+ logger.debug(f"{stats.value.system_prompt=}")
475
+
476
+ def max_new_tokens_fn(max_new_tokens):
477
+ stats.value.config.max_new_tokens = max_new_tokens
478
+ logger.debug(f"{stats.value.config.max_new_tokens=}")
479
+
480
+ def repetition_penalty_fn(repetition_penalty):
481
+ stats.value.config.repetition_penalty = repetition_penalty
482
+ logger.debug(f"{stats.value=}")
483
+
484
+ def temperature_fn(temperature):
485
+ stats.value.config.temperature = temperature
486
+ logger.debug(f"{stats.value=}")
487
+
488
+ def top_p_fn(top_p):
489
+ stats.value.config.top_p = top_p
490
+ logger.debug(f"{stats.value=}")
491
+
492
+ def top_k_fn(top_k):
493
+ stats.value.config.top_k = top_k
494
+ logger.debug(f"{stats.value=}")
495
+
496
+ system_prompt.change(system_prompt_fn, system_prompt)
497
+ max_new_tokens.change(max_new_tokens_fn, max_new_tokens)
498
+ repetition_penalty.change(repetition_penalty_fn, repetition_penalty)
499
+ temperature.change(temperature_fn, temperature)
500
+ top_p.change(top_p_fn, top_p)
501
+ top_k.change(top_k_fn, top_k)
502
+
503
+ def reset_fn(stats_):
504
+ logger.debug("reset_fn")
505
+ stats_ = gr.State(stats_default)
506
+ logger.debug(f"{stats_.value=}")
507
+ return (
508
+ stats_,
509
+ stats_default.system_prompt,
510
+ stats_default.config.max_new_tokens,
511
+ stats_default.config.repetition_penalty,
512
+ stats_default.config.temperature,
513
+ stats_default.config.top_p,
514
+ stats_default.config.top_k,
515
+ )
516
+
517
+ reset_btn = gr.Button("Reset")
518
+ reset_btn.click(
519
+ reset_fn,
520
+ stats,
521
+ [
522
+ stats,
523
+ system_prompt,
524
+ max_new_tokens,
525
+ repetition_penalty,
526
+ temperature,
527
+ top_p,
528
+ top_k,
529
+ ],
530
+ )
531
+
532
+ with gr.Accordion("Example inputs", open=True):
533
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
534
+ examples = gr.Examples(
535
+ examples=example_list,
536
+ inputs=[msg],
537
+ examples_per_page=60,
538
+ )
539
+ with gr.Accordion("Disclaimer", open=False):
540
+ _ = model_name.lower()
541
+ gr.Markdown(
542
+ f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
543
+ f"factually accurate information. {_} was trained on various public datasets; while great efforts "
544
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
545
+ "biased, or otherwise offensive outputs.",
546
+ elem_classes=["disclaimer"],
547
+ )
548
+
549
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
550
+ input_text = gr.Text()
551
+ api_history = gr.Chatbot(value=[])
552
+ api_btn = gr.Button("Go", variant="primary")
553
+ out_text = gr.Text()
554
+
555
+ # api_fn args order
556
+ # input_text max_new_tokens temperature repetition_penalty top_k top_p system_prompt history
557
+ api_btn.click(
558
+ api_fn,
559
+ [
560
+ input_text,
561
+ max_new_tokens,
562
+ temperature,
563
+ repetition_penalty,
564
+ top_k,
565
+ top_p,
566
+ system_prompt,
567
+ api_history, # dont know how to pass this in gradio_client.Client calls
568
+ ],
569
+ out_text,
570
+ api_name="api",
571
+ )
572
+
573
+
574
+ if __name__ == "__main__":
575
+ logger.info("Just record start time")
576
+
577
+ _ = """
578
+ ret = sp.run("lsb_release -a", capture_output=1, check=0, shell=1, encoding='utf8')
579
+ if ret.stdout:
580
+ rich.print(ret.stdout)
581
+ if ret.stderr:
582
+ rich.print("[red bold]" + ret.stdout)
583
+
584
+ ret = sp.run("nvidia-smi", capture_output=1, check=0, shell=1, encoding='utf8')
585
+ if ret.stdout:
586
+ rich.print(ret.stdout)
587
+ if ret.stderr:
588
+ rich.print("[red bold]" + ret.stdout)
589
+ raise SystemExit("Interrupt by intentioin")
590
+ # """
591
+
592
+ block.queue(max_size=8).launch(debug=True)
requirements.txt CHANGED
@@ -1,14 +1,4 @@
1
  gradio>=5.0,<6
2
  spaces
3
- transformers
4
- accelerate
5
- safetensors
6
- sentencepiece
7
- einops
8
- tiktoken
9
- scipy
10
- loguru
11
- about-time
12
- rich
13
- bitsandbytes
14
- transformers_stream_generator
 
1
  gradio>=5.0,<6
2
  spaces
3
+ huggingface_hub
4
+ requests