|
|
|
|
|
|
|
|
import os |
|
|
from collections.abc import Iterator |
|
|
from threading import Thread |
|
|
|
|
|
import gradio as gr |
|
|
import spaces |
|
|
import torch |
|
|
from peft import PeftModel |
|
|
from transformers import (AutoModelForCausalLM, AutoTokenizer, |
|
|
TextIteratorStreamer, pipeline) |
|
|
|
|
|
DESCRIPTION = "# 真空ジェネレータ (v3)\n<p>Imitate 真空 (@vericava)'s posts interactively</p>" |
|
|
|
|
|
if not torch.cuda.is_available(): |
|
|
DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>" |
|
|
|
|
|
MAX_MAX_NEW_TOKENS = 128 |
|
|
DEFAULT_MAX_NEW_TOKENS = 64 |
|
|
MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "32768")) |
|
|
|
|
|
|
|
|
if torch.cuda.is_available(): |
|
|
my_pipeline=pipeline( |
|
|
task="text-generation", |
|
|
model="vericava/llm-jp-3-vericava-posts-v1", |
|
|
do_sample=True, |
|
|
num_beams=1, |
|
|
) |
|
|
|
|
|
@spaces.GPU |
|
|
@torch.inference_mode() |
|
|
def generate( |
|
|
message: str, |
|
|
chat_history, |
|
|
max_new_tokens: int = 1024, |
|
|
temperature: float = 0.7, |
|
|
top_p: float = 0.95, |
|
|
top_k: int = 50, |
|
|
repetition_penalty: float = 1.0, |
|
|
) -> Iterator[str]: |
|
|
user_input = " ".join(message.strip().split("\n")) |
|
|
|
|
|
user_input = user_input if ( |
|
|
user_input.endswith("。") |
|
|
or user_input.endswith("?") |
|
|
or user_input.endswith("!") |
|
|
or user_input.endswith("？") |
|
|
or user_input.endswith("！") |
|
|
) else user_input + "。" |
|
|
|
|
|
output = my_pipeline( |
|
|
user_input, |
|
|
temperature=temperature * 1.0, |
|
|
max_new_tokens=max_new_tokens, |
|
|
repetition_penalty=repetition_penalty * 1.0, |
|
|
top_k=top_k, |
|
|
top_p=top_p * 1.0, |
|
|
)[-1]["generated_text"] |
|
|
print(output) |
|
|
gen_text = output[len(user_input):] |
|
|
|
|
|
|
|
|
yield gen_text |
|
|
|
|
|
demo = gr.ChatInterface( |
|
|
fn=generate, |
|
|
type="messages", |
|
|
additional_inputs_accordion=gr.Accordion(label="詳細設定", open=False), |
|
|
additional_inputs=[ |
|
|
gr.Slider( |
|
|
label="Max new tokens", |
|
|
minimum=1, |
|
|
maximum=MAX_MAX_NEW_TOKENS, |
|
|
step=1, |
|
|
value=DEFAULT_MAX_NEW_TOKENS, |
|
|
), |
|
|
gr.Slider( |
|
|
label="Temperature", |
|
|
minimum=0.1, |
|
|
maximum=4.0, |
|
|
step=0.1, |
|
|
value=1.0, |
|
|
), |
|
|
gr.Slider( |
|
|
label="Top-p (nucleus sampling)", |
|
|
minimum=0.05, |
|
|
maximum=1.0, |
|
|
step=0.05, |
|
|
value=0.90, |
|
|
), |
|
|
gr.Slider( |
|
|
label="Top-k", |
|
|
minimum=1, |
|
|
maximum=100, |
|
|
step=1, |
|
|
value=20, |
|
|
), |
|
|
gr.Slider( |
|
|
label="Repetition penalty", |
|
|
minimum=1.0, |
|
|
maximum=4.0, |
|
|
step=0.05, |
|
|
value=2.0, |
|
|
), |
|
|
], |
|
|
stop_btn=None, |
|
|
examples=[ |
|
|
["おはよ"], |
|
|
["えらいね"], |
|
|
["にゃん"], |
|
|
["よしよし"], |
|
|
], |
|
|
description=DESCRIPTION, |
|
|
css_paths="style.css", |
|
|
fill_height=True, |
|
|
) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
demo.launch() |
|
|
|
