create app

app.py

@@ -0,0 +1,49 @@
+import gradio as gr
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from datetime import datetime
+
+print('{}:loading...'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+
+tokenizer = AutoTokenizer.from_pretrained('microsoft/Phi-3-mini-128k-instruct')
+model = AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-mini-128k-instruct', torch_dtype='auto', trust_remote_code=True)
+
+if torch.cuda.is_available():
+	model.half()
+	model = model.to('cuda')
+
+generator = pipeline('text-generation', model=model, tokenizer=tokenizer, device=model.device)
+print('{}:done.'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+
+def generate(input_text, maxlen):
+	messages = [
+		{'role': 'user', 'content': input_text}
+	]
+	output = generator(
+		messages,
+		max_new_tokens=maxlen,
+		return_full_text=False,
+		do_sample=True,
+		temperature=0.7,
+	)
+	generated_text = output[0]['generated_text']
+	return generated_text
+
+with gr.Blocks(title='phi3 demo') as app:
+	gr.Markdown('# Phi3 Demo')
+
+	chatbot = gr.Chatbot(label='answer')
+	msg = gr.Textbox(label='question')
+	maxlen = gr.Slider(minimum=30, maximum=100, value=30, step=1, label='max length')
+	clear = gr.ClearButton([msg, chatbot])
+
+	def respond(message, maxlen, chat_history):
+		if message == '':
+			return '', chat_history
+		bot_message = generate(message, maxlen)
+		chat_history.append((message, bot_message))
+		return '', chat_history
+
+	msg.submit(respond, [msg, maxlen, chatbot], [msg, chatbot], concurrency_limit=20)
+
+app.launch()

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+torch
+torchvision
+torchaudio
+transformers
+sentencepiece