Create app.py

app.py

@@ -0,0 +1,36 @@
+import gradio as gr
+from transformers import AutoTokenizer
+from petals import AutoDistributedModelForCausalLM
+
+# Choose any model available at https://health.petals.dev
+model_name = "daekeun-ml/Llama-2-ko-instruct-13B"
+
+#daekeun-ml/Llama-2-ko-instruct-13B
+#quantumaikr/llama-2-70b-fb16-korean
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoDistributedModelForCausalLM.from_pretrained(model_name)
+
+# Run the model as if it were on your computer
+def chat(id, npc, text):
+    prom = ""
+    inputs = tokenizer(prom, return_tensors="pt")["input_ids"]
+    outputs = model.generate(inputs, max_new_tokens=100)
+    print(tokenizer.decode(outputs[0]))
+
+
+
+
+
+
+with gr.Blocks() as demo:
+    count = 0
+    aa = gr.Interface(
+      fn=chat,
+      inputs=["text","text","text"],
+      outputs="text",
+      description="chat, ai 응답을 반환합니다. 내부적으로 트랜잭션 생성. \n /run/predict",
+    )
+
+    
+    demo.queue(max_size=32).launch(enable_queue=True)