Upload 3 files

app.py

@@ -0,0 +1,55 @@
+import gradio as gr
+import torch
+from threading import Thread
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+
+MODEL_ID = "NoesisLab/Spartacus-1B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
+
+
+def respond(message, history):
+    messages = [{"role": "system", "content": "You are Spartacus, a helpful assistant."}]
+    for user_msg, bot_msg in history:
+        messages.append({"role": "user", "content": user_msg})
+        messages.append({"role": "assistant", "content": bot_msg})
+    messages.append({"role": "user", "content": message})
+
+    input_ids = tokenizer.apply_chat_template(
+        messages, add_generation_prompt=True, return_tensors="pt"
+    ).to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    generate_kwargs = dict(
+        input_ids=input_ids,
+        streamer=streamer,
+        max_new_tokens=1024,
+        temperature=0.7,
+        top_p=0.9,
+        do_sample=True,
+    )
+
+    thread = Thread(target=model.generate, kwargs=generate_kwargs)
+    thread.start()
+
+    response = ""
+    for token in streamer:
+        response += token
+        yield response
+
+
+demo = gr.ChatInterface(
+    fn=respond,
+    title="Spartacus Chat",
+    description="Chat with NoesisLab/Spartacus-1B-Instruct",
+    theme=gr.themes.Soft(),
+)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+torch
+transformers
+accelerate