Create app.py with ctransformers

app.py

@@ -0,0 +1,28 @@
+import gradio as gr
+from ctransformers import AutoModelForCausalLM
+
+MODEL_REPO = "bartowski/Hermes-3-Llama-3.1-8B-GGUF"
+MODEL_FILE = "Hermes-3-Llama-3.1-8B-Q4_K_M.gguf"
+
+print("Loading model...")
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_REPO,
+    model_file=MODEL_FILE,
+    hf_model=MODEL_REPO,
+    gpu_layers=0,
+    context_length=4096,
+)
+
+def respond(message, history):
+    prompt = ""
+    for user_msg, bot_msg in history:
+        prompt += f"<|im_start|>user\n{user_msg}\n<|im_end|>\n"
+        prompt += f"<|im_start|>assistant\n{bot_msg}\n<|im_end|>\n"
+    prompt += f"<|im_start|>user\n{message}\n<|im_end|>\n<|im_start|>assistant\n"
+    output = model(prompt, max_new_tokens=512, temperature=0.7, top_p=0.9)
+    return output
+
+iface = gr.ChatInterface(respond)
+
+if __name__ == "__main__":
+    iface.launch()