requirements.txt

gradio
requests

app.py

@@ -0,0 +1,45 @@
+import os
+import subprocess
+import gradio as gr
+
+MODEL_PATH = "model.gguf"
+
+def start_server():
+    if not os.path.exists("llama.cpp"):
+        subprocess.run(["git","clone","https://github.com/ggerganov/llama.cpp"])
+
+    os.chdir("llama.cpp")
+
+    subprocess.run(["make","-j"])
+
+    subprocess.Popen([
+        "./server",
+        "-m",f"../{MODEL_PATH}",
+        "--port","8000",
+        "--ctx-size","4096",
+        "--api"
+    ])
+
+def chat(prompt):
+    import requests
+
+    r = requests.post(
+        "http://localhost:8000/v1/chat/completions",
+        json={
+            "model":"local",
+            "messages":[{"role":"user","content":prompt}]
+        }
+    )
+
+    return r.json()["choices"][0]["message"]["content"]
+
+start_server()
+
+demo = gr.Interface(
+    fn=chat,
+    inputs="text",
+    outputs="text",
+    title="Operon Dev LLM"
+)
+
+demo.launch(server_port=7860)