Create app.py

app.py

@@ -0,0 +1,34 @@
+import gradio as gr
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+# Model download & load
+MODEL_REPO = "dexcommunity/indexQ4"
+MODEL_FILE = "indexq4.gguf"
+model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
+
+# Initialize Llama
+llm = Llama(model_path=model_path, n_ctx=2048, n_threads=2)
+
+def generate_api(prompt, max_tokens=256, temp=0.7):
+    # Gemma 2B Chat Template
+    formatted_prompt = f"<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model\n"
+    
+    output = llm(
+        formatted_prompt,
+        max_tokens=int(max_tokens),
+        temperature=float(temp),
+        stop=["<end_of_turn>"],
+        echo=False
+    )
+    return output["choices"][0]["text"].strip()
+
+# Gradio interface (API only mode)
+demo = gr.Interface(
+    fn=generate_api,
+    inputs=[gr.Textbox(label="Prompt"), gr.Slider(1, 512, 256), gr.Slider(0, 1, 0.7)],
+    outputs=gr.Textbox(label="Reply")
+)
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)