app.py

import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama

# Download the GGUF from your repo
model_path = hf_hub_download(
    repo_id="Raazi29/Nyaya-Llama-3.1-8B-Indian-Legal",
    filename="nyaya_model_q4-unsloth.Q4_K_M.gguf" # Change to your exact filename
)

# Load model (uses only ~5GB RAM)
llm = Llama(model_path=model_path, n_ctx=2048)

def chat(message, history):
    prompt = f"System: You are Nyaya, an expert Indian Legal AI.\nUser: {message}\nAssistant:"
    response = llm(prompt, max_tokens=512, stop=["User:", "\n"], echo=False)
    return response["choices"][0]["text"]

gr.ChatInterface(chat, title="Nyaya AI (High Speed GGUF)").launch()