Blame for app.py

import os
import subprocess
import gradio as gr
import spaces

MODEL_REPO = "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive"
MODEL_FILE = "Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf"
MMPROJ_FILE = "mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf"

MODEL_PATH = os.path.join("/tmp", MODEL_FILE)
MMPROJ_PATH = os.path.join("/tmp", MMPROJ_FILE)

server_process = None

LLAMA_DIR = "/tmp/llama"
LLAMA_SERVER = os.path.join(LLAMA_DIR, "llama-server")
LLAMA_RELEASE = "b10182"
LLAMA_ARCHIVE = f"llama.cpp-{LLAMA_RELEASE}-cuda-12.8-amd64.tar.gz"
LLAMA_URL = f"https://github.com/ai-dock/llama.cpp-cuda/releases/download/{LLAMA_RELEASE}/{LLAMA_ARCHIVE}"


def download_llama():
    if os.path.exists(LLAMA_SERVER):
        return

    os.makedirs(LLAMA_DIR, exist_ok=True)

    archive = os.path.join("/tmp", LLAMA_ARCHIVE)

    subprocess.run(
        ["wget", "-q", "--show-progress", "-O", archive, LLAMA_URL],
        check=True,
    )

    subprocess.run(
        ["tar", "-xzf", archive, "-C", LLAMA_DIR, "--strip-components=1"],
        check=True,
    )

    if not os.path.exists(LLAMA_SERVER):
        raise RuntimeError(
            f"llama-server was not found after extraction: {LLAMA_SERVER}"
        )

    os.chmod(LLAMA_SERVER, 0o755)


def download_models():
    from huggingface_hub import hf_hub_download

    if not os.path.exists(MODEL_PATH):
        hf_hub_download(
            repo_id=MODEL_REPO,
            filename=MODEL_FILE,
            local_dir="/tmp",
        )

    if not os.path.exists(MMPROJ_PATH):
        hf_hub_download(
            repo_id=MODEL_REPO,
            filename=MMPROJ_FILE,
            local_dir="/tmp",
        )


@spaces.GPU
def generate(prompt, max_tokens, temperature):
    global server_process

    download_llama()
    download_models()

    if server_process is None or server_process.poll() is not None:
        cmd = [
            LLAMA_SERVER,
            "-m", MODEL_PATH,
            "--mmproj", MMPROJ_PATH,
            "--host", "127.0.0.1",
            "--port", "8080",
            "-c", "32768",
            "-n", "4096",
            "--n-gpu-layers", "99",
            "-t", "2",
        ]

        server_env = os.environ.copy()
        server_env["LD_LIBRARY_PATH"] = (
            LLAMA_DIR
            + ":"
            + server_env.get("LD_LIBRARY_PATH", "")
        )

        server_process = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            text=True,
            env=server_env,
        )

    import time
    import requests

    for _ in range(120):
        try:
            r = requests.get("http://127.0.0.1:8080/health", timeout=1)
            if r.ok:
                break
        except Exception:
            pass
        time.sleep(1)
    else:
        raise RuntimeError("llama-server did not become ready.")

    response = requests.post(
        "http://127.0.0.1:8080/completion",
        json={
            "prompt": prompt,
            "n_predict": int(max_tokens),
            "temperature": float(temperature),
            "stream": False,
        },
        timeout=600,
    )

    response.raise_for_status()
    return response.json().get("content", "")


with gr.Blocks(title="Gemma-4-E4B Uncensored") as demo:
    gr.Markdown("# Gemma-4-E4B-Uncensored-HauhauCS-Aggressive")

    prompt = gr.Textbox(
        label="Prompt",
        placeholder="Enter your prompt...",
        lines=8,
    )

    with gr.Row():
        max_tokens = gr.Slider(
            minimum=64,
            maximum=4096,
            value=1024,
            step=64,
            label="Max tokens",
        )

        temperature = gr.Slider(
            minimum=0.0,
            maximum=2.0,
            value=0.7,
            step=0.05,
            label="Temperature",
        )

    output = gr.Textbox(
        label="Response",
        lines=20,
    )

    generate_button = gr.Button("Generate", variant="primary")

    generate_button.click(
        generate,
        inputs=[prompt, max_tokens, temperature],
        outputs=output,
    )


if __name__ == "__main__":
    demo.launch(ssr_mode=False)