import os
import io
import time
import sys
import subprocess
import requests
from PIL import Image, ImageSequence
import gradio as gr
# llama-cpp-python import
try:
from llama_cpp import Llama
except Exception as e:
raise RuntimeError("llama-cpp-python import failed: " + str(e))
MODEL_DIR = "model"
EXPECTED_TARGET = os.path.join(MODEL_DIR, "llama-joycaption-q4_k_m.gguf")
# Candidate direct-download URLs (try in order)
CANDIDATES = [
# Jasaga7818 copy (often a direct GGUF)
("https://huggingface.co/Jasaga7818/llama-joycaption-beta-one-hf-llava-Q4_K_M-GGUF/resolve/main/llama-joycaption-beta-one-hf-llava-q4_k_m.gguf",
EXPECTED_TARGET),
# mradermacher (alternate host)
("https://huggingface.co/mradermacher/llama-joycaption-beta-one-hf-llava-GGUF/resolve/main/llama-joycaption-beta-one-hf-llava-q4_k_m.gguf",
EXPECTED_TARGET),
# Fallback to Q4_K_S (Jasaga)
("https://huggingface.co/Jasaga7818/llama-joycaption-beta-one-hf-llava-Q4_K_M-GGUF/resolve/main/llama-joycaption-beta-one-hf-llava-q4_k_s.gguf",
os.path.join(MODEL_DIR, "llama-joycaption-q4_k_s.gguf")),
("https://huggingface.co/mradermacher/llama-joycaption-beta-one-hf-llava-GGUF/resolve/main/llama-joycaption-beta-one-hf-llava-q4_k_s.gguf",
os.path.join(MODEL_DIR, "llama-joycaption-q4_k_s.gguf")),
]
def download_curl(url: str, path: str) -> bool:
os.makedirs(os.path.dirname(path), exist_ok=True)
try:
# Use curl for resume support and progress in logs
subprocess.check_call(["curl", "-L", "-C", "-", "-o", path, url])
return True
except Exception:
try:
if os.path.exists(path):
os.remove(path)
except Exception:
pass
return False
def is_valid_gguf(path: str) -> bool:
# GGUF files start with "GGUF" in ASCII at offset 0 (0x47 0x47 0x55 0x46).
# Some converted uploads may be HTML pages or redirects; check header.
try:
with open(path, "rb") as f:
head = f.read(8)
return head.startswith(b"GGUF")
except Exception:
return False
def ensure_model() -> str:
# If already present (and valid), use it.
if os.path.exists(EXPECTED_TARGET) and is_valid_gguf(EXPECTED_TARGET):
sys.stderr.write(f"Model already present and valid at {EXPECTED_TARGET}\n")
return EXPECTED_TARGET
sys.stderr.write("Model not found locally or invalid, attempting download (several GB)...\n")
for url, dest in CANDIDATES:
sys.stderr.write(f"Attempting download: {url} -> {dest}\n")
if download_curl(url, dest):
sys.stderr.write(f"Downloaded candidate to {dest}; verifying header...\n")
if is_valid_gguf(dest):
# If candidate wasn't the expected filename, create symlink so rest of code can use EXPECTED_TARGET.
if os.path.abspath(dest) != os.path.abspath(EXPECTED_TARGET):
try:
if os.path.exists(EXPECTED_TARGET):
os.remove(EXPECTED_TARGET)
os.symlink(os.path.basename(dest), EXPECTED_TARGET)
sys.stderr.write(f"Created symlink {EXPECTED_TARGET} -> {os.path.basename(dest)}\n")
except Exception:
# fallback: copy
try:
import shutil
shutil.copyfile(dest, EXPECTED_TARGET)
sys.stderr.write(f"Copied {dest} to {EXPECTED_TARGET}\n")
except Exception:
sys.stderr.write("Warning: failed to symlink or copy candidate to expected filename.\n")
sys.stderr.write("Model verified as GGUF and ready.\n")
return EXPECTED_TARGET
else:
sys.stderr.write("Downloaded file is not a valid GGUF (header mismatch). Removing and trying next.\n")
try:
os.remove(dest)
except Exception:
pass
else:
sys.stderr.write("Download failed for candidate; trying next.\n")
raise FileNotFoundError("Failed to download a valid GGUF model from candidates. Check URLs and repo availability.")
# Ensure model exists and is a GGUF before importing/initializing Llama
MODEL_PATH = ensure_model()
if not os.path.exists(MODEL_PATH):
raise FileNotFoundError(f"Model not found at {MODEL_PATH} after download attempt.")
def download_bytes(url: str, timeout: int = 30) -> bytes:
with requests.get(url, stream=True, timeout=timeout) as r:
r.raise_for_status()
return r.content
def load_first_frame_from_bytes(raw: bytes):
img = Image.open(io.BytesIO(raw))
if getattr(img, "is_animated", False):
img = next(ImageSequence.Iterator(img))
if img.mode != "RGB":
img = img.convert("RGB")
return img
def make_prompt_for_image(image_path: str, user_prompt: str = "Describe the image."):
# JoyCaption-style multimodal GGUFs accept
{path}
return f"
{image_path}\nUser: {user_prompt}\nAssistant:"
# Initialize model (low-resource options)
print("Loading GGUF model (this can take 30–120s)...", file=sys.stderr)
# Adjust n_threads for the Space CPU; increase if you know you have more cores available.
llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=2)
def generate_caption_from_url(url: str, prompt: str = "Describe the image."):
if not url:
return "No URL provided."
try:
raw = download_bytes(url)
except Exception as e:
return f"Download error: {e}"
try:
img = load_first_frame_from_bytes(raw)
except Exception as e:
return f"Image processing error: {e}"
tmp_dir = "/tmp/joycap"
os.makedirs(tmp_dir, exist_ok=True)
ts = int(time.time() * 1000)
tmp_path = os.path.join(tmp_dir, f"{ts}.jpg")
try:
img.save(tmp_path, format="JPEG", quality=85)
except Exception as e:
return f"Failed to save temp image: {e}"
prompt_full = make_prompt_for_image(tmp_path, prompt)
try:
resp = llm.create(
prompt=prompt_full,
max_tokens=256,
temperature=0.2,
top_p=0.95,
stop=["User:", "Assistant:"],
)
text = resp.get("choices", [{}])[0].get("text", "").strip()
return text or "No caption generated."
except Exception as e:
return f"Inference error: {e}"
finally:
try:
os.remove(tmp_path)
except Exception:
pass
iface = gr.Interface(
fn=generate_caption_from_url,
inputs=[
gr.Textbox(label="Image URL", placeholder="https://example.com/photo.jpg"),
gr.Textbox(label="Prompt (optional)", value="Describe the image."),
],
outputs=gr.Textbox(label="Generated caption"),
title="JoyCaption GGUF (Q4_K)",
description="Runs a quantized JoyCaption GGUF locally via llama.cpp (no external API).",
)
if __name__ == "__main__":
iface.launch(server_name="0.0.0.0", server_port=7860)