File size: 4,715 Bytes
ac021c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | #!/usr/bin/env python3
"""
Janus-35B — vision (image-text-to-text) via llama-cpp-python.
Why this script exists:
Ollama's Go engine has the qwen35 / qwen35moe arch entries (text
inference works on 0.24+), but the C++ llama.cpp fallback that
Ollama switches to when an mmproj is attached still lacks them.
Both `FROM mmproj.gguf` and `ADAPTER mmproj.gguf` fail at first
inference with:
unknown model architecture: 'qwen35moe'
See ollama/ollama#15898 (still open). Until that lands, vision via
Ollama is broken for Qwen 3.5 / 3.6 while text remains fine.
Upstream ggml-org/llama.cpp **does** have the architecture across
both code paths, so vision works fine via llama.cpp directly. This
script uses the python binding.
Install:
pip install llama-cpp-python pillow
# GPU offload? rebuild with the matching backend:
# CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python --no-binary :all:
# CMAKE_ARGS="-DGGML_METAL=on" pip install llama-cpp-python --no-binary :all:
# CMAKE_ARGS="-DGGML_HIPBLAS=on" pip install llama-cpp-python --no-binary :all:
Files you need:
1. A text GGUF (any quant): the bundled Janus-35B-A3B.Q4_K_M.gguf (~19 GB),
or another quant from llmfan46/Qwen3.6-35B-A3B-uncensored-heretic-GGUF
2. A vision projector: Qwen3.6-35B-A3B-uncensored-heretic-mmproj-BF16.gguf (~903 MB)
— run ./scripts/fetch_vision.sh to pull it.
Usage:
python llama_cpp_vision.py \
--gguf Janus-35B-A3B.Q4_K_M.gguf \
--mmproj Qwen3.6-35B-A3B-uncensored-heretic-mmproj-BF16.gguf \
--image /path/to/photo.jpg \
--prompt "What is in this image? Be specific."
# CLI alternative without python binding (ships with llama.cpp):
# llama-mtmd-cli \
# -m Janus-35B-A3B.Q4_K_M.gguf \
# --mmproj Qwen3.6-35B-A3B-uncensored-heretic-mmproj-BF16.gguf \
# --image photo.jpg \
# -p "Describe this image."
"""
from __future__ import annotations
import argparse
import base64
import sys
from pathlib import Path
try:
from llama_cpp import Llama
from llama_cpp.llama_chat_format import Qwen25VLChatHandler
except ImportError: # pragma: no cover
sys.exit(
"Missing llama-cpp-python (>=0.3 with VL handlers).\n"
" pip install --upgrade llama-cpp-python pillow"
)
JANUS_SYSTEM = (
"You are Janus, a precise vision-language assistant. Describe images "
"accurately, do not invent details, and ground every claim in the "
"pixels you can actually see."
)
def encode_image_data_uri(path: Path) -> str:
suffix = path.suffix.lower().lstrip(".")
mime = {"jpg": "jpeg", "jpeg": "jpeg", "png": "png", "webp": "webp", "gif": "gif"}.get(suffix, "jpeg")
return f"data:image/{mime};base64,{base64.b64encode(path.read_bytes()).decode()}"
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--gguf", required=True, help="Text GGUF (e.g. Janus-35B-A3B.Q4_K_M.gguf).")
ap.add_argument("--mmproj", required=True, help="Vision projector GGUF (mmproj-BF16.gguf).")
ap.add_argument("--image", required=True, help="Image to analyze.")
ap.add_argument("--prompt", default="Describe this image in detail.")
ap.add_argument("--ctx", type=int, default=8192)
ap.add_argument(
"--gpu-layers",
type=int,
default=0,
help="Layers to offload to GPU (-1 or 99 = all).",
)
ap.add_argument("--max-tokens", type=int, default=512)
args = ap.parse_args()
image_path = Path(args.image)
if not image_path.exists():
sys.exit(f"Image not found: {image_path}")
# Qwen 2.5 VL chat handler is the closest match shipped with
# llama-cpp-python; Qwen 3.5/3.6 vision uses the same projector layout.
# If/when llama-cpp-python ships a Qwen3VLChatHandler, swap it in.
handler = Qwen25VLChatHandler(clip_model_path=args.mmproj)
llm = Llama(
model_path=args.gguf,
chat_handler=handler,
n_ctx=args.ctx,
n_gpu_layers=args.gpu_layers,
verbose=False,
)
out = llm.create_chat_completion(
messages=[
{"role": "system", "content": JANUS_SYSTEM},
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": encode_image_data_uri(image_path)}},
{"type": "text", "text": args.prompt},
],
},
],
temperature=0.6,
top_p=0.95,
top_k=20,
repeat_penalty=1.05,
max_tokens=args.max_tokens,
)
print(out["choices"][0]["message"]["content"])
if __name__ == "__main__":
main()
|