wip fix

apps/gradio-space/src/gradio_space/app.py

@@ -57,19 +57,42 @@ def chat(message: str, history: list, model_key: str) -> str:
     return get_backend(model_key).chat(messages)
 
 
+def _runtime_device_hint(model_key: str) -> str:
+    model = get_model_config(model_key)
+    if model.backend == "transformers":
+        try:
+            import torch
+
+            if torch.cuda.is_available():
+                return f"GPU ({torch.cuda.get_device_name(0)})"
+        except ImportError:
+            pass
+        return "CPU"
+    if model.n_gpu_layers > 0:
+        return f"llama.cpp GPU offload ({model.n_gpu_layers} layers)"
+    return "CPU"
+
+
 def warmup(model_key: str | None = None) -> str:
     key = model_key or _app_config.active_model
     model = get_model_config(key)
 
     if _load_state.get(key):
-        return f"Model ready: {model.label}"
+        backend = get_backend(key)
+        device = (
+            backend.device_label
+            if hasattr(backend, "device_label")
+            else _runtime_device_hint(key)
+        )
+        return f"Model ready: {model.label} on {device}"
 
     if key in _load_errors:
         return _load_errors[key]
 
+    device_hint = _runtime_device_hint(key)
     return (
-        f"Preset `{key}` selected ({model.backend}). "
-        "Weights load on the first chat message — this can take a few minutes on CPU."
+        f"Preset `{key}` selected ({model.backend}, {device_hint}). "
+        "Weights load on the first chat message."
     )
 
 

libs/inference/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "llama-cpp-python>=0.3.0",
     "pyyaml>=6.0.2",
     "torch>=2.5.0",
+    "torchvision>=0.20.0",
     "transformers>=5.7.0",
 ]
 

libs/inference/src/inference/config.py

@@ -24,6 +24,7 @@ class ModelConfig:
     model_path: str | None = None
     model_id: str | None = None
     trust_remote_code: bool = False
+    multimodal: bool = False
     n_ctx: int = 4096
     n_gpu_layers: int = 0
     max_tokens: int = 512
@@ -37,6 +38,7 @@ class ModelConfig:
             self.model_path,
             self.model_id,
             self.trust_remote_code,
+            self.multimodal,
             self.n_ctx,
             self.n_gpu_layers,
         )
@@ -148,6 +150,7 @@ def _parse_model_entry(key: str, raw: dict[str, Any]) -> ModelConfig:
         model_path=raw.get("model_path"),
         model_id=raw.get("model_id"),
         trust_remote_code=bool(raw.get("trust_remote_code", False)),
+        multimodal=bool(raw.get("multimodal", False)),
         n_ctx=int(raw.get("n_ctx", 4096)),
         n_gpu_layers=int(raw.get("n_gpu_layers", 0)),
         max_tokens=int(raw.get("max_tokens", 512)),

libs/inference/src/inference/transformers.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from inference.config import ModelConfig
 
 
@@ -6,6 +8,15 @@ class TransformersBackend:
         self._config = config
         self._model = None
         self._tokenizer = None
+        self._processor = None
+        self._device_label: str | None = None
+
+    def _resolve_device(self):
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda", torch.float16, "auto"
+        return "cpu", torch.float32, None
 
     def load(self) -> None:
         if self._model is not None:
@@ -18,44 +29,123 @@ class TransformersBackend:
 
         try:
             import torch
-            from transformers import AutoModelForCausalLM, AutoTokenizer
+            from transformers import (
+                AutoModelForCausalLM,
+                AutoModelForImageTextToText,
+                AutoProcessor,
+                AutoTokenizer,
+            )
         except ImportError as exc:
             raise ImportError(
                 "transformers backend requires torch and transformers. "
                 "Install with: uv sync --all-packages"
             ) from exc
 
-        device = "cuda" if torch.cuda.is_available() else "cpu"
-
-        self._tokenizer = AutoTokenizer.from_pretrained(
-            self._config.model_id,
-            trust_remote_code=self._config.trust_remote_code,
-        )
-        self._model = AutoModelForCausalLM.from_pretrained(
-            self._config.model_id,
-            torch_dtype=torch.float16 if device == "cuda" else torch.float32,
-            device_map="auto" if device == "cuda" else None,
-            trust_remote_code=self._config.trust_remote_code,
+        device, torch_dtype, device_map = self._resolve_device()
+        self._device_label = (
+            f"cuda ({torch.cuda.get_device_name(0)})"
+            if device == "cuda"
+            else "cpu"
         )
+
+        common_kwargs = {
+            "trust_remote_code": self._config.trust_remote_code,
+        }
+        model_kwargs = {
+            **common_kwargs,
+            "torch_dtype": torch_dtype,
+            "device_map": device_map,
+        }
+
+        if self._config.multimodal:
+            self._processor = AutoProcessor.from_pretrained(
+                self._config.model_id,
+                **common_kwargs,
+            )
+            self._model = AutoModelForImageTextToText.from_pretrained(
+                self._config.model_id,
+                **model_kwargs,
+            )
+        else:
+            self._tokenizer = AutoTokenizer.from_pretrained(
+                self._config.model_id,
+                **common_kwargs,
+            )
+            self._model = AutoModelForCausalLM.from_pretrained(
+                self._config.model_id,
+                **model_kwargs,
+            )
+
         if device == "cpu":
             self._model.to(device)
 
-    def generate(
+    @property
+    def device_label(self) -> str:
+        self.load()
+        return self._device_label or "unknown"
+
+    def _normalize_messages(
+        self, messages: list[dict[str, str]]
+    ) -> list[dict[str, object]]:
+        if not self._config.multimodal:
+            return messages
+
+        normalized: list[dict[str, object]] = []
+        for message in messages:
+            content = message["content"]
+            if isinstance(content, str):
+                content = [{"type": "text", "text": content}]
+            normalized.append({"role": message["role"], "content": content})
+        return normalized
+
+    def _generate_from_messages(
         self,
-        prompt: str,
+        messages: list[dict[str, object]],
         *,
         max_tokens: int | None = None,
         temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
-        assert self._tokenizer is not None
-
-        import torch
 
         max_new_tokens = max_tokens or self._config.max_tokens
         temp = self._config.temperature if temperature is None else temperature
 
+        if self._config.multimodal:
+            assert self._processor is not None
+            inputs = self._processor.apply_chat_template(
+                messages,
+                tokenize=True,
+                add_generation_prompt=True,
+                return_dict=True,
+                return_tensors="pt",
+            )
+            inputs = inputs.to(self._model.device)
+            output = self._model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                temperature=temp,
+                do_sample=temp > 0,
+            )
+            generated = output[0][inputs["input_ids"].shape[-1] :]
+            return self._processor.decode(generated, skip_special_tokens=True).strip()
+
+        assert self._tokenizer is not None
+        if hasattr(self._tokenizer, "apply_chat_template"):
+            prompt = self._tokenizer.apply_chat_template(
+                messages,
+                tokenize=False,
+                add_generation_prompt=True,
+            )
+        else:
+            parts = []
+            for message in messages:
+                role = str(message["role"])
+                content = str(message["content"])
+                parts.append(f"{role}: {content}")
+            parts.append("assistant:")
+            prompt = "\n".join(parts)
+
         inputs = self._tokenizer(prompt, return_tensors="pt").to(self._model.device)
         output = self._model.generate(
             **inputs,
@@ -66,6 +156,19 @@ class TransformersBackend:
         generated = output[0][inputs["input_ids"].shape[-1] :]
         return self._tokenizer.decode(generated, skip_special_tokens=True).strip()
 
+    def generate(
+        self,
+        prompt: str,
+        *,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> str:
+        return self.chat(
+            [{"role": "user", "content": prompt}],
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
+
     def chat(
         self,
         messages: list[dict[str, str]],
@@ -73,23 +176,9 @@ class TransformersBackend:
         max_tokens: int | None = None,
         temperature: float | None = None,
     ) -> str:
-        self.load()
-        assert self._model is not None
-        assert self._tokenizer is not None
-
-        if hasattr(self._tokenizer, "apply_chat_template"):
-            prompt = self._tokenizer.apply_chat_template(
-                messages,
-                tokenize=False,
-                add_generation_prompt=True,
-            )
-        else:
-            parts = []
-            for message in messages:
-                role = message["role"]
-                content = message["content"]
-                parts.append(f"{role}: {content}")
-            parts.append("assistant:")
-            prompt = "\n".join(parts)
-
-        return self.generate(prompt, max_tokens=max_tokens, temperature=temperature)
+        normalized = self._normalize_messages(messages)
+        return self._generate_from_messages(
+            normalized,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )

models.yaml

@@ -13,6 +13,7 @@ models:
     backend: transformers
     model_id: openbmb/MiniCPM-V-4.6
     trust_remote_code: true
+    multimodal: true
 
   qwen3b-gguf:
     label: Qwen 2.5 3B Instruct (GGUF)