Update handler.py

handler.py

@@ -1,11 +1,32 @@
-
-import os, io, base64
+# -*- coding: utf-8 -*-
+import os, io, sys, subprocess, base64
 from typing import Any, Dict, List, Optional
 
 import torch
 from PIL import Image
+import requests
+
+# ===== Kullanılacak HF model id =====
+MODEL_ID = os.getenv("HF_MODEL_ID", "PULSE-ECG/PULSE-7B")
+
+# ===== LLaVA kaynak kodunu runtime'da getir (pip yok) =====
+LLAVA_GIT_URL = os.getenv("LLAVA_GIT_URL", "https://github.com/haotian-liu/LLaVA.git")
+LLAVA_GIT_REF = os.getenv("LLAVA_GIT_REF", "v1.2.2.post1")  # kanıtlı, stabil
+LLAVA_SRC_DIR = os.getenv("LLAVA_SRC_DIR", "/tmp/llava_src/LLaVA")
+
+def _ensure_llava():
+    if not os.path.isdir(LLAVA_SRC_DIR):
+        os.makedirs(os.path.dirname(LLAVA_SRC_DIR), exist_ok=True)
+        subprocess.run(
+            ["git", "clone", "--depth", "1", "--branch", LLAVA_GIT_REF, LLAVA_GIT_URL, LLAVA_SRC_DIR],
+            check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+    if LLAVA_SRC_DIR not in sys.path:
+        sys.path.insert(0, LLAVA_SRC_DIR)
 
-# --- LLaVA (demo) parçaları ---
+_ensure_llava()
+
+# ---- LLaVA parçaları (demo akışı) ----
 from llava.model.builder import load_pretrained_model, get_model_name_from_path
 from llava.mm_utils import tokenizer_image_token, process_images
 from llava.constants import (
@@ -17,60 +38,39 @@ from llava.constants import (
 from llava.conversation import conv_templates
 from llava.utils import disable_torch_init
 
-
-# =========================
-# Ortam / Varsayılanlar
-# =========================
-# 1) Yerelden yüklemek için (bu repository içi): boş bırakın veya HF_MODEL_LOCAL_DIR=/repository
-HF_MODEL_LOCAL_DIR = os.getenv("HF_MODEL_LOCAL_DIR", "").strip()
-# 2) Hub'dan yüklemek isterseniz: HF_MODEL_ID=org/name
-HF_MODEL_ID = os.getenv("HF_MODEL_ID", "").strip()
-
-# Demo ile aynı conv_mode
-DEFAULT_CONV_MODE = os.getenv("LLAVA_CONV_MODE", "llava_v2")
-# Güvenli varsayılan (çok büyük tutmayalım)
+# Varsayılanlar
+DEFAULT_CONV_MODE  = os.getenv("LLAVA_CONV_MODE", "llava_v2")
 MAX_NEW_TOKENS_DEF = int(os.getenv("MAX_NEW_TOKENS", "256"))
-
-# Flash-Attention zorunluluğunu kaldır, SDPA kullan
-os.environ.setdefault("ATTN_IMPLEMENTATION", "sdpa")
-
+os.environ.setdefault("ATTN_IMPLEMENTATION", os.getenv("ATTN_IMPLEMENTATION", "sdpa"))
 
 class EndpointHandler:
     """
-    Hugging Face Inference Toolkit tarafından çağrılan handler.
-    Girdi şeması (demo ile uyumlu):
+    Girdi:
     {
       "inputs": { "query": "...", "image": "<url|dataurl|path>" },
-      "parameters": {
-        "max_new_tokens": 256, "temperature": 0.0, "top_p": 1.0,
-        "repetition_penalty": 1.0, "do_sample": false, "use_cache": true
-      },
+      "parameters": { "max_new_tokens": 256, "temperature": 0.0, "top_p": 1.0,
+                      "repetition_penalty": 1.0, "do_sample": false, "use_cache": true },
       "conv_mode": "llava_v2"  # opsiyonel
     }
-    Dönüş:
-    [ { "generated_text": "..." } ]
+    Çıktı: [ { "generated_text": "..." } ]
     """
-
     def __init__(self, path: str = "") -> None:
-        # path -> /repository
         disable_torch_init()
 
-        # Modelin yüklenme yolu seçimi
-        if HF_MODEL_LOCAL_DIR:
-            model_path = HF_MODEL_LOCAL_DIR
-        elif HF_MODEL_ID:
-            model_path = HF_MODEL_ID
+        # PULSE-7B HF’den/yerelden nereden yükleniyorsa yolu belirle
+        if os.getenv("HF_MODEL_LOCAL_DIR", "").strip():
+            model_path = os.getenv("HF_MODEL_LOCAL_DIR").strip()
+        elif os.getenv("HF_MODEL_ID", "").strip():
+            model_path = os.getenv("HF_MODEL_ID").strip()
         else:
-            # Ağırlıklar bu repoda ise
-            model_path = path
+            model_path = MODEL_ID  # default: HF Hub PULSE-7B
 
-        # Model adı (LLaVA yardımcı)
         self.model_name = get_model_name_from_path(model_path)
 
-        # LLaVA yüklemesi (demo ile aynı giriş noktası)
+        # PULSE, LLaVA tabanlı olduğundan LLaVA loader ile yüklenir
         self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model(
             model_path=model_path,
-            model_base=None,  # LoRA yoksa None
+            model_base=None,
             model_name=self.model_name,
             torch_dtype="auto",
             attn_implementation=os.getenv("ATTN_IMPLEMENTATION", "sdpa"),
@@ -78,89 +78,73 @@ class EndpointHandler:
         )
         self.model.eval()
 
-        # Görsel token işaretleri (model config'ine bağlı)
+        # Görsel token işaretleri (LLaVA config)
         self.use_im_start_end = getattr(self.model.config, "mm_use_im_start_end", False)
         self.image_token = DEFAULT_IMAGE_TOKEN
         self.im_start = DEFAULT_IM_START_TOKEN
         self.im_end = DEFAULT_IM_END_TOKEN
 
-    # ---------------------------
-    # Yardımcılar
-    # ---------------------------
+    # ---- yardımcılar ----
     def _load_image(self, img_field: str) -> Optional[Image.Image]:
-        """URL / data URL / yerel path -> PIL.Image"""
+        """URL / base64 / path -> PIL.Image"""
         if not img_field:
             return None
         try:
             if img_field.startswith("data:image"):
-                head, b64 = img_field.split(",", 1)
+                _, b64 = img_field.split(",", 1)
                 return Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
-            elif img_field.startswith("http://") or img_field.startswith("https://"):
-                import requests
+            if img_field.startswith(("http://", "https://")):
                 r = requests.get(img_field, timeout=20)
                 r.raise_for_status()
                 return Image.open(io.BytesIO(r.content)).convert("RGB")
-            else:
-                return Image.open(img_field).convert("RGB")
+            return Image.open(img_field).convert("RGB")
         except Exception as e:
-            # Görsel okunamadıysa açıklayıcı hata bırak
-            raise RuntimeError(f"Image load failed: {e}") from e
+            # Görsel opsiyoneldir; okunamazsa kullanıcıya hata dönmek yerine None bırakabiliriz.
+            print(f"[warn] image load failed: {e}")
+            return None
 
     def _build_prompt(self, user_text: str, conv_mode: str) -> str:
-        """Demodaki gibi conv_templates ile diyalog şablonu kur."""
-        # Yanlış conv_mode gelirse default'a düş
         if conv_mode not in conv_templates:
             conv_mode = DEFAULT_CONV_MODE
-
         conv = conv_templates[conv_mode].copy()
         if self.use_im_start_end:
             content = f"{self.im_start}{self.image_token}{self.im_end}\n{user_text}"
         else:
             content = f"{self.image_token}\n{user_text}"
-
-        conv.append_message(conv.roles[0], content)  # user
-        conv.append_message(conv.roles[1], None)     # assistant (boş)
+        conv.append_message(conv.roles[0], content)
+        conv.append_message(conv.roles[1], None)
         return conv.get_prompt()
 
-    # ---------------------------
-    # Inference giriş noktası
-    # ---------------------------
+    # ---- inference ----
     def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
         inputs = data.get("inputs") or {}
         params = data.get("parameters") or {}
         conv_mode_req = data.get("conv_mode")
-        conv_mode = conv_mode_req if conv_mode_req in conv_templates else DEFAULT_CONV_MODE
 
-        query_text = inputs.get("query", "")
-        image_f = inputs.get("image", "")
-        pil_img = self._load_image(image_f) if image_f else None
+        conv_mode = conv_mode_req if conv_mode_req in conv_templates else DEFAULT_CONV_MODE
+        query_text = inputs.get("query", "") or inputs.get("text", "") or inputs.get("prompt", "")
+        image_f = inputs.get("image") or inputs.get("image_url") or inputs.get("image_base64")
 
-        # 1) Prompt hazırla
+        # 1) prompt
         prompt = self._build_prompt(query_text, conv_mode)
 
-        # 2) Görsel tensörü
+        # 2) image -> tensor (opsiyonel)
         image_tensors = None
-        if pil_img is not None:
-            image_tensors = process_images([pil_img], self.image_processor, self.model.config)
+        if image_f:
+            pil = self._load_image(image_f)
+            if pil is not None:
+                image_tensors = process_images([pil], self.image_processor, self.model.config)
+                image_tensors = image_tensors.to(self.model.device, dtype=self.model.dtype, non_blocking=True)
 
-        # 3) Tokenize (görüntü tokenını göm)
+        # 3) tokenize (image token’ı gömülü)
         input_ids = tokenizer_image_token(
-            prompt,
-            self.tokenizer,
-            IMAGE_TOKEN_INDEX,
-            return_tensors="pt",
-        )
-        input_ids = input_ids.to(self.model.device, non_blocking=True)
+            prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt"
+        ).to(self.model.device, non_blocking=True)
 
-        # 4) context_len'e göre güvenli max_new_tokens
-        requested_max_new = int(params.get("max_new_tokens", MAX_NEW_TOKENS_DEF))
-        # ufak tampon ile aşımı engelle
+        # 4) güvenli max_new_tokens
+        requested = int(params.get("max_new_tokens", MAX_NEW_TOKENS_DEF))
         avail = max(16, int(self.context_len) - int(input_ids.shape[-1]) - 8)
-        max_new_tokens = max(1, min(requested_max_new, avail))
-
-        # Görseli cihaza taşı
-        if image_tensors is not None:
-            image_tensors = image_tensors.to(self.model.device, dtype=self.model.dtype, non_blocking=True)
+        max_new_tokens = max(1, min(requested, avail))
 
         gen_kwargs = {
             "max_new_tokens": max_new_tokens,
@@ -172,11 +156,7 @@ class EndpointHandler:
         }
 
         with torch.inference_mode():
-            output_ids = self.model.generate(
-                input_ids,
-                images=image_tensors,
-                **gen_kwargs,
-            )
-
-        outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
-        return [{"generated_text": outputs}]
+            output_ids = self.model.generate(input_ids, images=image_tensors, **gen_kwargs)
+
+        text = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+        return [{"generated_text": text}]