Update handler.py

handler.py

@@ -1,5 +1,12 @@
-
 # -*- coding: utf-8 -*-
+# handler.py — PULSE-7B / LLaVA robust endpoint
+# - LLaVA kaynak kodunu runtime'da git clone ile getirir
+# - image_processor fallback (AutoProcessor / vision_tower)
+# - anyres -> pad güvenli düşüş
+# - gelişmiş _load_image: UA, redirect, Content-Type kontrolü, base64 padding, boyut sınırı
+# - forward patch (cache_position/input_positions sessizce düşür)
+# - pad_token garanti + conditional attention_mask (+ retry) — HF generate hatalarını önler
+
 import os, io, sys, subprocess, base64
 from typing import Any, Dict, List, Optional
 
@@ -8,9 +15,8 @@ from PIL import Image
 import requests
 import math
 import ast
-from io import BytesIO
 from urllib.parse import urlparse
-
+import inspect
 
 # ===== Kullanılacak HF model id =====
 MODEL_ID = os.getenv("HF_MODEL_ID", "PULSE-ECG/PULSE-7B")
@@ -42,7 +48,7 @@ try:
 except ImportError:
     # Fallback: kendi implementasyonumuzu kullan
     from llava.constants import IMAGE_TOKEN_INDEX
-    
+
     def expand2square(pil_img, background_color):
         width, height = pil_img.size
         if width == height:
@@ -61,13 +67,11 @@ except ImportError:
         best_fit = None
         max_effective_resolution = 0
         min_wasted_resolution = float('inf')
-
         for width, height in possible_resolutions:
             scale = min(width / original_width, height / original_height)
             downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)
             effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)
             wasted_resolution = (width * height) - effective_resolution
-
             if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution):
                 max_effective_resolution = effective_resolution
                 min_wasted_resolution = wasted_resolution
@@ -77,17 +81,14 @@ except ImportError:
     def resize_and_pad_image(image, target_resolution):
         original_width, original_height = image.size
         target_width, target_height = target_resolution
-
         scale_w = target_width / original_width
         scale_h = target_height / original_height
-
         if scale_w < scale_h:
             new_width = target_width
             new_height = min(math.ceil(original_height * scale_w), target_height)
         else:
             new_height = target_height
             new_width = min(math.ceil(original_width * scale_h), target_width)
-
         resized_image = image.resize((new_width, new_height))
         new_image = Image.new('RGB', (target_width, target_height), (0, 0, 0))
         paste_x = (target_width - new_width) // 2
@@ -181,6 +182,9 @@ from llava.constants import (
 from llava.conversation import conv_templates
 from llava.utils import disable_torch_init
 
+# HF processor fallback'ları
+from transformers import AutoProcessor, AutoImageProcessor, CLIPImageProcessor
+
 # Varsayılanlar
 DEFAULT_CONV_MODE  = os.getenv("LLAVA_CONV_MODE", "llava_v1")
 MAX_NEW_TOKENS_DEF = int(os.getenv("MAX_NEW_TOKENS", "1024"))
@@ -211,11 +215,11 @@ class EndpointHandler:
 
         # Attention implementation otomatik seç
         try:
-            import flash_attn
+            import flash_attn  # noqa: F401
             attn_impl = "flash_attention_2"
         except ImportError:
             attn_impl = "sdpa"
-        
+
         # PULSE, LLaVA tabanlı olduğundan LLaVA loader ile yüklenir
         self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model(
             model_path=model_path,
@@ -227,18 +231,31 @@ class EndpointHandler:
         )
         self.model.eval()
 
+        # --- PAD TOKEN FIX -------------------------------------------------
+        if self.tokenizer.pad_token_id is None:
+            self.tokenizer.add_special_tokens({'pad_token': '<pad>'})
+            try:
+                self.model.resize_token_embeddings(len(self.tokenizer))
+            except Exception as e:
+                print(f"[warn] resize_token_embeddings failed: {e}")
+
+        self.model.config.pad_token_id = self.tokenizer.pad_token_id
+        try:
+            self.model.generation_config.pad_token_id = self.tokenizer.pad_token_id
+        except Exception:
+            pass
+        # -------------------------------------------------------------------
+
+        # ---- forward patch (HF yeni arg uyumu) ----
         def _patch_forward(obj, label="model"):
             try:
                 if not hasattr(obj, "forward"):
                     return False
                 orig_forward = obj.forward
-
                 def patched_forward(*args, **kwargs):
-                    # Sessizce düşürülecek yeni anahtarlar
                     kwargs.pop("cache_position", None)
                     kwargs.pop("input_positions", None)
                     return orig_forward(*args, **kwargs)
-
                 obj.forward = patched_forward
                 print(f"[hotfix] Patched forward on {label}")
                 return True
@@ -246,63 +263,88 @@ class EndpointHandler:
                 print(f"[warn] forward patch failed on {label}: {e}")
                 return False
 
-        # Ana modelde dene
         _patch_forward(self.model, "self.model")
-
-        # Bazı sürümlerde forward zinciri iç modüle de gider
         if hasattr(self.model, "model"):
             _patch_forward(self.model.model, "self.model.model")
         if hasattr(self.model, "base_model"):
             _patch_forward(self.model.base_model, "self.model.base_model")
-        # =======================================================================
+
+        # ---- image_processor fallback ----
+        if self.image_processor is None:
+            print("[hotfix] image_processor None, AutoProcessor/vision_tower fallback deneniyor...")
+            try:
+                proc = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+                self.image_processor = getattr(proc, "image_processor", proc)
+            except Exception as e:
+                print(f"[warn] AutoProcessor başarısız: {e}")
+                vt = getattr(self.model.config, "vision_tower", None)
+                if vt:
+                    try:
+                        self.image_processor = AutoImageProcessor.from_pretrained(vt, trust_remote_code=True)
+                    except Exception as e2:
+                        print(f"[warn] AutoImageProcessor failed: {e2}")
+                        try:
+                            self.image_processor = CLIPImageProcessor.from_pretrained(vt)
+                        except Exception as e3:
+                            print(f"[warn] CLIPImageProcessor failed: {e3}")
+
+        # anyres -> pad fallback (processor/crop_size yoksa)
+        iar = getattr(self.model.config, "mm_image_aspect_ratio", None) or getattr(self.model.config, "image_aspect_ratio", None)
+        needs_crop = (self.image_processor is None) or (getattr(self.image_processor, "crop_size", None) is None)
+        if iar == "anyres" and needs_crop:
+            print("[hotfix] image_aspect_ratio:anyres -> pad (processor/crop_size eksik)")
+            if hasattr(self.model.config, "image_aspect_ratio"):
+                self.model.config.image_aspect_ratio = "pad"
+            if hasattr(self.model.config, "mm_image_aspect_ratio"):
+                self.model.config.mm_image_aspect_ratio = "pad"
 
         # Model worker'dan: multimodal check
         self.is_multimodal = 'llava' in self.model_name.lower() or 'pulse' in self.model_name.lower()
-        
+
         # Görsel token işaretleri (LLaVA config)
         self.use_im_start_end = getattr(self.model.config, "mm_use_im_start_end", False)
 
-    # ---- yardımcılar ----
+        # attention_mask desteğini tespit et
+        try:
+            sig = inspect.signature(self.model.forward)
+            self._supports_attention_mask = ("attention_mask" in sig.parameters)
+        except Exception:
+            self._supports_attention_mask = False
+
+    # ---- gelişmiş image loader ----
     def _load_image(self, image_input: str) -> Optional[Image.Image]:
         """
         URL / base64 / yerel path -> PIL.Image
-        - URL için: içerik tipi/uzantı kontrolü, UA header, redirect, boyut sınırı
-        - base64 için: data URL prefix temizliği + padding düzeltme
-        - path için: doğrudan aç
+        - URL: UA header, redirect, Content-Type kontrolü, boyut sınırı
+        - base64: data URL prefix temizliği + padding düzeltme
+        - path: doğrudan aç
         """
         if not image_input:
             return None
-    
-        # 25MB üstünü reddet (isteğe göre ayarlanabilir)
-        MAX_IMAGE_BYTES = int(os.getenv("MAX_IMAGE_BYTES", "26214400"))
+
+        MAX_IMAGE_BYTES = int(os.getenv("MAX_IMAGE_BYTES", "26214400"))  # 25MB
         ALLOWED_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".gif", ".webp"}
-    
+
         def _is_valid_image_format(url: str) -> bool:
             try:
                 ext = os.path.splitext(urlparse(url).path.lower())[1]
-                # Uzantı yoksa da kabul et; Content-Type ile kontrol ederiz
                 return (ext in ALLOWED_EXTS) or (ext == "")
             except Exception:
                 return True
-    
+
         try:
-            # ---- URL input ----
+            # URL
             if isinstance(image_input, str) and image_input.startswith(("http://", "https://")):
                 if not _is_valid_image_format(image_input):
                     print("[warn] Invalid image extension in URL")
                     return None
-    
                 headers = {"User-Agent": "Mozilla/5.0"}
                 resp = requests.get(image_input, timeout=20, headers=headers, allow_redirects=True, stream=True)
                 resp.raise_for_status()
-    
-                # Content-Type kontrolü (varsa)
                 ctype = resp.headers.get("Content-Type", "").lower()
                 if ctype and not ctype.startswith("image/"):
                     print(f"[warn] Non-image content-type: {ctype}")
                     return None
-    
-                # Boyut kontrolü (Content-Length varsa)
                 clen = resp.headers.get("Content-Length")
                 if clen is not None:
                     try:
@@ -311,68 +353,54 @@ class EndpointHandler:
                             return None
                     except Exception:
                         pass
-    
-                # Stream’den kontrollü oku
                 data = resp.content
                 if len(data) > MAX_IMAGE_BYTES:
                     print(f"[warn] Image too large (actual): {len(data)} bytes")
                     return None
-    
-                img = Image.open(BytesIO(data)).convert("RGB")
+                img = Image.open(io.BytesIO(data)).convert("RGB")
                 print(f"[info] URL image loaded: size={img.size}")
                 return img
-    
-            # ---- Base64 input (data URL dahil) ----
+
+            # Base64 (data URL dahil)
             if isinstance(image_input, str):
                 b64 = image_input.strip()
-    
-                # data URL prefix varsa ayıkla
                 if b64.startswith("data:image"):
-                    # ör: data:image/png;base64,AAAA...
                     if "base64," in b64:
                         b64 = b64.split("base64,", 1)[1]
                     else:
-                        # bazen ;base64 sonrası newline vb olabilir
                         b64 = b64.split(",", 1)[-1]
-    
-                # boşluk/newline temizliği + padding düzeltme
                 b64 = b64.replace("\n", "").replace("\r", "").replace(" ", "")
                 missing = (4 - len(b64) % 4) % 4
                 b64 += "=" * missing
-    
                 try:
                     data = base64.b64decode(b64, validate=False)
                     if len(data) > MAX_IMAGE_BYTES:
                         print(f"[warn] Base64 image too large: {len(data)} bytes")
                         return None
-                    img = Image.open(BytesIO(data)).convert("RGB")
+                    img = Image.open(io.BytesIO(data)).convert("RGB")
                     print(f"[info] Base64 image loaded: size={img.size}")
                     return img
                 except Exception as e:
                     print(f"[warn] Base64 decode/open failed: {e}")
-                    # Devam edip path olarak deneyeceğiz
-    
-            # ---- Yerel path ----
+                    # path olarak denemeye devam
+
+            # Yerel path
             if isinstance(image_input, str) and os.path.exists(image_input):
                 img = Image.open(image_input).convert("RGB")
                 print(f"[info] Local image loaded: size={img.size}")
                 return img
-    
+
         except Exception as e:
             print(f"[warn] image load failed: {e}")
             return None
-    
-        return None
 
+        return None
 
     def _build_prompt(self, user_text: str, conv_mode: str) -> str:
         """Model worker tarzında prompt oluştur"""
         if conv_mode not in conv_templates:
             conv_mode = DEFAULT_CONV_MODE
         conv = conv_templates[conv_mode].copy()
-        
-        # Model worker'da görüntüler sonradan replace edilir
-        # Şimdilik sadece text ile başlayalım
         conv.append_message(conv.roles[0], user_text)
         conv.append_message(conv.roles[1], None)
         return conv.get_prompt()
@@ -387,95 +415,81 @@ class EndpointHandler:
         query_text = inputs.get("query", "") or inputs.get("text", "") or inputs.get("prompt", "")
         image_f = inputs.get("image") or inputs.get("image_url") or inputs.get("image_base64")
 
-        # 1) İlk prompt oluştur (görüntü olmadan)
+        # 1) İlk prompt
         prompt = self._build_prompt(query_text, conv_mode)
-        
-        # 2) Görüntü işleme (model worker tarzında)
+
+        # 2) Görüntü işleme
         images = None
         image_sizes = None
-        
         if image_f and self.is_multimodal:
             try:
                 pil_image = self._load_image(image_f)
-                if pil_image is not None:
+                if pil_image is not None and self.image_processor is not None:
                     images_list = [pil_image]
                     image_sizes = [pil_image.size]
-                    
-                    # Model worker'daki gibi process et
                     processed_images = process_images(images_list, self.image_processor, self.model.config)
-                    
                     if isinstance(processed_images, list):
                         images = [img.to(self.model.device, dtype=torch.float16) for img in processed_images]
                     else:
                         images = processed_images.to(self.model.device, dtype=torch.float16)
-                    
-                    # Model worker'daki gibi prompt'u düzenle
-                    # DEFAULT_IMAGE_TOKEN'ı prompt'a ekle
+                    # Görsel token ekle + im_start/end sarma
                     prompt = DEFAULT_IMAGE_TOKEN + '\n' + prompt
-                    
-                    # Replace token hesapla (model worker'dan)
                     replace_token = DEFAULT_IMAGE_TOKEN
                     if self.use_im_start_end:
                         replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN
-                    
-                    # Prompt'taki image token'ları replace et
                     prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)
-                    
                     print(f"[info] Image processed successfully")
-                    print(f"[debug] Final prompt: {repr(prompt[:200])}")
                 else:
-                    print("[warn] Could not load image")
+                    print("[warn] Could not load image or image_processor is None.")
             except Exception as e:
                 print(f"[warn] Image processing failed: {e}")
-                import traceback
-                traceback.print_exc()
-                images = None
-                image_sizes = None
+                import traceback; traceback.print_exc()
+                images = None; image_sizes = None
 
-        # 3) Tokenize (model worker tarzında)
+        # 3) Tokenize
         try:
             input_ids = tokenizer_image_token(
                 prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt'
             ).unsqueeze(0).to(self.model.device)
-            
-            print(f"[debug] input_ids shape: {input_ids.shape}")
-            print(f"[debug] Has images: {images is not None}")
-            
+            print(f"[debug] input_ids shape: {input_ids.shape} | has images: {images is not None}")
         except Exception as e:
             print(f"[error] Tokenization failed: {e}")
             # Fallback to text-only
-            input_ids = self.tokenizer(query_text, return_tensors="pt").input_ids
-            input_ids = input_ids.to(self.model.device)
+            input_ids = self.tokenizer(query_text, return_tensors="pt").input_ids.to(self.model.device)
             images = None
             image_sizes = None
 
-        # 4) Generation parameters (model worker tarzında)
+        # 4) Generation parameters
         temperature = float(params.get("temperature", 0.0))
         top_p = float(params.get("top_p", 1.0))
         repetition_penalty = float(params.get("repetition_penalty", 1.0))
         max_new_tokens = min(int(params.get("max_new_tokens", MAX_NEW_TOKENS_DEF)), 1024)
         do_sample = bool(params.get("do_sample", temperature > 0.001))
-        
-        # Context length check
-        max_context_length = getattr(self.model.config, 'max_position_embeddings', 2048)
-        max_new_tokens = min(max_new_tokens, max_context_length - input_ids.shape[-1] - 50)
-        
+
+        # Context length check (güvenli boşluk)
+        max_context_length = getattr(self.model.config, 'max_position_embeddings', 4096)
+        max_new_tokens = min(max_new_tokens, max(1, max_context_length - input_ids.shape[-1] - 50))
         if max_new_tokens < 1:
             return [{"generated_text": "Error: Input too long, exceeds max token length."}]
 
-        # 5) Generation kwargs (model worker tarzında)
+        # 5) Generation kwargs
         gen_kwargs = {
-            "inputs": input_ids,  # model worker 'inputs' kullanır
+            "inputs": input_ids,
             "max_new_tokens": max_new_tokens,
             "temperature": temperature,
             "top_p": top_p,
             "repetition_penalty": repetition_penalty,
             "do_sample": do_sample,
             "use_cache": bool(params.get("use_cache", True)),
-            "pad_token_id": self.tokenizer.eos_token_id,
+            "pad_token_id": self.tokenizer.pad_token_id,  # pad garanti
         }
 
-        # Image args (model worker tarzında)
+        # attention_mask'i model destekliyorsa ekle
+        if getattr(self, "_supports_attention_mask", False):
+            attention_mask = torch.ones_like(input_ids, dtype=torch.long, device=input_ids.device)
+            gen_kwargs["attention_mask"] = attention_mask
+
+        # Image args
         if images is not None and image_sizes is not None:
             gen_kwargs["images"] = images
             gen_kwargs["image_sizes"] = image_sizes
@@ -483,21 +497,26 @@ class EndpointHandler:
         else:
             print("[info] Text-only generation")
 
+        # 6) Generate (+ unused kwargs için retry)
         try:
             with torch.inference_mode():
                 output_ids = self.model.generate(**gen_kwargs)
-            
-            # Output'u input'tan ayır
-            if output_ids.shape[-1] > input_ids.shape[-1]:
-                response_ids = output_ids[:, input_ids.shape[-1]:]
-                text = self.tokenizer.batch_decode(response_ids, skip_special_tokens=True)[0].strip()
+        except ValueError as e:
+            if "model_kwargs" in str(e) and "attention_mask" in str(e):
+                print("[hotfix] model doesn't accept attention_mask; retrying without it")
+                gen_kwargs.pop("attention_mask", None)
+                with torch.inference_mode():
+                    output_ids = self.model.generate(**gen_kwargs)
             else:
-                text = "Error: No response generated"
-                
-        except Exception as e:
-            print(f"Generation error: {e}")
-            import traceback
-            traceback.print_exc()
-            text = f"Error during generation: {str(e)}"
-            
-        return [{"generated_text": text}]
+                print(f"Generation error: {e}")
+                import traceback; traceback.print_exc()
+                return [{"generated_text": f"Error during generation: {str(e)}"}]
+
+        # 7) Output'u input'tan ayır
+        if output_ids.shape[-1] > input_ids.shape[-1]:
+            response_ids = output_ids[:, input_ids.shape[-1]:]
+            text = self.tokenizer.batch_decode(response_ids, skip_special_tokens=True)[0].strip()
+        else:
+            text = "Error: No response generated"
+
+        return [{"generated_text": text}]