Update handler.py

handler.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 """
-PULSE ECG Handler (demo-like streaming, stable & clean)
-- TextIteratorStreamer + skip_prompt=True  → baş kesilmesi yok (Step 1 korunur)
-- do_sample=True (demo davranışı), temperature/top_p payload’dan
-- Anti-tekrar: no_repeat_ngram_size + repetition_penalty
-- Opsiyonel: custom_stop (örn. "END OF REPORT") → çıktı sonunda trim
-- Deterministik mod: aynı görüntü+mesaj için aynı seed (deterministic=True)
-- Görsel tensörü 3D/4D/5D uyumlu; device/dtype eşleştirme
+PULSE ECG Handler — Demo Parity Mode
+- Demo app.py ile aynı üretim ayarları:
+  do_sample=True, temperature=0.05, top_p=1.0, max_new_tokens=4096
+- Stopping: konuşma ayırıcıda (conv.sep/sep2) güvenli token-eşleşmeli kriter
+- Görsel tensörü: .half() ve model cihazında
+- Streamer: TextIteratorStreamer (demo gibi), thread ile generate
+- Seed/deterministic KAPALI (göndermezseniz); demo gibi stokastik
 """
 
 import os
@@ -16,13 +16,13 @@ import hashlib
 import datetime
 from io import BytesIO
 from threading import Thread
-from typing import Optional, List, Union
+from typing import Optional, Union
 
 import torch
 from PIL import Image
 import requests
 
-# ---------- LLaVA & Transformers ----------
+# ====== LLaVA & Transformers ======
 try:
     from llava.constants import (
         IMAGE_TOKEN_INDEX,
@@ -41,16 +41,16 @@ try:
     LLAVA_AVAILABLE = True
 except Exception as e:
     LLAVA_AVAILABLE = False
-    print(f"[WARN] LLaVA modules not available: {e}")
+    print(f"[WARN] LLaVA not available: {e}")
 
 try:
-    from transformers import TextIteratorStreamer
+    from transformers import TextIteratorStreamer, StoppingCriteria
     TRANSFORMERS_AVAILABLE = True
 except Exception as e:
     TRANSFORMERS_AVAILABLE = False
     print(f"[WARN] transformers not available: {e}")
 
-# ---------- HF Hub (opsiyonel logging) ----------
+# ====== HF Hub logging (opsiyonel) ======
 try:
     from huggingface_hub import HfApi, login
     HF_HUB_AVAILABLE = True
@@ -72,7 +72,7 @@ if HF_HUB_AVAILABLE and "HF_TOKEN" in os.environ:
 LOGDIR = "./logs"
 os.makedirs(LOGDIR, exist_ok=True)
 
-# ---------- Global Model State ----------
+# ====== Global State ======
 tokenizer = None
 model = None
 image_processor = None
@@ -81,7 +81,7 @@ args = None
 model_initialized = False
 
 
-# ======================== Utilities ========================
+# ===================== Utilities =====================
 
 def _safe_upload(path: str):
     if api and repo_name and path and os.path.isfile(path):
@@ -97,22 +97,20 @@ def _safe_upload(path: str):
 
 def _conv_log_path() -> str:
     t = datetime.datetime.now()
-    p = os.path.join(LOGDIR, f"{t.year:04d}-{t.month:02d}-{t.day:02d}-user_conv.json")
-    os.makedirs(os.path.dirname(p), exist_ok=True)
-    return p
+    return os.path.join(LOGDIR, f"{t.year:04d}-{t.month:02d}-{t.day:02d}-user_conv.json")
 
 def load_image_any(image_input: Union[str, dict]) -> Image.Image:
     """
     Desteklenen:
       - URL (http/https)
-      - Yerel dosya yolu
+      - yerel dosya yolu
       - base64 (opsiyonel data URL prefix ile)
       - {"image": <base64|dataurl>}
     """
     if isinstance(image_input, str):
         s = image_input.strip()
         if s.startswith(("http://", "https://")):
-            r = requests.get(s, timeout=(5, 15))
+            r = requests.get(s, timeout=(5, 20))
             r.raise_for_status()
             return Image.open(BytesIO(r.content)).convert("RGB")
         if os.path.exists(s):
@@ -142,8 +140,36 @@ def _wrap_image_token_if_needed(model_cfg) -> bool:
     except Exception:
         return False
 
+
+# ====== Güvenli Stop Kriteri (demo eşleniği) ======
+class SafeKeywordsStoppingCriteria(StoppingCriteria):
+    """
+    LLaVA'nın KeywordsStoppingCriteria'sına karşılık, token bazlı
+    anahtar dizi (separator) eşleşmesi; tensör → bool hatası yok.
+    """
+    def __init__(self, keyword: str, tokenizer):
+        self.tokenizer = tokenizer
+        tok = tokenizer(keyword, add_special_tokens=False, return_tensors="pt").input_ids[0]
+        self.kw_ids = tok  # shape: (n,)
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        # input_ids: (bsz, seq_len)
+        if input_ids is None or input_ids.shape[0] == 0:
+            return False
+        out = input_ids[0]  # assume bsz=1
+        n = self.kw_ids.shape[0]
+        if out.shape[0] < n:
+            return False
+        tail = out[-n:]
+        # cihaz hizası
+        kw = self.kw_ids.to(tail.device)
+        return torch.equal(tail, kw)
+
+
+# ===================== Core Generation =====================
+
 def _build_prompt_and_ids(chatbot, user_text: str, device: torch.device):
-    # Demo gibi: <image> token + text (IM_START/END gerekiyorsa sar)
+    # demo gibi: <image> + text (IM_START/END gerekiyorsa sar)
     use_wrap = _wrap_image_token_if_needed(chatbot.model.config)
     if use_wrap:
         # <im_start><image><im_end>\n + user text
@@ -160,50 +186,43 @@ def _build_prompt_and_ids(chatbot, user_text: str, device: torch.device):
     ).unsqueeze(0).to(device)
     return prompt, input_ids
 
-def _stable_seed_from(image_hash: str, message_text: str) -> int:
-    """Aynı resim+mesaj için aynı seed (deterministik örnekleme)"""
-    h = hashlib.md5((image_hash + "||" + message_text).encode("utf-8")).digest()
-    # 32-bit pozitif int
-    return int.from_bytes(h[:4], "big", signed=False)
-
-
-# ======================== Core Generation ========================
-
 def generate_response(
     message_text: str,
     image_input,
     *,
-    max_new_tokens: int = 1800,
-    min_new_tokens: Optional[int] = 700,
-    temperature: float = 0.20,
-    top_p: float = 0.95,
-    repetition_penalty: float = 1.20,
-    no_repeat_ngram_size: Optional[int] = 6,
+    temperature: Optional[float] = None,
+    top_p: Optional[float] = None,
+    max_new_tokens: Optional[int] = None,
     conv_mode_override: Optional[str] = None,
-    deterministic: bool = False,  # True → do_sample=False (tam deterministik)
-    det_seed: Optional[int] = None,  # verilirse sabit seed
-    custom_stop: Optional[List[str]] = None,  # ["END OF REPORT"] gibi
-    no_stop: bool = False,  # True → eos/stop yok (önerilmez)
+    repetition_penalty: Optional[float] = None,  # demo'da yok; verilirse 1.0 yaparız
+    # NOT: no_repeat_ngram_size / min_new_tokens / custom_stop KULLANMIYORUZ → demo-parite
+    det_seed: Optional[int] = None,              # seed gönderilmezse stokastik (demo gibi)
 ):
     if not (LLAVA_AVAILABLE and TRANSFORMERS_AVAILABLE):
         return {"error": "Required libraries not available (llava/transformers)"}
     if not message_text or image_input is None:
         return {"error": "Both 'message' and 'image' are required"}
 
-    # Chat oturumu (her çağrıda taze template; demo benzeri)
+    # Varsayılanlar → demo
+    if temperature is None: temperature = 0.05
+    if top_p is None: top_p = 1.0
+    if max_new_tokens is None: max_new_tokens = 4096
+    if repetition_penalty is None: repetition_penalty = 1.0  # etkisiz
+
+    # Chat session: her çağrıda taze template
     chatbot = chat_manager.get_chatbot(args, args.model_path, tokenizer, model, image_processor, context_len)
     if conv_mode_override and conv_mode_override in conv_templates:
         chatbot.conversation = conv_templates[conv_mode_override].copy()
     else:
         chatbot.conversation = conv_templates[chatbot.conv_mode].copy()
 
-    # Görseli yükle
+    # Görüntü yükle
     try:
         pil_img = load_image_any(image_input)
     except Exception as e:
         return {"error": f"Failed to load image: {e}"}
 
-    # Log için kaydet (hash + path)
+    # Log için hash+path
     img_hash, img_path = "NA", None
     try:
         buf = BytesIO(); pil_img.save(buf, format="JPEG"); raw = buf.getvalue()
@@ -214,117 +233,75 @@ def generate_response(
         if not os.path.isfile(img_path):
             pil_img.save(img_path)
     except Exception as e:
-        print(f"[log] saving image failed: {e}")
+        print(f"[log] save image failed: {e}")
 
-    # Cihaza/dtype’a taşı
+    # Cihaz/dtype
     device = next(chatbot.model.parameters()).device
-    dtype  = next(chatbot.model.parameters()).dtype
+    # demo half: .half() kullanacağız
+    dtype = torch.float16
 
-    # Görüntü ön-işleme → tensör (3D/4D/5D destek)
+    # Görüntü ön-işleme → tensör
     try:
         processed = process_images([pil_img], chatbot.image_processor, chatbot.model.config)
-        if isinstance(processed, torch.Tensor):
-            if processed.ndim == 3:   image_tensor = processed.unsqueeze(0)     # (1,C,H,W)
-            elif processed.ndim == 4: image_tensor = processed                  # (B,C,H,W)
-            elif processed.ndim == 5:                                          # (B,T,C,H,W) → (B*T,C,H,W)
-                b,t,c,h,w = processed.shape
-                image_tensor = processed.reshape(b*t, c, h, w)
-            else:
-                return {"error": f"Unexpected image tensor shape: {tuple(processed.shape)}"}
-        elif isinstance(processed, (list, tuple)) and len(processed) > 0:
-            first = processed[0]
-            image_tensor = first.unsqueeze(0) if isinstance(first, torch.Tensor) and first.ndim == 3 else first
+        # LLaVA genelde list döndürür
+        if isinstance(processed, (list, tuple)) and len(processed) > 0:
+            image_tensor = processed[0]
+        elif isinstance(processed, torch.Tensor):
+            image_tensor = processed[0] if processed.ndim == 4 else processed  # güvenlik
         else:
             return {"error": "Image processing returned empty"}
+        if image_tensor.ndim == 3:
+            image_tensor = image_tensor.unsqueeze(0)  # (1,C,H,W)
+        # demo: half + device
         image_tensor = image_tensor.to(device=device, dtype=dtype)
     except Exception as e:
         return {"error": f"Image processing failed: {e}"}
 
-    # Prompt & ids
+    # Prompt & input ids
     _, input_ids = _build_prompt_and_ids(chatbot, message_text, device)
 
-    # Seed ayarı
+    # Stop string from conv
+    stop_str = chatbot.conversation.sep if chatbot.conversation.sep_style != SeparatorStyle.TWO else chatbot.conversation.sep2
+    stopping = SafeKeywordsStoppingCriteria(stop_str, chatbot.tokenizer)
+
+    # Seed (gönderilmediyse stokastik → demo gibi)
     if det_seed is not None:
         try:
             s = int(det_seed)
+            torch.manual_seed(s)
+            if torch.cuda.is_available():
+                torch.cuda.manual_seed(s)
+                torch.cuda.manual_seed_all(s)
         except Exception:
-            s = None
-    elif deterministic:
-        s = _stable_seed_from(img_hash, message_text)
-    else:
-        # Deterministik örnekleme istiyorsan; aynı girdide aynı sonuç için stabil seed de kullanabiliriz
-        s = _stable_seed_from(img_hash, message_text)
-
-    if s is not None:
-        torch.manual_seed(s)
-        if torch.cuda.is_available():
-            torch.cuda.manual_seed(s)
-            torch.cuda.manual_seed_all(s)
-
-    # Stopping / EOS
-    eos_id = chatbot.tokenizer.eos_token_id
-    pad_id = chatbot.tokenizer.pad_token_id if chatbot.tokenizer.pad_token_id is not None else (eos_id if eos_id is not None else 0)
-    eos_for_gen = None if no_stop else eos_id
+            pass
 
-    # Streamer (demo gibi; manuel dilimleme yok → Step 1 korunur)
+    # Streamer (demo gibi)
     streamer = TextIteratorStreamer(
         chatbot.tokenizer, skip_prompt=True, skip_special_tokens=True
     )
 
-    # do_sample: demo gibi (True). deterministic=True ise greedy’ye geç
-    do_sample = not deterministic
-
+    # Generate kwargs — demo ayarları
     gen_kwargs = dict(
         inputs=input_ids,
         images=image_tensor,
         streamer=streamer,
-        do_sample=do_sample,
-        temperature=float(temperature),
-        top_p=float(top_p),
-        repetition_penalty=float(repetition_penalty),
-        max_new_tokens=int(max_new_tokens),
+        do_sample=True,                     # DEMO
+        temperature=float(temperature),     # DEMO default 0.05
+        top_p=float(top_p),                 # DEMO default 1.0
+        max_new_tokens=int(max_new_tokens), # DEMO slider
+        repetition_penalty=float(repetition_penalty),  # default 1.0 → etkisiz
         use_cache=False,
-        pad_token_id=pad_id,
-        eos_token_id=eos_for_gen,
-        length_penalty=1.0,
-        early_stopping=False,
-        # stopping_criteria vermiyoruz → LLaVA'daki KeywordsStoppingCriteria hatalarından kaçınmak için
+        stopping_criteria=[stopping],       # DEMO-benzeri durdurma
     )
 
-    if no_repeat_ngram_size:
-        try:
-            n = int(no_repeat_ngram_size)
-            if n > 0:
-                gen_kwargs["no_repeat_ngram_size"] = n
-        except Exception:
-            pass
-
-    if min_new_tokens is not None:
-        try:
-            mn = int(min_new_tokens)
-            if 1 <= mn <= int(max_new_tokens):
-                gen_kwargs["min_new_tokens"] = mn
-        except Exception:
-            pass
-
-    # Üretim (arka thread) + stream toplama
+    # Üretim (arka thread) + akışı topla
     try:
         t = Thread(target=chatbot.model.generate, kwargs=gen_kwargs)
         t.start()
-        chunks: List[str] = []
+        chunks = []
         for piece in streamer:
             chunks.append(piece)
         text = "".join(chunks)
-        # custom_stop varsa çıktıdan itibaren kırp
-        if custom_stop:
-            if isinstance(custom_stop, str):
-                custom_stop = [custom_stop]
-            for tag in custom_stop:
-                if isinstance(tag, str) and tag:
-                    idx = text.find(tag)
-                    if idx != -1:
-                        text = text[:idx].rstrip()
-                        break
         chatbot.conversation.messages[-1][-1] = text
     except Exception as e:
         return {"error": f"Generation failed: {e}"}
@@ -348,10 +325,10 @@ def generate_response(
     return {"status": "success", "response": text, "conversation_id": id(chatbot.conversation)}
 
 
-# ======================== Public API ========================
+# ===================== Public API =====================
 
 def query(payload: dict):
-    """HF Endpoint entry (demo-like streaming)"""
+    """HF Endpoint entry (demo parity)."""
     global model_initialized, tokenizer, model, image_processor, context_len, args
     if not model_initialized:
         if not initialize_model():
@@ -364,47 +341,27 @@ def query(payload: dict):
         if not message.strip(): return {"error": "Missing 'message' text"}
         if image is None:       return {"error": "Missing 'image'. Use 'image', 'image_url', or 'img'."}
 
-        # Demo-like varsayılanlar
-        max_new_tokens      = int(payload.get("max_output_tokens", payload.get("max_new_tokens", payload.get("max_tokens", 1800))))
-        min_new_tokens      = payload.get("min_new_tokens", 700)
-        try:
-            min_new_tokens = int(min_new_tokens) if min_new_tokens is not None else None
-        except Exception:
-            min_new_tokens = None
-
-        temperature         = float(payload.get("temperature", 0.20))
-        top_p               = float(payload.get("top_p", 0.95))
-        repetition_penalty  = float(payload.get("repetition_penalty", 1.20))
-        no_repeat_ngram     = payload.get("no_repeat_ngram_size", 6)
-        try:
-            no_repeat_ngram = int(no_repeat_ngram) if no_repeat_ngram is not None else None
-        except Exception:
-            no_repeat_ngram = None
+        # Demo varsayılanları — payload override edebilir
+        temperature        = float(payload.get("temperature", 0.05))
+        top_p              = float(payload.get("top_p", 1.0))
+        max_new_tokens     = int(payload.get("max_output_tokens", payload.get("max_new_tokens", payload.get("max_tokens", 4096))))
+        repetition_penalty = float(payload.get("repetition_penalty", 1.0))  # etkisiz default
 
-        conv_mode_override  = payload.get("conv_mode", None)
-        deterministic       = bool(payload.get("deterministic", False))
-        det_seed            = payload.get("det_seed", None)
+        conv_mode_override = payload.get("conv_mode", None)
+        det_seed           = payload.get("det_seed", None)
         if det_seed is not None:
             try: det_seed = int(det_seed)
             except Exception: det_seed = None
 
-        custom_stop         = payload.get("custom_stop", None)
-        no_stop             = bool(payload.get("no_stop", False))  # genelde False kalsın
-
         return generate_response(
             message_text=message,
             image_input=image,
-            max_new_tokens=max_new_tokens,
-            min_new_tokens=min_new_tokens,
             temperature=temperature,
             top_p=top_p,
-            repetition_penalty=repetition_penalty,
-            no_repeat_ngram_size=no_repeat_ngram,
+            max_new_tokens=max_new_tokens,
             conv_mode_override=conv_mode_override,
-            deterministic=deterministic,
+            repetition_penalty=repetition_penalty,
             det_seed=det_seed,
-            custom_stop=custom_stop,
-            no_stop=no_stop,
         )
     except Exception as e:
         return {"error": f"Query failed: {e}"}
@@ -428,7 +385,7 @@ def get_model_info():
     }
 
 
-# ======================== Init & Session ========================
+# ===================== Init & Session =====================
 
 class _Args:
     def __init__(self):
@@ -436,23 +393,22 @@ class _Args:
         self.model_base = None
         self.num_gpus   = int(os.getenv("NUM_GPUS", "1"))
         self.conv_mode  = None
-        self.max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "1800"))
+        self.max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "4096"))
         self.num_frames = 16
         self.load_8bit  = bool(int(os.getenv("LOAD_8BIT", "0")))
-        # 4bit/8bit hız için açık bırakılabilir; accelerate devicemap kullanıyorsanız .to(cuda) gerekmez
         self.load_4bit  = bool(int(os.getenv("LOAD_4BIT", "0")))
         self.debug      = bool(int(os.getenv("DEBUG", "0")))
 
 class InferenceDemo:
     def __init__(self, args, model_path, tokenizer_, model_, image_processor_, context_len_):
         if not LLAVA_AVAILABLE:
-            raise ImportError("LLaVA modules not available")
+            raise ImportError("LLaVA not available")
         disable_torch_init()
         self.tokenizer, self.model, self.image_processor, self.context_len = (
             tokenizer_, model_, image_processor_, context_len_
         )
-        conv_mode_auto = _guess_conv_mode(model_path)
-        self.conv_mode = args.conv_mode if args.conv_mode else conv_mode_auto
+        auto = _guess_conv_mode(model_path)
+        self.conv_mode = args.conv_mode if args.conv_mode else auto
         args.conv_mode = self.conv_mode
         self.conversation = conv_templates[self.conv_mode].copy()
         self.num_frames = args.num_frames
@@ -484,7 +440,7 @@ def initialize_model():
         tokenizer_, model_, image_processor_, context_len_ = load_pretrained_model(
             args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit
         )
-        # Device
+        # demo: model'ı genelde cuda’da çalıştırır
         try:
             _ = next(model_.parameters()).device
         except Exception:
@@ -506,7 +462,7 @@ def initialize_model():
         return False
 
 
-# ======================== HF EndpointHandler ========================
+# ===================== HF EndpointHandler =====================
 
 class EndpointHandler:
     """Hugging Face Endpoint uyumlu sınıf"""
@@ -523,4 +479,4 @@ class EndpointHandler:
         return get_model_info()
 
 if __name__ == "__main__":
-    print("Handler ready. Use `EndpointHandler` or `query` for HF Inference Endpoints.")
+    print("Handler ready (Demo Parity Mode). Use `EndpointHandler` or `query`.")