experiment/config/eval_config.py

"""
Evaluation Configuration — All eval pipeline knobs in one place.
"""

from __future__ import annotations

from dataclasses import dataclass, field
from typing import Literal


@dataclass
class EvalConfig:
    num_per_category: int = 50
    prompts: list[str] = field(default_factory=lambda: [
        "Describe this image.",
        "Is there a toilet in this image?",
    ])
    max_new_tokens: int = 300

    # Model loading
    model_type: Literal["lora", "merged", "delta_w"] = "lora"
    base_model_name: str = "llava-hf/llava-1.5-7b-hf"

    # Mention detection
    mention_method: Literal["keyword", "llm", "both"] = "both"

    # Image-text alignment model (SigLIP/CLIP style)
    clip_model: str = "google/siglip-base-patch16-224"

    # Local LLM Judge
    judge_model: str = "Qwen/Qwen3-VL-32B-Instruct"
    judge_device: str = "cuda"   # use "cuda:1" when model runs on cuda:0
    judge_max_tokens: int = 150

    # Inference backend
    inference_backend: Literal["transformers", "vllm"] = "vllm"
    vllm_batch_size: int = 64
    vllm_tensor_parallel_size: int = 1
    vllm_gpu_memory_utilization: float = 0.9
    vllm_max_model_len: int = 4096