"""Format samples from various VLM datasets into unified instruction-following format. LLaVA-1.5 proved that format prompts are critical: - VQA: "Answer the question using a single word or phrase." - MCQ: "Answer with the option's letter from the given choices directly." - Conversation: Full multi-turn instruction following """ def format_for_task(question: str, answer: str, task_type: str) -> dict: """Format a QA pair with task-appropriate instruction prompt. Args: question: The question or instruction answer: The expected answer task_type: One of "vqa", "mcq", "caption", "conversation", "detection", "alert", "counting", "ocr" Returns: dict with "instruction", "answer", "task_token" """ task_tokens = { "vqa": "", "mcq": "", "caption": "", "conversation": "", "detection": "", "alert": "", "counting": "", "ocr": "", "tracking": "", "reasoning": "", } task_token = task_tokens.get(task_type, "") if task_type == "vqa": instruction = f"{question}\nAnswer the question using a single word or phrase." elif task_type == "mcq": instruction = f"{question}\nAnswer with the option's letter from the given choices directly." elif task_type == "caption": instruction = "Describe this image in detail." if question: instruction = question elif task_type == "detection": instruction = f"{question}\nList all objects you can identify." if question else "List all objects visible in this image." elif task_type == "alert": instruction = f"{question}\nDescribe any anomalies or safety concerns." if question else "Describe any anomalies or safety concerns visible in this image." elif task_type == "counting": instruction = f"{question}\nProvide the count as a number." if question else "Count the objects in this image." elif task_type == "ocr": instruction = f"{question}\nRead and transcribe any visible text." if question else "Read and transcribe all visible text in this image." elif task_type == "conversation": instruction = question else: instruction = question return { "instruction": instruction, "answer": answer, "task_token": task_token, } def classify_dataset_task(dataset_name: str) -> str: """Map dataset name to default task type.""" mapping = { "vqav2": "vqa", "gqa": "vqa", "okvqa": "vqa", "aokvqa": "vqa", "textvqa": "ocr", "ocrvqa": "ocr", "textcaps": "caption", "llava_instruct": "conversation", "visual_genome": "vqa", "sharegpt": "conversation", "scienceqa": "mcq", "cc3m": "caption", "sbu": "caption", "laion_coco": "caption", "coco_detect": "detection", "visdrone": "detection", "mot": "tracking", "ucf_crime": "alert", "activitynet": "caption", "synthetic_rtsp": "detection", } return mapping.get(dataset_name, "vqa")