tests/test_run_vlm.py

import json
import tempfile
import unittest
from pathlib import Path

from approach.run_vlm import build_parser, run


class RunVlmTests(unittest.TestCase):
    def test_configurable_vlm_runner_writes_selected_records(self):
        calls = []

        def processor(profile, question, image_path, ablation, key_index):
            calls.append((profile, question, Path(image_path).name, ablation, key_index))
            return {"objects": {"button": "round red"}}

        with tempfile.TemporaryDirectory() as tmpdir:
            root = Path(tmpdir)
            questions = root / "questions.jsonl"
            questions.write_text(
                "\n".join(
                    [
                        json.dumps({"question_id": 0, "image": "123_4.jpg", "text": "mine"}),
                        json.dumps({"question_id": 1, "image": "456_7.jpg", "text": "mine"}),
                    ]
                )
                + "\n"
            )
            args = build_parser().parse_args(
                [
                    "--questions",
                    str(questions),
                    "--images-dir",
                    str(root / "images"),
                    "--output",
                    str(root / "answers.jsonl"),
                    "--start-index",
                    "0",
                    "--end-index",
                    "1",
                    "--profile",
                    "paper_claude",
                ]
            )

            report = run(args, processor=processor)
            output = root / "answers.rows0-1.jsonl"
            answer = json.loads(output.read_text().strip())

        self.assertEqual(report["completed"], 1)
        self.assertEqual(answer["question_id"], 0)
        self.assertEqual(answer["model_id"], "anthropic/claude-3.5-sonnet")
        self.assertEqual(calls[0][:4], ("paper_claude", "mine", "123_4.jpg", False))

    def test_default_processor_uses_app_metadata_cache_without_live_fallback(self):
        from approach.vlm.gpt4v.gpt4v import get_steam_app_data, load_app_metadata_cache

        with tempfile.TemporaryDirectory() as tmpdir:
            root = Path(tmpdir)
            cache_path = root / "metadata.json"
            cache_path.write_text(
                json.dumps(
                    {
                        "123": {
                            "app_name": "Test VR",
                            "app_description": "Synthetic cache-only description.",
                        }
                    }
                ),
                encoding="utf-8",
            )
            cache = load_app_metadata_cache(cache_path)

        self.assertEqual(
            get_steam_app_data("123", "123_4.jpg", cache),
            ("Test VR", "Synthetic cache-only description."),
        )
        with self.assertRaises(KeyError):
            get_steam_app_data("456", "456_7.jpg", cache)


if __name__ == "__main__":
    unittest.main()