File size: 2,934 Bytes
3f3265f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | import json
import tempfile
import unittest
from pathlib import Path
from approach.run_vlm import build_parser, run
class RunVlmTests(unittest.TestCase):
def test_configurable_vlm_runner_writes_selected_records(self):
calls = []
def processor(profile, question, image_path, ablation, key_index):
calls.append((profile, question, Path(image_path).name, ablation, key_index))
return {"objects": {"button": "round red"}}
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
questions = root / "questions.jsonl"
questions.write_text(
"\n".join(
[
json.dumps({"question_id": 0, "image": "123_4.jpg", "text": "mine"}),
json.dumps({"question_id": 1, "image": "456_7.jpg", "text": "mine"}),
]
)
+ "\n"
)
args = build_parser().parse_args(
[
"--questions",
str(questions),
"--images-dir",
str(root / "images"),
"--output",
str(root / "answers.jsonl"),
"--start-index",
"0",
"--end-index",
"1",
"--profile",
"paper_claude",
]
)
report = run(args, processor=processor)
output = root / "answers.rows0-1.jsonl"
answer = json.loads(output.read_text().strip())
self.assertEqual(report["completed"], 1)
self.assertEqual(answer["question_id"], 0)
self.assertEqual(answer["model_id"], "anthropic/claude-3.5-sonnet")
self.assertEqual(calls[0][:4], ("paper_claude", "mine", "123_4.jpg", False))
def test_default_processor_uses_app_metadata_cache_without_live_fallback(self):
from approach.vlm.gpt4v.gpt4v import get_steam_app_data, load_app_metadata_cache
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
cache_path = root / "metadata.json"
cache_path.write_text(
json.dumps(
{
"123": {
"app_name": "Test VR",
"app_description": "Synthetic cache-only description.",
}
}
),
encoding="utf-8",
)
cache = load_app_metadata_cache(cache_path)
self.assertEqual(
get_steam_app_data("123", "123_4.jpg", cache),
("Test VR", "Synthetic cache-only description."),
)
with self.assertRaises(KeyError):
get_steam_app_data("456", "456_7.jpg", cache)
if __name__ == "__main__":
unittest.main()
|