File size: 4,836 Bytes
3f3265f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | import json
import sys
import tempfile
import unittest
from itertools import chain, repeat
from pathlib import Path
from unittest import mock
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from evaluation import context_eval_baselines
from evaluation.tools import to_pred
class ToPredTests(unittest.TestCase):
def test_task_is_required(self):
parser = to_pred.build_parser()
with self.assertRaises(SystemExit):
parser.parse_args(["--input", "pred.json", "--output", "out.json"])
def test_convert_old_predictions_to_semantics_fields(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pred_path = tmp / "pred.json"
question_path = tmp / "questions.jsonl"
output_path = tmp / "out.json"
pred_path.write_text(
json.dumps(
{
"q1": {
"oovd_result": {
"button": [
{
"bbox_pixels": [1, 2, 3, 4],
"probability": 0.7,
}
]
}
}
}
)
)
question_path.write_text(
json.dumps({"question_id": "q1", "image": "1026760_11.jpg"}) + "\n"
)
to_pred.main(
[
"--task",
"semantics",
"--input",
str(pred_path),
"--questions",
str(question_path),
"--output",
str(output_path),
]
)
converted = json.loads(output_path.read_text())
self.assertEqual(
converted,
[
{
"image_id": 1026760011,
"category_id": "button",
"bbox": [1, 2, 3, 4],
"score": 0.7,
}
],
)
def test_all_outputs_do_not_overwrite(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmp = Path(tmpdir)
pred_path = tmp / "pred.json"
output_path = tmp / "converted.json"
pred_path.write_text(
json.dumps(
[
{
"image_id": 1,
"category_id": "trigger",
"bbox": [1, 2, 3, 4],
"score": 0.5,
}
]
)
)
to_pred.main(
[
"--task",
"all",
"--input",
str(pred_path),
"--output",
str(output_path),
]
)
self.assertFalse(output_path.exists())
for task in to_pred.TASKS:
self.assertTrue((tmp / f"converted_{task}.json").exists())
interactable = json.loads((tmp / "converted_interactable.json").read_text())
interaction = json.loads((tmp / "converted_interaction.json").read_text())
semantics = json.loads((tmp / "converted_semantics.json").read_text())
self.assertEqual(interactable[0]["category_id"], 1)
self.assertEqual(interaction[0]["category_id"], "trigger")
self.assertEqual(semantics[0]["category_id"], "trigger")
class ContextEvalBaselineTests(unittest.TestCase):
def test_validate_unique_methods_rejects_duplicates(self):
with self.assertRaises(ValueError):
context_eval_baselines.validate_unique_methods(["Seed-E2E", "Seed-E2E"])
def test_main_uses_subprocess_check_true(self):
args = context_eval_baselines.build_parser().parse_args([])
methods = ["CenterNet2"]
with mock.patch.object(
context_eval_baselines, "METHODS", methods
), mock.patch.object(
context_eval_baselines, "LLM_METHODS", []
), mock.patch.object(
context_eval_baselines.os.path,
"exists",
side_effect=chain([True, False], repeat(False)),
), mock.patch.object(
context_eval_baselines.subprocess, "run"
) as run:
context_eval_baselines.main(args)
self.assertTrue(run.called)
self.assertTrue(run.call_args.kwargs["check"])
self.assertIsInstance(run.call_args.args[0], list)
if __name__ == "__main__":
unittest.main()
|