Hugging Face's logo

Spaces:
Viviannnyang
/
VideoGLaMM_ActivityNet_Object_Review20
Running

App Files Community
VideoGLaMM_ActivityNet_Object_Review20 / data /annotations.json
Viviannnyang's picture
Viviannnyang
Publish 20 completed ActivityNet GCG object annotations and mask review UI
a4d1b37 verified
Raw History Blame Contribute Delete
200 kB
[
{
"sample_id": "v_fUJ88Ir7Dgc____1",
"video_id": "v_fUJ88Ir7Dgc",
"segment_id": "1",
"selection_index": 1,
"caption": "In the video, the athlete is seen running down a track and jumping into a pit. He then sits down while another stands next to him and prepares.",
"caption_sha256": "e9272bf3689d5986d2bd30c1b01832876f25b66c84d1a3bb1189b95bef37ebdd",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, the <p> athlete </p> [SEG:0] is seen running down a track and jumping into a pit. <p> He </p> [SEG:0] then sits down while another stands next to <p> him </p> [SEG:0] and prepares.",
"reference_frame_id": 6,
"source_timestamps": [
26.37,
57.53
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_fUJ88Ir7Dgc____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"athlete",
"he"
],
"reference_frame_id": 6,
"source_bbox": [
373,
193,
525,
357
],
"source_mask_member": "activitynet_entities_gcg/masks/v_fUJ88Ir7Dgc____1/00/mask.png",
"mask_path": "samples/v_fUJ88Ir7Dgc____1/m0_mask.png",
"source_mask_sha256": "b7ed1fc6ce779362c63d28ef63d4a39feda01e02a54d2eef69f0dcc8e97d687c",
"mask_foreground_pixels": 12202,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "seated athlete in the foreground",
"entity_kind": "thing",
"mentions": [
{
"text": "the athlete",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
25
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
79,
81
]
},
{
"text": "him",
"occurrence": 1,
"model_occurrence": 1,
"span": [
126,
129
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The supplied mask covers the seated foreground athlete; both pronouns refer to him.",
"char_spans": [
[
14,
25
],
[
79,
81
],
[
126,
129
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 6,
"mask_path": "samples/v_fUJ88Ir7Dgc____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_fUJ88Ir7Dgc____1/00/mask.png",
"source_mask_sha256": "b7ed1fc6ce779362c63d28ef63d4a39feda01e02a54d2eef69f0dcc8e97d687c"
}
},
{
"group_id": "g2",
"entity_label": "athletics track",
"entity_kind": "stuff",
"mentions": [
{
"text": "a track",
"occurrence": 1,
"model_occurrence": 1,
"span": [
47,
54
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reddish track surface is visible in the reference frame, but no track mask is supplied.",
"char_spans": [
[
47,
54
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "jumping pit",
"entity_kind": "thing",
"mentions": [
{
"text": "a pit",
"occurrence": 1,
"model_occurrence": 1,
"span": [
72,
77
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "medium",
"reason": "No distinct jumping pit is identifiable in the reference frame or verifiable in the supplied context frames.",
"char_spans": [
[
72,
77
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "other athlete standing to the left of the seated athlete",
"entity_kind": "thing",
"mentions": [
{
"text": "another",
"occurrence": 1,
"model_occurrence": 1,
"span": [
103,
110
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The other athlete is visible standing on the left in dark sportswear, but has no supplied mask.",
"char_spans": [
[
103,
110
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "caption_visual_mismatch",
"text": "a pit",
"description": "The caption names a jumping pit, but none can be verified in the supplied images; the caption is preserved unchanged."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The visible track and other athlete lack supplied masks.",
"The caption's jumping pit is not visually verifiable and has no supplied mask."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 3,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0924b2ce8f609b03016ae4c5e758b087d1b66d81039e33bc44",
"api_usage": {
"input_tokens": 15933,
"input_tokens_details": {
"cache_write_tokens": 6430,
"cached_tokens": 9500
},
"output_tokens": 805,
"output_tokens_details": {
"reasoning_tokens": 126
},
"total_tokens": 16738
},
"request_custom_id": "gcg-object20::v_fUJ88Ir7Dgc____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_fUJ88Ir7Dgc____1",
"estimated_frame_times_s": [
27.928,
31.044,
34.160000000000004,
37.276,
40.392,
43.507999999999996,
46.623999999999995,
49.739999999999995,
52.856,
55.971999999999994
],
"new_caption_original": "The <p> athlete </p> [SEG:0] then sits down while another stands next to <p> him </p> [SEG:0] and prepares"
},
{
"sample_id": "v_JBnHqQjeVMk____1",
"video_id": "v_JBnHqQjeVMk",
"segment_id": "1",
"selection_index": 2,
"caption": "Cowboys are practicing their lasso work. In the video, a young boy is seen swinging a rope in a circle while a woman is standing next to him, watching. The boy then swings the rope around and around.",
"caption_sha256": "1827555ff6db1afb1d9faf572587383de4e4f195eeca3fddebacc4407f75f79b",
"caption_source": "refined_caption",
"refined_caption_original": "Cowboys are practicing <p> their </p> [SEG:0] lasso work. In the video, a young boy is seen swinging a rope in a circle while a woman is standing next to him, watching. The boy then swings the rope around and around.",
"reference_frame_id": 3,
"source_timestamps": [
6.91,
7.94
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_JBnHqQjeVMk____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"they"
],
"reference_frame_id": 3,
"source_bbox": [
317,
115,
383,
289
],
"source_mask_member": "activitynet_entities_gcg/masks/v_JBnHqQjeVMk____1/00/mask.png",
"mask_path": "samples/v_JBnHqQjeVMk____1/m0_mask.png",
"source_mask_sha256": "d916600c53ae429a87adf654dbbab02500e455a2f8053e88301412af4f6fed75",
"mask_foreground_pixels": 5254,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Cowboys practicing roping",
"entity_kind": "collective",
"mentions": [
{
"text": "Cowboys",
"occurrence": 1,
"model_occurrence": 1,
"span": [
0,
7
]
},
{
"text": "their",
"occurrence": 1,
"model_occurrence": 1,
"span": [
23,
28
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The plural refers to the visible roping participants as a group. Only an individual boy mask is supplied; it cannot represent this collective.",
"char_spans": [
[
0,
7
],
[
23,
28
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "Young boy in a red shirt and dark cowboy hat",
"entity_kind": "thing",
"mentions": [
{
"text": "a young boy",
"occurrence": 1,
"model_occurrence": 1,
"span": [
55,
66
]
},
{
"text": "him",
"occurrence": 1,
"model_occurrence": 1,
"span": [
137,
140
]
},
{
"text": "The boy",
"occurrence": 1,
"model_occurrence": 1,
"span": [
152,
159
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Mask m0 covers the central red-shirted boy swinging the rope.",
"char_spans": [
[
55,
66
],
[
137,
140
],
[
152,
159
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 3,
"mask_path": "samples/v_JBnHqQjeVMk____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_JBnHqQjeVMk____1/00/mask.png",
"source_mask_sha256": "d916600c53ae429a87adf654dbbab02500e455a2f8053e88301412af4f6fed75"
}
},
{
"group_id": "g3",
"entity_label": "Rope swung by the boy",
"entity_kind": "thing",
"mentions": [
{
"text": "a rope",
"occurrence": 1,
"model_occurrence": 1,
"span": [
84,
90
]
},
{
"text": "the rope",
"occurrence": 1,
"model_occurrence": 1,
"span": [
172,
180
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The thin rope is discernible near the boy's raised hand and body, with its movement clearer across context frames. No separate rope mask is supplied.",
"char_spans": [
[
84,
90
],
[
172,
180
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "Woman in a light green shirt beside the boy",
"entity_kind": "thing",
"mentions": [
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
109,
116
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The light-green-shirted woman stands immediately to the boy's right in the reference frame. No mask for her is supplied.",
"char_spans": [
[
109,
116
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The cowboys collective, the boy's rope, and the woman lack matching supplied masks."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 3,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_00a0fb98fdbafac5016ae4c5e8ed9c87d19cda64daadccfbe9",
"api_usage": {
"input_tokens": 15339,
"input_tokens_details": {
"cache_write_tokens": 5836,
"cached_tokens": 9500
},
"output_tokens": 906,
"output_tokens_details": {
"reasoning_tokens": 173
},
"total_tokens": 16245
},
"request_custom_id": "gcg-object20::v_JBnHqQjeVMk____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_JBnHqQjeVMk____1",
"estimated_frame_times_s": [
6.9615,
7.0645,
7.1675,
7.2705,
7.3735,
7.476500000000001,
7.5795,
7.6825,
7.785500000000001,
7.8885000000000005
],
"new_caption_original": "Cowboys practice <p> their </p> [SEG:0] lasso work"
},
{
"sample_id": "v_gTh9bd9yV9E____0",
"video_id": "v_gTh9bd9yV9E",
"segment_id": "0",
"selection_index": 3,
"caption": "In the video, A teen is putting makeup on a man 's face . The man and a woman are seen sitting in a kitchen. The woman is wiping her face with a rag, and the man is rubbing his face with his hand. The woman then wipes her face again, while the man continues to rub his face.",
"caption_sha256": "ed899e06efedc0862bf2a3b27ceaf2b99799e8ab5626c2eff9d5d9c6aaaac99b",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> A teen </p> [SEG:1] is putting makeup on a <p> man </p> [SEG:0]'s <p> face </p> [SEG:2]. The man and a woman are seen sitting in a kitchen. The woman is wiping her face with a rag, and the man is rubbing his face with his hand. The woman then wipes her face again, while the man continues to rub his face.",
"reference_frame_id": 1,
"source_timestamps": [
8.63,
51.76
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_gTh9bd9yV9E____0.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"man",
"he"
],
"reference_frame_id": 1,
"source_bbox": [
345,
21,
716,
402
],
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/00/mask.png",
"mask_path": "samples/v_gTh9bd9yV9E____0/m0_mask.png",
"source_mask_sha256": "f344dcf0e487548cde8660dba34b2c7f594cbe1db222ece8d9afdfa409f19b58",
"mask_foreground_pixels": 76044,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"teen"
],
"reference_frame_id": 1,
"source_bbox": [
2,
2,
544,
401
],
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/01/mask.png",
"mask_path": "samples/v_gTh9bd9yV9E____0/m1_mask.png",
"source_mask_sha256": "97533073e666f801f8879cf481d316d2dd1e0bf57596cb9d970cb361c61fe0c2",
"mask_foreground_pixels": 113158,
"color": "#db2777"
},
{
"mask_id": "m2",
"source_seg_token": "[SEG:2]",
"source_classes": [
"face"
],
"reference_frame_id": 1,
"source_bbox": [
467,
45,
619,
210
],
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/02/mask.png",
"mask_path": "samples/v_gTh9bd9yV9E____0/m2_mask.png",
"source_mask_sha256": "47e3fd2a47b3a74f12e638fc3771dfaf283dfc0eb18641d96ab91891985f4db1",
"mask_foreground_pixels": 19590,
"color": "#16a34a"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Young woman applying makeup on the left",
"entity_kind": "thing",
"mentions": [
{
"text": "A teen",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
20
]
},
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
70,
77
]
},
{
"text": "The woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
109,
118
]
},
{
"text": "her",
"occurrence": 1,
"model_occurrence": 1,
"span": [
129,
132
]
},
{
"text": "The woman",
"occurrence": 2,
"model_occurrence": 2,
"span": [
197,
206
]
},
{
"text": "her",
"occurrence": 2,
"model_occurrence": 2,
"span": [
218,
221
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The teen and subsequent woman references identify the same woman on the left; m1 covers her.",
"char_spans": [
[
14,
20
],
[
70,
77
],
[
109,
118
],
[
129,
132
],
[
197,
206
],
[
218,
221
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 1,
"mask_path": "samples/v_gTh9bd9yV9E____0/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/01/mask.png",
"source_mask_sha256": "97533073e666f801f8879cf481d316d2dd1e0bf57596cb9d970cb361c61fe0c2"
}
},
{
"group_id": "g2",
"entity_label": "Makeup being applied",
"entity_kind": "stuff",
"mentions": [
{
"text": "makeup",
"occurrence": 1,
"model_occurrence": 1,
"span": [
32,
38
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "uncertain",
"confidence": "medium",
"reason": "Application is shown, but the cosmetic substance is not clearly distinguishable as a separate region, and no makeup mask is supplied.",
"char_spans": [
[
32,
38
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "Man seated on the right",
"entity_kind": "thing",
"mentions": [
{
"text": "a man 's",
"occurrence": 1,
"model_occurrence": 1,
"span": [
42,
50
]
},
{
"text": "The man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
58,
65
]
},
{
"text": "the man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
154,
161
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
173,
176
]
},
{
"text": "his",
"occurrence": 2,
"model_occurrence": 2,
"span": [
187,
190
]
},
{
"text": "the man",
"occurrence": 2,
"model_occurrence": 2,
"span": [
240,
247
]
},
{
"text": "his",
"occurrence": 3,
"model_occurrence": 3,
"span": [
265,
268
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "All man references and masculine possessives identify the seated man covered by m0.",
"char_spans": [
[
42,
50
],
[
58,
65
],
[
154,
161
],
[
173,
176
],
[
187,
190
],
[
240,
247
],
[
265,
268
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 1,
"mask_path": "samples/v_gTh9bd9yV9E____0/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/00/mask.png",
"source_mask_sha256": "f344dcf0e487548cde8660dba34b2c7f594cbe1db222ece8d9afdfa409f19b58"
}
},
{
"group_id": "g4",
"entity_label": "Man's face",
"entity_kind": "body_part",
"mentions": [
{
"text": "a man 's face",
"occurrence": 1,
"model_occurrence": 1,
"span": [
42,
55
]
},
{
"text": "his face",
"occurrence": 1,
"model_occurrence": 1,
"span": [
173,
181
]
},
{
"text": "his face",
"occurrence": 2,
"model_occurrence": 2,
"span": [
265,
273
]
}
],
"mask_id": "m2",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "m2 isolates the man's visible facial/head region, excluding the woman's occluding hand.",
"char_spans": [
[
42,
55
],
[
173,
181
],
[
265,
273
]
],
"mask_evidence": {
"mask_id": "m2",
"source_seg_token": "[SEG:2]",
"reference_frame_id": 1,
"mask_path": "samples/v_gTh9bd9yV9E____0/m2_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_gTh9bd9yV9E____0/02/mask.png",
"source_mask_sha256": "47e3fd2a47b3a74f12e638fc3771dfaf283dfc0eb18641d96ab91891985f4db1"
}
},
{
"group_id": "g5",
"entity_label": "Kitchen",
"entity_kind": "thing",
"mentions": [
{
"text": "a kitchen",
"occurrence": 1,
"model_occurrence": 1,
"span": [
98,
107
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The kitchen interior is visible behind the people, but no corresponding mask is supplied.",
"char_spans": [
[
98,
107
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "Woman's face",
"entity_kind": "body_part",
"mentions": [
{
"text": "her face",
"occurrence": 1,
"model_occurrence": 1,
"span": [
129,
137
]
},
{
"text": "her face",
"occurrence": 2,
"model_occurrence": 2,
"span": [
218,
226
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The woman's face is visible, but m1 is a whole-person mask and m2 belongs to the man.",
"char_spans": [
[
129,
137
],
[
218,
226
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "Rag described as used by the woman",
"entity_kind": "thing",
"mentions": [
{
"text": "a rag",
"occurrence": 1,
"model_occurrence": 1,
"span": [
143,
148
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "No identifiable rag is observed in the supplied frames; the small cosmetic items are not evidence of a rag.",
"char_spans": [
[
143,
148
]
],
"mask_evidence": null
},
{
"group_id": "g8",
"entity_label": "Man's hand",
"entity_kind": "body_part",
"mentions": [
{
"text": "his hand",
"occurrence": 1,
"model_occurrence": 1,
"span": [
187,
195
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "context_only",
"confidence": "medium",
"reason": "The man's raised hand is glimpsed with motion blur in context sample 03, but is not identifiable in the reference frame and has no separate mask.",
"char_spans": [
[
187,
195
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "caption_visual_mismatch",
"text": "The woman is wiping her face with a rag",
"description": "The supplied frames show the woman applying makeup to the man's face, not wiping her own face with an identifiable rag."
},
{
"type": "caption_visual_mismatch",
"text": "The woman then wipes her face again",
"description": "The repeated own-face wiping description is not supported by the sampled frames."
}
],
"model_needs_review": true,
"model_review_reasons": [
"Makeup, kitchen, and woman's face lack supplied masks.",
"The rag is not observed, and the man's hand is only visible in context.",
"The visibility of the makeup substance as a separate entity is uncertain.",
"The caption's descriptions of the woman wiping her own face conflict with the sampled visual evidence."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g6"
],
"spans": [
[
129,
132
],
[
129,
137
]
]
},
{
"groups": [
"g1",
"g6"
],
"spans": [
[
218,
221
],
[
218,
226
]
]
},
{
"groups": [
"g3",
"g4"
],
"spans": [
[
42,
50
],
[
42,
55
]
]
},
{
"groups": [
"g3",
"g4"
],
"spans": [
[
173,
176
],
[
173,
181
]
]
},
{
"groups": [
"g3",
"g4"
],
"spans": [
[
265,
268
],
[
265,
273
]
]
},
{
"groups": [
"g3",
"g8"
],
"spans": [
[
187,
190
],
[
187,
195
]
]
}
],
"unresolved_groups": 5,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_09e0029ebd6a84a9016ae4c5e8fb7487d18f3c357cda1c7489",
"api_usage": {
"input_tokens": 15990,
"input_tokens_details": {
"cache_write_tokens": 6487,
"cached_tokens": 9500
},
"output_tokens": 1732,
"output_tokens_details": {
"reasoning_tokens": 255
},
"total_tokens": 17722
},
"request_custom_id": "gcg-object20::v_gTh9bd9yV9E____0",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_gTh9bd9yV9E____0",
"estimated_frame_times_s": [
10.7865,
15.099499999999999,
19.4125,
23.725499999999997,
28.0385,
32.3515,
36.6645,
40.9775,
45.2905,
49.6035
],
"new_caption_original": "A <p> teen </p> [SEG:1] puts make up to an <p> man </p> [SEG:0] on <p> his </p> [SEG:0] <p> face </p> [SEG:2]"
},
{
"sample_id": "v_RQO61KXeBos____1",
"video_id": "v_RQO61KXeBos",
"segment_id": "1",
"selection_index": 4,
"caption": "In the video, a man is seen taking his turn. He uses a stick to get over an obstacle and jumps up with excitement after landing.",
"caption_sha256": "79a1e9c7e043b91c4654194b4f7f325e83c3613d305bf5d28be21743b80a156b",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:1] is seen taking <p> his </p> [SEG:1] turn. He uses a <p> stick </p> [SEG:0] to get over an obstacle and jumps up with excitement after landing.",
"reference_frame_id": 1,
"source_timestamps": [
39.4,
82.38
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_RQO61KXeBos____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"stick"
],
"reference_frame_id": 1,
"source_bbox": [
112,
134,
631,
439
],
"source_mask_member": "activitynet_entities_gcg/masks/v_RQO61KXeBos____1/00/mask.png",
"mask_path": "samples/v_RQO61KXeBos____1/m0_mask.png",
"source_mask_sha256": "fe03d68eafeb60b7347b49c320e11be2ebe5e3fcb50652a6feb77000f26e2e3c",
"mask_foreground_pixels": 18255,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"man",
"he",
"he"
],
"reference_frame_id": 1,
"source_bbox": [
134,
133,
407,
385
],
"source_mask_member": "activitynet_entities_gcg/masks/v_RQO61KXeBos____1/01/mask.png",
"mask_path": "samples/v_RQO61KXeBos____1/m1_mask.png",
"source_mask_sha256": "3e8df7acdc9996a196e12852f6731e16ae3e6e12992ecdd5d9696d498ee451fc",
"mask_foreground_pixels": 12096,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "male pole vaulter",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
35,
38
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
45,
47
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reference-frame athlete holding the bent vaulting pole is isolated by m1; all three mentions refer to him.",
"char_spans": [
[
14,
19
],
[
35,
38
],
[
45,
47
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 1,
"mask_path": "samples/v_RQO61KXeBos____1/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_RQO61KXeBos____1/01/mask.png",
"source_mask_sha256": "3e8df7acdc9996a196e12852f6731e16ae3e6e12992ecdd5d9696d498ee451fc"
}
},
{
"group_id": "g2",
"entity_label": "vaulting pole",
"entity_kind": "thing",
"mentions": [
{
"text": "a stick",
"occurrence": 1,
"model_occurrence": 1,
"span": [
53,
60
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m0"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The bent pole is clearly visible, but the supplied m0 binary mask includes the athlete as well as the pole, rather than isolating the pole.",
"char_spans": [
[
53,
60
]
],
"model_mask_id": "m0",
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "vaulting obstacle or crossbar",
"entity_kind": "thing",
"mentions": [
{
"text": "an obstacle",
"occurrence": 1,
"model_occurrence": 1,
"span": [
73,
84
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "uncertain",
"confidence": "medium",
"reason": "The pole-vault context suggests a crossbar, but the specific obstacle is not confidently distinguishable in the reference frame. Neither supplied mask isolates it.",
"char_spans": [
[
73,
84
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "a stick",
"description": "The m0 binary-mask panel contains both the pole and the athlete, despite the overlay presenting it as the pole mask. The spatial annotation needs correction."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The pole mask m0 includes the athlete and is not an isolated entity mask.",
"The obstacle has no supplied mask, and its reference-frame visibility and precise identity require verification."
],
"local_adjustments": [
{
"type": "unconfirmed_quality_mask_to_candidate",
"group_id": "g2",
"model_mask_id": "m0",
"resolved_mask_id": null,
"reason": "Preserve the proposed ID as a review candidate, not a confirmed correspondence."
}
],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 2,
"unused_source_mask_ids": [
"m0"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0bde5d33bdfece93016ae4c5e9804c87d1a988325843d709f8",
"api_usage": {
"input_tokens": 16567,
"input_tokens_details": {
"cache_write_tokens": 7064,
"cached_tokens": 9500
},
"output_tokens": 844,
"output_tokens_details": {
"reasoning_tokens": 245
},
"total_tokens": 17411
},
"request_custom_id": "gcg-object20::v_RQO61KXeBos____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_RQO61KXeBos____1",
"estimated_frame_times_s": [
41.549,
45.847,
50.144999999999996,
54.443,
58.741,
63.039,
67.337,
71.63499999999999,
75.93299999999999,
80.231
],
"new_caption_original": "The next <p> man </p> [SEG:1] takes <p> his </p> [SEG:1] turn and <p> he </p> [SEG:1] does the same using the <p> stick </p> [SEG:0] to get over the obstacle and jumping up with excitement after landing"
},
{
"sample_id": "v_XHQLkc06XIc____1",
"video_id": "v_XHQLkc06XIc",
"segment_id": "1",
"selection_index": 5,
"caption": "In the video, a young girl is playing the violin in a joyful and professional manner. She is standing in a room, wearing a white shirt and shorts, and is holding the violin with both hands. With her left hand, she controls the strings, and her right hand manipulates the bow. Her focused body language indicates a deep engagement in the performance, with precise hand movements and expressive gestures.",
"caption_sha256": "4eb8d1ebd1f5090da9410cf96af5d6ecf76c7bbc812c14ef7142bf9e767c4907",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a young girl </p> [SEG:0] is playing the <p> violin </p> [SEG:1] in a joyful and professional manner. She is standing in a room, wearing a white shirt and shorts, and is holding the violin with both hands. With her left hand, she controls the strings, and her right hand manipulates the bow. Her focused body language indicates a deep engagement in the performance, with precise hand movements and expressive gestures.",
"reference_frame_id": 4,
"source_timestamps": [
14.21,
141.12
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_XHQLkc06XIc____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"she"
],
"reference_frame_id": 4,
"source_bbox": [
161,
97,
546,
539
],
"source_mask_member": "activitynet_entities_gcg/masks/v_XHQLkc06XIc____1/00/mask.png",
"mask_path": "samples/v_XHQLkc06XIc____1/m0_mask.png",
"source_mask_sha256": "ae3e8e4ad332f27922cd77b2d2ca13d2cba8e6e970a2ee053b40b2e3ba5bb764",
"mask_foreground_pixels": 72590,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"violin"
],
"reference_frame_id": 4,
"source_bbox": [
263,
180,
607,
330
],
"source_mask_member": "activitynet_entities_gcg/masks/v_XHQLkc06XIc____1/01/mask.png",
"mask_path": "samples/v_XHQLkc06XIc____1/m1_mask.png",
"source_mask_sha256": "40c5db480132346e832832aad705e5fec595ed14273eb89b39015fda2458187d",
"mask_foreground_pixels": 12212,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "young girl playing violin",
"entity_kind": "thing",
"mentions": [
{
"text": "a young girl",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
26
]
},
{
"text": "She",
"occurrence": 1,
"model_occurrence": 1,
"span": [
86,
89
]
},
{
"text": "her",
"occurrence": 1,
"model_occurrence": 1,
"span": [
195,
198
]
},
{
"text": "she",
"occurrence": 1,
"model_occurrence": 1,
"span": [
210,
213
]
},
{
"text": "her",
"occurrence": 2,
"model_occurrence": 2,
"span": [
240,
243
]
},
{
"text": "Her",
"occurrence": 1,
"model_occurrence": 1,
"span": [
276,
279
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The person mask identifies the girl throughout the caption.",
"char_spans": [
[
14,
26
],
[
86,
89
],
[
195,
198
],
[
210,
213
],
[
240,
243
],
[
276,
279
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 4,
"mask_path": "samples/v_XHQLkc06XIc____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_XHQLkc06XIc____1/00/mask.png",
"source_mask_sha256": "ae3e8e4ad332f27922cd77b2d2ca13d2cba8e6e970a2ee053b40b2e3ba5bb764"
}
},
{
"group_id": "g2",
"entity_label": "violin",
"entity_kind": "thing",
"mentions": [
{
"text": "the violin",
"occurrence": 1,
"model_occurrence": 1,
"span": [
38,
48
]
},
{
"text": "the violin",
"occurrence": 2,
"model_occurrence": 2,
"span": [
162,
172
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m1"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The mask identifies the violin but also includes the separate bow, so it is not a clean violin-only mask.",
"char_spans": [
[
38,
48
],
[
162,
172
]
],
"model_mask_id": "m1",
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "room",
"entity_kind": "thing",
"mentions": [
{
"text": "a room",
"occurrence": 1,
"model_occurrence": 1,
"span": [
105,
111
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The room interior is visible, but no room mask is supplied.",
"char_spans": [
[
105,
111
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "white shirt",
"entity_kind": "thing",
"mentions": [
{
"text": "a white shirt",
"occurrence": 1,
"model_occurrence": 1,
"span": [
121,
134
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The shirt is visible; the person mask does not supply a separate clothing mask.",
"char_spans": [
[
121,
134
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "shorts",
"entity_kind": "thing",
"mentions": [
{
"text": "shorts",
"occurrence": 1,
"model_occurrence": 1,
"span": [
139,
145
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The shorts are partly visible at the bottom of the reference frame and more fully visible in context frames; no separate mask is supplied.",
"char_spans": [
[
139,
145
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "girl's left hand",
"entity_kind": "body_part",
"mentions": [
{
"text": "both hands",
"occurrence": 1,
"model_occurrence": 1,
"span": [
178,
188
]
},
{
"text": "her left hand",
"occurrence": 1,
"model_occurrence": 1,
"span": [
195,
208
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The left hand on the violin neck is visible without a separate mask. The explicit pair and subsequent left/right distinctions establish two hand groups sharing the pair mention.",
"char_spans": [
[
178,
188
],
[
195,
208
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "girl's right hand",
"entity_kind": "body_part",
"mentions": [
{
"text": "both hands",
"occurrence": 1,
"model_occurrence": 1,
"span": [
178,
188
]
},
{
"text": "her right hand",
"occurrence": 1,
"model_occurrence": 1,
"span": [
240,
254
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The right hand holding the bow is visible, but no separate hand mask is supplied.",
"char_spans": [
[
178,
188
],
[
240,
254
]
],
"mask_evidence": null
},
{
"group_id": "g8",
"entity_label": "violin strings",
"entity_kind": "collective",
"mentions": [
{
"text": "the strings",
"occurrence": 1,
"model_occurrence": 1,
"span": [
223,
234
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The strings are visible along the violin, though fine detail is limited. No separate strings mask is supplied.",
"char_spans": [
[
223,
234
]
],
"mask_evidence": null
},
{
"group_id": "g9",
"entity_label": "violin bow",
"entity_kind": "thing",
"mentions": [
{
"text": "the bow",
"occurrence": 1,
"model_occurrence": 1,
"span": [
267,
274
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The bow is clearly visible. Although included in the combined violin mask m1, it has no separate source mask and cannot be linked to that mask as its own entity.",
"char_spans": [
[
267,
274
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "the violin",
"description": "Source mask m1 combines violin and bow pixels, preventing a clean one-entity correspondence."
},
{
"type": "caption_visual_mismatch",
"text": "holding the violin with both hands",
"description": "The left hand supports the violin neck while the right hand holds the separate bow, rather than both hands directly holding the violin."
}
],
"model_needs_review": true,
"model_review_reasons": [
"Separate masks are missing for the room, clothing, hands, strings, and bow.",
"The violin source mask includes the bow and requires spatial correction.",
"The caption's description of holding the violin with both hands does not precisely match the visible hand-object relationships."
],
"local_adjustments": [
{
"type": "unconfirmed_quality_mask_to_candidate",
"group_id": "g2",
"model_mask_id": "m1",
"resolved_mask_id": null,
"reason": "Preserve the proposed ID as a review candidate, not a confirmed correspondence."
}
],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g6"
],
"spans": [
[
195,
198
],
[
195,
208
]
]
},
{
"groups": [
"g1",
"g7"
],
"spans": [
[
240,
243
],
[
240,
254
]
]
},
{
"groups": [
"g6",
"g7"
],
"spans": [
[
178,
188
],
[
178,
188
]
]
}
],
"unresolved_groups": 8,
"unused_source_mask_ids": [
"m1"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_041f4a98ea68edfe016ae4c5e9c25087d1950afa0bc284498e",
"api_usage": {
"input_tokens": 16908,
"input_tokens_details": {
"cache_write_tokens": 7405,
"cached_tokens": 9500
},
"output_tokens": 1716,
"output_tokens_details": {
"reasoning_tokens": 229
},
"total_tokens": 18624
},
"request_custom_id": "gcg-object20::v_XHQLkc06XIc____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_XHQLkc06XIc____1",
"estimated_frame_times_s": [
20.555500000000002,
33.2465,
45.9375,
58.6285,
71.3195,
84.01050000000001,
96.70150000000001,
109.39249999999998,
122.08349999999999,
134.7745
],
"new_caption_original": "<p> She </p> [SEG:0] begins to play the <p> violin </p> [SEG:1] in a joyful and professional manner"
},
{
"sample_id": "v_1dM62Xpm9Ns____3",
"video_id": "v_1dM62Xpm9Ns",
"segment_id": "3",
"selection_index": 6,
"caption": "In the video, a man is seen speaking to the camera while holding a piece of food. He then proceeds to cut his onion and continues speaking.",
"caption_sha256": "d5225d538da0ccc2f2775d9b292edc1e808beaced127de672623a3c040c05468",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:0] is seen speaking to the camera while holding a piece of food. He then proceeds to cut <p> his </p> [SEG:0] onion and continues speaking.",
"reference_frame_id": 0,
"source_timestamps": [
156.27,
187.15
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_1dM62Xpm9Ns____3.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"he",
"he"
],
"reference_frame_id": 0,
"source_bbox": [
193,
21,
512,
402
],
"source_mask_member": "activitynet_entities_gcg/masks/v_1dM62Xpm9Ns____3/00/mask.png",
"mask_path": "samples/v_1dM62Xpm9Ns____3/m0_mask.png",
"source_mask_sha256": "76bbee92a3c132b2bf22016dd548e6a5a793b1b9eac88e8673595a33f35e34d3",
"mask_foreground_pixels": 74939,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Man speaking and preparing the onion",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
82,
84
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
106,
109
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reference-frame mask clearly covers the foreground man.",
"char_spans": [
[
14,
19
],
[
82,
84
],
[
106,
109
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 0,
"mask_path": "samples/v_1dM62Xpm9Ns____3/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_1dM62Xpm9Ns____3/00/mask.png",
"source_mask_sha256": "76bbee92a3c132b2bf22016dd548e6a5a793b1b9eac88e8673595a33f35e34d3"
}
},
{
"group_id": "g2",
"entity_label": "Camera addressed by the man",
"entity_kind": "thing",
"mentions": [
{
"text": "the camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
40,
50
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The physical camera is mentioned but is not visible in any supplied frame; no camera mask is supplied.",
"char_spans": [
[
40,
50
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "Onion referred to as a piece of food",
"entity_kind": "thing",
"mentions": [
{
"text": "a piece of food",
"occurrence": 1,
"model_occurrence": 1,
"span": [
65,
80
]
},
{
"text": "his onion",
"occurrence": 1,
"model_occurrence": 1,
"span": [
106,
115
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "context_only",
"confidence": "high",
"reason": "The food is subsequently identified as the onion, visible during preparation in context samples 01–04 but not in the reference frame. No onion mask is supplied.",
"char_spans": [
[
65,
80
],
[
106,
115
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The camera is unobserved and has no supplied mask.",
"The onion is visible only in context frames and has no reference-frame mask."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g3"
],
"spans": [
[
106,
109
],
[
106,
115
]
]
}
],
"unresolved_groups": 2,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_051bf50de9612189016ae4c5f5fcd087d1846949528a0125ef",
"api_usage": {
"input_tokens": 15330,
"input_tokens_details": {
"cache_write_tokens": 5827,
"cached_tokens": 9500
},
"output_tokens": 696,
"output_tokens_details": {
"reasoning_tokens": 124
},
"total_tokens": 16026
},
"request_custom_id": "gcg-object20::v_1dM62Xpm9Ns____3",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_1dM62Xpm9Ns____3",
"estimated_frame_times_s": [
157.81400000000002,
160.90200000000002,
163.99,
167.078,
170.166,
173.25400000000002,
176.342,
179.43,
182.518,
185.606
],
"new_caption_original": "<p> He </p> [SEG:0] then continues cutting <p> his </p> [SEG:0] onion"
},
{
"sample_id": "v_t6vWMTMooDc____6",
"video_id": "v_t6vWMTMooDc",
"segment_id": "6",
"selection_index": 7,
"caption": "Now begins round two and the girl cheats to win. In the video, a man is seen sitting on a bed, and a woman is sitting on the floor next to him. The man is wearing a ghost costume, and the woman is wearing a witch costume. They are both laughing and having a good time.",
"caption_sha256": "d968be745eb8c1168c1a3972f553597181e10ac0b89df698a73a175161cd604a",
"caption_source": "refined_caption",
"refined_caption_original": "Now begins round two and <p> the girl </p> [SEG:0] cheats to win. In the video, <p> a man </p> is seen sitting on a bed, and a woman is sitting on the floor next to him. The man is wearing a ghost costume, and the woman is wearing a witch costume. They are both laughing and having a good time.",
"reference_frame_id": 3,
"source_timestamps": [
192.2,
222.19
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_t6vWMTMooDc____6.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"girl"
],
"reference_frame_id": 3,
"source_bbox": [
49,
68,
343,
360
],
"source_mask_member": "activitynet_entities_gcg/masks/v_t6vWMTMooDc____6/00/mask.png",
"mask_path": "samples/v_t6vWMTMooDc____6/m0_mask.png",
"source_mask_sha256": "b6d1ebe3623f54b490cc24a3a5904d0d29b83a70048134a205c01f1042b007ef",
"mask_foreground_pixels": 46687,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "woman in the red-belt padded suit",
"entity_kind": "thing",
"mentions": [
{
"text": "the girl",
"occurrence": 1,
"model_occurrence": 1,
"span": [
25,
33
]
},
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
99,
106
]
},
{
"text": "the woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
184,
193
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reference mask covers the seated woman and her padded suit; the girl and woman mentions co-refer.",
"char_spans": [
[
25,
33
],
[
99,
106
],
[
184,
193
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 3,
"mask_path": "samples/v_t6vWMTMooDc____6/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_t6vWMTMooDc____6/00/mask.png",
"source_mask_sha256": "b6d1ebe3623f54b490cc24a3a5904d0d29b83a70048134a205c01f1042b007ef"
}
},
{
"group_id": "g2",
"entity_label": "man in the blue-belt padded suit",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
63,
68
]
},
{
"text": "him",
"occurrence": 1,
"model_occurrence": 1,
"span": [
139,
142
]
},
{
"text": "The man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
144,
151
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "Context identifies the man as the participant bent down in the foreground at the reference frame, largely concealed by his padded suit. No mask is supplied for him.",
"char_spans": [
[
63,
68
],
[
139,
142
],
[
144,
151
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "alleged bed supporting the man",
"entity_kind": "thing",
"mentions": [
{
"text": "a bed",
"occurrence": 1,
"model_occurrence": 1,
"span": [
88,
93
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "No bed supporting this participant is identifiable in the reference or activity context. Beds in later inset videos do not establish this referent.",
"char_spans": [
[
88,
93
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "floor beneath the participants",
"entity_kind": "stuff",
"mentions": [
{
"text": "the floor",
"occurrence": 1,
"model_occurrence": 1,
"span": [
121,
130
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The padded floor surface is visible around the participants, but no floor mask is supplied.",
"char_spans": [
[
121,
130
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "man's costume described as a ghost costume",
"entity_kind": "thing",
"mentions": [
{
"text": "a ghost costume",
"occurrence": 1,
"model_occurrence": 1,
"span": [
163,
178
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The man's bulky padded costume is visible in the foreground, but it appears to be a sumo-style suit rather than a ghost costume. No separate costume mask is supplied.",
"char_spans": [
[
163,
178
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "woman's costume described as a witch costume",
"entity_kind": "thing",
"mentions": [
{
"text": "a witch costume",
"occurrence": 1,
"model_occurrence": 1,
"span": [
205,
220
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The woman's padded sumo-style suit is visible, but does not resemble a witch costume. The person mask m0 is not a separate costume mask.",
"char_spans": [
[
205,
220
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "the man and woman together",
"entity_kind": "collective",
"mentions": [
{
"text": "They",
"occurrence": 1,
"model_occurrence": 1,
"span": [
222,
226
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "They refers jointly to the two participants. Only the woman's mask is supplied, so the pair cannot be grounded to a complete supplied mask.",
"char_spans": [
[
222,
226
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "caption_visual_mismatch",
"text": "a man is seen sitting on a bed",
"description": "The activity frames show participants wrestling in padded suits on a padded floor; no bed supporting the man is identifiable."
},
{
"type": "caption_visual_mismatch",
"text": "a ghost costume",
"description": "The man's visible costume appears to be a padded sumo-style suit, not a ghost costume."
},
{
"type": "caption_visual_mismatch",
"text": "a witch costume",
"description": "The woman's visible costume appears to be a padded sumo-style suit, not a witch costume."
},
{
"type": "uncertain_coreference",
"text": "They",
"description": "Local review: inspect whether the shared plural pronoun should be included in the existing antecedent instance groups under section 3.4, rather than only in a new collective group. No automatic regrouping was performed.",
"origin": "local_review"
}
],
"model_needs_review": true,
"model_review_reasons": [
"The man, floor, individual costumes, and participant pair lack matching supplied masks.",
"The alleged bed is not observed in the relevant activity frames.",
"The costume descriptions conflict with the visible padded suits."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 6,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0bcaa892785edc1f016ae4c5dc6ed087d1abc51d3bf4db26b0",
"api_usage": {
"input_tokens": 15357,
"input_tokens_details": {
"cache_write_tokens": 5854,
"cached_tokens": 9500
},
"output_tokens": 1567,
"output_tokens_details": {
"reasoning_tokens": 288
},
"total_tokens": 16924
},
"request_custom_id": "gcg-object20::v_t6vWMTMooDc____6",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_t6vWMTMooDc____6",
"estimated_frame_times_s": [
193.6995,
196.6985,
199.6975,
202.6965,
205.69549999999998,
208.6945,
211.6935,
214.6925,
217.6915,
220.6905
],
"new_caption_original": "Now begins round two and the outcome is the same as the <p> girl </p> [SEG:0] cheats to win"
},
{
"sample_id": "v_xbA294b3Z5c____1",
"video_id": "v_xbA294b3Z5c",
"segment_id": "1",
"selection_index": 8,
"caption": "In the video, a woman pours liquid out into a container and then demonstrates how to properly put in and take on a contact lens. She is also seen washing her face with liquid soap, rinsing it off with water, applying lotion, and drying it with a towel.",
"caption_sha256": "5c7323b8550986f4d57c6ad9c048fcd2cb834f3a09ce110ae1af2a2784a1243c",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a woman </p> [SEG:1] pours <p> liquid </p> [SEG:0] out into a <p> container </p> [SEG:2] and then demonstrates how to properly put in and take on a contact lens. She is also seen washing her face with liquid soap, rinsing it off with water, applying lotion, and drying it with a towel.",
"reference_frame_id": 6,
"source_timestamps": [
66.36,
157.51
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_xbA294b3Z5c____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"liquid"
],
"reference_frame_id": 6,
"source_bbox": [
284,
2,
491,
265
],
"source_mask_member": "activitynet_entities_gcg/masks/v_xbA294b3Z5c____1/00/mask.png",
"mask_path": "samples/v_xbA294b3Z5c____1/m0_mask.png",
"source_mask_sha256": "bcb361ca321f553ea82fb19298c1a32ffe68474ff9fa75eb27f2d77ebfd84360",
"mask_foreground_pixels": 18646,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"girl"
],
"reference_frame_id": 6,
"source_bbox": [
0,
5,
714,
396
],
"source_mask_member": "activitynet_entities_gcg/masks/v_xbA294b3Z5c____1/01/mask.png",
"mask_path": "samples/v_xbA294b3Z5c____1/m1_mask.png",
"source_mask_sha256": "a66291dfffdc0156b5161006eff75d9df23965649c44572e702756bd43f9a3d9",
"mask_foreground_pixels": 73100,
"color": "#db2777"
},
{
"mask_id": "m2",
"source_seg_token": "[SEG:2]",
"source_classes": [
"container"
],
"reference_frame_id": 6,
"source_bbox": [
332,
266,
544,
326
],
"source_mask_member": "activitynet_entities_gcg/masks/v_xbA294b3Z5c____1/02/mask.png",
"mask_path": "samples/v_xbA294b3Z5c____1/m2_mask.png",
"source_mask_sha256": "fdd7478afed358d56ee527740e8e0be8c7e19e234badb2424e5906076494cdf4",
"mask_foreground_pixels": 1545,
"color": "#16a34a"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "woman demonstrating contact-lens care",
"entity_kind": "thing",
"mentions": [
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
21
]
},
{
"text": "She",
"occurrence": 1,
"model_occurrence": 1,
"span": [
129,
132
]
},
{
"text": "her",
"occurrence": 1,
"model_occurrence": 1,
"span": [
154,
157
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m1"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The woman is clearly visible, but m1 primarily covers the upper black border and background regions rather than her body.",
"char_spans": [
[
14,
21
],
[
129,
132
],
[
154,
157
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "poured contact-lens solution",
"entity_kind": "stuff",
"mentions": [
{
"text": "liquid",
"occurrence": 1,
"model_occurrence": 1,
"span": [
28,
34
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m0"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Liquid is being dispensed toward the lens case, but m0 is a rectangle in the upper black border and does not segment the liquid.",
"char_spans": [
[
28,
34
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "receiving contact-lens case",
"entity_kind": "thing",
"mentions": [
{
"text": "a container",
"occurrence": 1,
"model_occurrence": 1,
"span": [
44,
55
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m2"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The receiving case is visible near the lower center. The supplied m2 consists of a small patch near the bottle nozzle and scattered pixels, not a usable case segmentation.",
"char_spans": [
[
44,
55
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "contact lens",
"entity_kind": "thing",
"mentions": [
{
"text": "a contact lens",
"occurrence": 1,
"model_occurrence": 1,
"span": [
113,
127
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "uncertain",
"confidence": "medium",
"reason": "Context shows eye and finger manipulation consistent with lens handling, but the transparent lens itself cannot be reliably distinguished and is not identifiable in the reference frame.",
"char_spans": [
[
113,
127
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "woman's face",
"entity_kind": "body_part",
"mentions": [
{
"text": "her face",
"occurrence": 1,
"model_occurrence": 1,
"span": [
154,
162
]
},
{
"text": "it",
"occurrence": 4,
"model_occurrence": 4,
"span": [
236,
238
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Her face is visible behind the bottle and hand, but no separate face mask is supplied. The final 'it' refers to the face being dried.",
"char_spans": [
[
154,
162
],
[
236,
238
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "liquid soap",
"entity_kind": "stuff",
"mentions": [
{
"text": "liquid soap",
"occurrence": 1,
"model_occurrence": 1,
"span": [
168,
179
]
},
{
"text": "it",
"occurrence": 2,
"model_occurrence": 2,
"span": [
189,
191
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "medium",
"reason": "No soap is verifiable in the supplied frames. The first 'it' is provisionally interpreted as the soap being rinsed off, though face coreference is also possible.",
"char_spans": [
[
168,
179
],
[
189,
191
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "rinsing water",
"entity_kind": "stuff",
"mentions": [
{
"text": "water",
"occurrence": 1,
"model_occurrence": 1,
"span": [
201,
206
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "Rinsing water is not verifiable in the supplied images; the visible dispensed contact-lens solution is not evidence of face-rinsing water.",
"char_spans": [
[
201,
206
]
],
"mask_evidence": null
},
{
"group_id": "g8",
"entity_label": "lotion",
"entity_kind": "stuff",
"mentions": [
{
"text": "lotion",
"occurrence": 1,
"model_occurrence": 1,
"span": [
217,
223
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "Neither lotion nor its application is verifiable in the supplied frames.",
"char_spans": [
[
217,
223
]
],
"mask_evidence": null
},
{
"group_id": "g9",
"entity_label": "towel",
"entity_kind": "thing",
"mentions": [
{
"text": "a towel",
"occurrence": 1,
"model_occurrence": 1,
"span": [
244,
251
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "No towel is identifiable in the supplied frames.",
"char_spans": [
[
244,
251
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "",
"description": "All three supplied masks have substantial spatial defects: m0 covers a black-border rectangle, m1 largely covers border/background, and m2 does not segment the receiving case."
},
{
"type": "uncertain_coreference",
"text": "rinsing it off with water",
"description": "The pronoun may refer to liquid soap or the face. It is provisionally grouped with soap; manual coreference review is required."
},
{
"type": "caption_visual_mismatch",
"text": "She is also seen washing her face with liquid soap, rinsing it off with water, applying lotion, and drying it with a towel.",
"description": "The sampled images show contact-lens care, not verifiable face washing, lotion application, or towel drying. These caption entities are retained without asserting their visual presence."
}
],
"model_needs_review": true,
"model_review_reasons": [
"All supplied masks appear spatially defective and require inspection or replacement.",
"The visible face has no separate supplied mask.",
"The contact lens cannot be reliably resolved.",
"Soap, rinsing water, lotion, and towel are not observed in the supplied frames.",
"The first 'it' has uncertain coreference.",
"These unresolved groups are incomplete spatial annotations, not completed one-to-one training correspondences."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g5"
],
"spans": [
[
154,
157
],
[
154,
162
]
]
}
],
"unresolved_groups": 9,
"unused_source_mask_ids": [
"m0",
"m1",
"m2"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0867fad21dad8ff2016ae4c5f6c8a887d1b189c04677a58ba7",
"api_usage": {
"input_tokens": 17802,
"input_tokens_details": {
"cache_write_tokens": 8299,
"cached_tokens": 9500
},
"output_tokens": 2054,
"output_tokens_details": {
"reasoning_tokens": 450
},
"total_tokens": 19856
},
"request_custom_id": "gcg-object20::v_xbA294b3Z5c____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_xbA294b3Z5c____1",
"estimated_frame_times_s": [
70.9175,
80.0325,
89.1475,
98.26249999999999,
107.3775,
116.49249999999999,
125.60749999999999,
134.72249999999997,
143.83749999999998,
152.9525
],
"new_caption_original": "The <p> girl </p> [SEG:1] then pours <p> liquid </p> [SEG:0] out into a <p> container </p> [SEG:2] and them demonstrates how to properly put in and take on a contact lens"
},
{
"sample_id": "v_dVCAykynRzs____0",
"video_id": "v_dVCAykynRzs",
"segment_id": "0",
"selection_index": 9,
"caption": "A camera pans around a McDonald's, leading into a man speaking to a cashier. The man is seen walking into a building and then walking to a counter where he orders a coffee. He then sits down and begins to eat.",
"caption_sha256": "2de6b13705922e4fb6e8041ec499744e569f00766fbb2a87dcc10dab6970b3f9",
"caption_source": "refined_caption",
"refined_caption_original": "A camera pans around a McDonald's, leading into <p> a man </p> [SEG:0] speaking to a cashier. The man is seen walking into a building and then walking to a counter where he orders a coffee. He then sits down and begins to eat.",
"reference_frame_id": 6,
"source_timestamps": [
0.35,
19.03
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_dVCAykynRzs____0.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"man"
],
"reference_frame_id": 6,
"source_bbox": [
469,
104,
658,
390
],
"source_mask_member": "activitynet_entities_gcg/masks/v_dVCAykynRzs____0/00/mask.png",
"mask_path": "samples/v_dVCAykynRzs____0/m0_mask.png",
"source_mask_sha256": "a2777f2ff684dd05ed0d9601511e134496205687adc77ea34590ca3f4852fd20",
"mask_foreground_pixels": 16101,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "filming camera",
"entity_kind": "thing",
"mentions": [
{
"text": "A camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
0,
8
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The caption names a physical camera, but no camera is visible in the supplied frames.",
"char_spans": [
[
0,
8
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "McDonald's restaurant building",
"entity_kind": "thing",
"mentions": [
{
"text": "a McDonald's",
"occurrence": 1,
"model_occurrence": 1,
"span": [
21,
33
]
},
{
"text": "a building",
"occurrence": 1,
"model_occurrence": 1,
"span": [
106,
116
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The exterior appears in sample 00 and the restaurant interior is visible in the reference frame. Both mentions refer to the same restaurant building; no building mask is supplied.",
"char_spans": [
[
21,
33
],
[
106,
116
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "male customer at the counter",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
48,
53
]
},
{
"text": "The man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
77,
84
]
},
{
"text": "he",
"occurrence": 4,
"model_occurrence": 1,
"span": [
153,
155
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
173,
175
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m0"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The customer is clearly visible on the right, but m0 covers window/background pixels around his silhouette rather than his body. It is not a valid person correspondence.",
"char_spans": [
[
48,
53
],
[
77,
84
],
[
153,
155
],
[
173,
175
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "cashier facing the customer",
"entity_kind": "thing",
"mentions": [
{
"text": "a cashier",
"occurrence": 1,
"model_occurrence": 1,
"span": [
66,
75
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The cashier in a light shirt is visible on the left side of the counter; no matching mask is supplied.",
"char_spans": [
[
66,
75
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "service counter",
"entity_kind": "thing",
"mentions": [
{
"text": "a counter",
"occurrence": 1,
"model_occurrence": 1,
"span": [
137,
146
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The service counter is visible between the customer and cashier, but has no supplied mask.",
"char_spans": [
[
137,
146
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "ordered coffee",
"entity_kind": "stuff",
"mentions": [
{
"text": "a coffee",
"occurrence": 1,
"model_occurrence": 1,
"span": [
163,
171
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "medium",
"reason": "No coffee beverage can be verified in the reference frame or other supplied samples. No container is explicitly mentioned.",
"char_spans": [
[
163,
171
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "a man",
"description": "The sole supplied mask m0 is labeled as a man, but its binary region primarily selects the window/background around the customer and excludes his silhouette."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The supplied m0 mask does not correctly segment the male customer.",
"The restaurant building, cashier, and counter lack matching masks.",
"The camera and coffee are not observed in the supplied frames.",
"These unresolved spatial annotations are not complete one-to-one training correspondences."
],
"local_adjustments": [
{
"type": "unique_whole_token_occurrence_resolution",
"group_id": "g3",
"text": "he",
"model_occurrence": 1,
"resolved_occurrence": 4,
"candidate_count": 1
}
],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 6,
"unused_source_mask_ids": [
"m0"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0362dfa4357d9d0a016ae4c5d84d5887d1859f0307f101324c",
"api_usage": {
"input_tokens": 17164,
"input_tokens_details": {
"cache_write_tokens": 17161,
"cached_tokens": 0
},
"output_tokens": 1224,
"output_tokens_details": {
"reasoning_tokens": 204
},
"total_tokens": 18388
},
"request_custom_id": "gcg-object20::v_dVCAykynRzs____0",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_dVCAykynRzs____0",
"estimated_frame_times_s": [
1.2839999999999998,
3.152,
5.02,
6.887999999999999,
8.756,
10.623999999999999,
12.491999999999999,
14.36,
16.228,
18.096000000000004
],
"new_caption_original": "A camera pans around a mcdonalds and leads into a <p> man </p> [SEG:0] speaking to a cashier"
},
{
"sample_id": "v_iksziMLKsBQ____2",
"video_id": "v_iksziMLKsBQ",
"segment_id": "2",
"selection_index": 10,
"caption": "In the video, photographers stand up at the end of her table , surrounded by multiple balls scattered across the table. The individuals appear engaged in a game of pool, navigating the cues and balls with focus and precision.",
"caption_sha256": "30df97037bd722959063e9a5353b5523886ac88c1b66b37173ae03026431ea42",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> photographers </p> [SEG:0] stand up at the end of <p> her table </p> [SEG:0], surrounded by multiple balls scattered across the table. The individuals appear engaged in a game of pool, navigating the cues and balls with focus and precision.",
"reference_frame_id": 9,
"source_timestamps": [
27.34,
29.7
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_iksziMLKsBQ____2.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"table"
],
"reference_frame_id": 9,
"source_bbox": [
68,
261,
713,
538
],
"source_mask_member": "activitynet_entities_gcg/masks/v_iksziMLKsBQ____2/00/mask.png",
"mask_path": "samples/v_iksziMLKsBQ____2/m0_mask.png",
"source_mask_sha256": "3aafb9e451d62f9baf6835a98156881a2de691ce598cdb94a99376dcec6e3742",
"mask_foreground_pixels": 151569,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "photographers around the table",
"entity_kind": "collective",
"mentions": [
{
"text": "photographers",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
27
]
},
{
"text": "The individuals",
"occurrence": 1,
"model_occurrence": 1,
"span": [
120,
135
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "People holding cameras are visible around the table, but no person masks are supplied. The repeated plural references form one group.",
"char_spans": [
[
14,
27
],
[
120,
135
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "person referred to by her",
"entity_kind": "thing",
"mentions": [
{
"text": "her",
"occurrence": 2,
"model_occurrence": 2,
"span": [
51,
54
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "ambiguous",
"visibility": "uncertain",
"confidence": "low",
"reason": "The possessive has no explicit antecedent, and the images do not establish which person owns the table.",
"char_spans": [
[
51,
54
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "foreground billiards table",
"entity_kind": "thing",
"mentions": [
{
"text": "her table",
"occurrence": 1,
"model_occurrence": 1,
"span": [
51,
60
]
},
{
"text": "the table",
"occurrence": 1,
"model_occurrence": 1,
"span": [
109,
118
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The supplied mask covers the foreground green billiards table bearing the scattered balls.",
"char_spans": [
[
51,
60
],
[
109,
118
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 9,
"mask_path": "samples/v_iksziMLKsBQ____2/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_iksziMLKsBQ____2/00/mask.png",
"source_mask_sha256": "3aafb9e451d62f9baf6835a98156881a2de691ce598cdb94a99376dcec6e3742"
}
},
{
"group_id": "g4",
"entity_label": "balls scattered on the foreground table",
"entity_kind": "collective",
"mentions": [
{
"text": "multiple balls",
"occurrence": 1,
"model_occurrence": 1,
"span": [
77,
91
]
},
{
"text": "balls",
"occurrence": 2,
"model_occurrence": 2,
"span": [
194,
199
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Multiple colored balls are visible, but no ball masks are supplied. The table mask is not a separate ball annotation.",
"char_spans": [
[
77,
91
],
[
194,
199
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "billiards cues",
"entity_kind": "collective",
"mentions": [
{
"text": "the cues",
"occurrence": 1,
"model_occurrence": 1,
"span": [
181,
189
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "medium",
"reason": "No billiards cues can be confidently identified in the supplied frames, and no cue masks are supplied.",
"char_spans": [
[
181,
189
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "uncertain_coreference",
"text": "her",
"description": "The caption supplies no clear antecedent for this possessive; its referent cannot be confidently identified or merged with the photographers."
},
{
"type": "caption_visual_mismatch",
"text": "The individuals appear engaged in a game of pool, navigating the cues and balls",
"description": "The visible individuals appear to be photographing the table rather than handling billiards cues or playing. Cues are not confidently observable."
}
],
"model_needs_review": true,
"model_review_reasons": [
"Visible photographers and balls lack supplied masks.",
"The possessive referent is unresolved.",
"The cues are not verifiable in the supplied images.",
"The caption's description of the individuals playing pool is not visually supported."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g2",
"g3"
],
"spans": [
[
51,
54
],
[
51,
60
]
]
}
],
"unresolved_groups": 4,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_076010cf83fb2af9016ae4c64d588087d1a72c2006c95d58bb",
"api_usage": {
"input_tokens": 17159,
"input_tokens_details": {
"cache_write_tokens": 7656,
"cached_tokens": 9500
},
"output_tokens": 1171,
"output_tokens_details": {
"reasoning_tokens": 251
},
"total_tokens": 18330
},
"request_custom_id": "gcg-object20::v_iksziMLKsBQ____2",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_iksziMLKsBQ____2",
"estimated_frame_times_s": [
27.458,
27.694,
27.93,
28.166,
28.402,
28.637999999999998,
28.874,
29.11,
29.346,
29.582
],
"new_caption_original": "Photographers stand up at the end of her <p> table </p> [SEG:0]"
},
{
"sample_id": "v_MFlYmFZ7xlo____0",
"video_id": "v_MFlYmFZ7xlo",
"segment_id": "0",
"selection_index": 11,
"caption": "In the video, a man is mopping a floor . The setting appears to be indoors, with the man standing in a room with a blurry background. He is in a stationary position, focused on mopping the floor. There are no discernible actions or movements taking place, and the lighting is not very bright, contributing to the overall blurred appearance of the surroundings. The man is the main focus of the video, and there are no other objects or individuals present that can be clearly identified.",
"caption_sha256": "35aadfc3785fcf6463a97bc2a0410a31c39a7d0ebb72f0e0a85a80f3b67b6a01",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:1] is mopping a <p> floor </p> [SEG:0]. The setting appears to be indoors, with the man standing in a room with a blurry background. He is in a stationary position, focused on mopping the floor. There are no discernible actions or movements taking place, and the lighting is not very bright, contributing to the overall blurred appearance of the surroundings. The man is the main focus of the video, and there are no other objects or individuals present that can be clearly identified.",
"reference_frame_id": 8,
"source_timestamps": [
0.0,
34.78
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_MFlYmFZ7xlo____0.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"floor"
],
"reference_frame_id": 8,
"source_bbox": [
433,
0,
716,
534
],
"source_mask_member": "activitynet_entities_gcg/masks/v_MFlYmFZ7xlo____0/00/mask.png",
"mask_path": "samples/v_MFlYmFZ7xlo____0/m0_mask.png",
"source_mask_sha256": "f4e04922dbe60467256207b6ff069e68ec90765823255c5ba4325e06dff7090c",
"mask_foreground_pixels": 102219,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"man"
],
"reference_frame_id": 8,
"source_bbox": [
198,
77,
562,
402
],
"source_mask_member": "activitynet_entities_gcg/masks/v_MFlYmFZ7xlo____0/01/mask.png",
"mask_path": "samples/v_MFlYmFZ7xlo____0/m1_mask.png",
"source_mask_sha256": "61a1f9442eaf0b8fb1393112317f9f08215c7539665da002af5c91c92b4b5c3b",
"mask_foreground_pixels": 62413,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "man in a striped shirt",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "the man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
81,
88
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
134,
136
]
},
{
"text": "The man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
361,
368
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The striped-shirt man is visible in the reference frame and corresponds to m1; all four mentions refer to him.",
"char_spans": [
[
14,
19
],
[
81,
88
],
[
134,
136
],
[
361,
368
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 8,
"mask_path": "samples/v_MFlYmFZ7xlo____0/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_MFlYmFZ7xlo____0/01/mask.png",
"source_mask_sha256": "61a1f9442eaf0b8fb1393112317f9f08215c7539665da002af5c91c92b4b5c3b"
}
},
{
"group_id": "g2",
"entity_label": "room floor",
"entity_kind": "stuff",
"mentions": [
{
"text": "a floor",
"occurrence": 1,
"model_occurrence": 1,
"span": [
31,
38
]
},
{
"text": "the floor",
"occurrence": 1,
"model_occurrence": 1,
"span": [
185,
194
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The floor surface on the right of the rotated reference image corresponds to m0.",
"char_spans": [
[
31,
38
],
[
185,
194
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 8,
"mask_path": "samples/v_MFlYmFZ7xlo____0/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_MFlYmFZ7xlo____0/00/mask.png",
"source_mask_sha256": "f4e04922dbe60467256207b6ff069e68ec90765823255c5ba4325e06dff7090c"
}
},
{
"group_id": "g3",
"entity_label": "indoor room",
"entity_kind": "thing",
"mentions": [
{
"text": "a room",
"occurrence": 1,
"model_occurrence": 1,
"span": [
101,
107
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The room interior is visible, but no room mask is supplied; the floor mask alone does not represent the room.",
"char_spans": [
[
101,
107
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "caption_visual_mismatch",
"text": "There are no discernible actions or movements taking place",
"description": "The sampled frames show clear changes in the man's posture and position, contradicting the assertion that no movement is discernible."
},
{
"type": "caption_visual_mismatch",
"text": "there are no other objects or individuals present that can be clearly identified",
"description": "Other objects, including a front-loading washing machine, are identifiable in the supplied frames. The negative noun phrases are not annotated."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The room entity has no supplied mask.",
"The caption contains visual inconsistencies concerning movement and identifiable objects."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 1,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_07446f3f899cfb0d016ae4c63df89087d1adfe74a0bed97216",
"api_usage": {
"input_tokens": 17544,
"input_tokens_details": {
"cache_write_tokens": 8041,
"cached_tokens": 9500
},
"output_tokens": 898,
"output_tokens_details": {
"reasoning_tokens": 136
},
"total_tokens": 18442
},
"request_custom_id": "gcg-object20::v_MFlYmFZ7xlo____0",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_MFlYmFZ7xlo____0",
"estimated_frame_times_s": [
1.739,
5.2170000000000005,
8.695,
12.173,
15.651,
19.129,
22.607,
26.085,
29.563,
33.041000000000004
],
"new_caption_original": "A <p> man </p> [SEG:1] is mopping a <p> floor </p> [SEG:0]"
},
{
"sample_id": "v_m3AbqJNPn-o____1",
"video_id": "v_m3AbqJNPn-o",
"segment_id": "1",
"selection_index": 12,
"caption": "Once it is marked, a woman begins smiling at the camera. She is seen putting on glasses and smiling, adjusting or examining them with a sense of contentment or satisfaction. The setting is indoors, brightly lit, indicating a casual and comfortable environment.",
"caption_sha256": "52f0a67213bf3392b746ba84ee02c12b5b4f3a1de02dc72899c1166052a66fd6",
"caption_source": "refined_caption",
"refined_caption_original": "Once it is marked, <p> a woman </p> [SEG:0] begins smiling at the camera. She is seen putting on glasses and smiling, adjusting or examining them with a sense of contentment or satisfaction. The setting is indoors, brightly lit, indicating a casual and comfortable environment.",
"reference_frame_id": 2,
"source_timestamps": [
12.89,
45.47
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_m3AbqJNPn-o____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"she"
],
"reference_frame_id": 2,
"source_bbox": [
3,
0,
513,
403
],
"source_mask_member": "activitynet_entities_gcg/masks/v_m3AbqJNPn-o____1/00/mask.png",
"mask_path": "samples/v_m3AbqJNPn-o____1/m0_mask.png",
"source_mask_sha256": "15ae85dccf4726344d486a3d3e3012bbd989064483f8421577fc3cb58c2faed0",
"mask_foreground_pixels": 112323,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "unspecified referent of it",
"entity_kind": "thing",
"mentions": [
{
"text": "it",
"occurrence": 1,
"model_occurrence": 1,
"span": [
5,
7
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "ambiguous",
"visibility": "uncertain",
"confidence": "low",
"reason": "The opening pronoun has no supplied antecedent; its referent and concrete-entity eligibility cannot be established.",
"char_spans": [
[
5,
7
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "smiling woman",
"entity_kind": "thing",
"mentions": [
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
19,
26
]
},
{
"text": "She",
"occurrence": 1,
"model_occurrence": 1,
"span": [
57,
60
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reference image and mask identify the same smiling woman.",
"char_spans": [
[
19,
26
],
[
57,
60
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 2,
"mask_path": "samples/v_m3AbqJNPn-o____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_m3AbqJNPn-o____1/00/mask.png",
"source_mask_sha256": "15ae85dccf4726344d486a3d3e3012bbd989064483f8421577fc3cb58c2faed0"
}
},
{
"group_id": "g3",
"entity_label": "camera",
"entity_kind": "thing",
"mentions": [
{
"text": "the camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
45,
55
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The referenced camera is not shown in any supplied frame and has no supplied mask.",
"char_spans": [
[
45,
55
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "eyeglasses",
"entity_kind": "thing",
"mentions": [
{
"text": "glasses",
"occurrence": 1,
"model_occurrence": 1,
"span": [
80,
87
]
},
{
"text": "them",
"occurrence": 1,
"model_occurrence": 1,
"span": [
124,
128
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "context_only",
"confidence": "high",
"reason": "The eyeglasses are visible in sample 00 but not in the reference frame; no glasses mask is supplied.",
"char_spans": [
[
80,
87
],
[
124,
128
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "uncertain_coreference",
"text": "it",
"description": "In 'Once it is marked', the pronoun lacks an identifiable antecedent. Manual review is needed to determine whether it denotes a concrete entity."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The opening pronoun has unresolved reference and entity eligibility.",
"The camera is unobserved and has no mask.",
"The glasses appear only in a context frame and have no reference-frame mask."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 3,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_087aae31cf01bf60016ae4c635eec487d1a536e0e790f0e216",
"api_usage": {
"input_tokens": 15344,
"input_tokens_details": {
"cache_write_tokens": 5841,
"cached_tokens": 9500
},
"output_tokens": 890,
"output_tokens_details": {
"reasoning_tokens": 181
},
"total_tokens": 16234
},
"request_custom_id": "gcg-object20::v_m3AbqJNPn-o____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_m3AbqJNPn-o____1",
"estimated_frame_times_s": [
14.519,
17.777,
21.035,
24.293,
27.551,
30.809,
34.067,
37.325,
40.583,
43.841
],
"new_caption_original": "Once it is marked , <p> she </p> [SEG:0] begins smiling at the camera"
},
{
"sample_id": "v_Mmdcsw_SEzc____1",
"video_id": "v_Mmdcsw_SEzc",
"segment_id": "1",
"selection_index": 13,
"caption": "In the video, a boy forms huge circles while standing on a sandy beach near the water's edge. He appears to be playing or exploring the area. The boy is wearing shorts and seems to be enjoying his time at the beach.",
"caption_sha256": "eb7c09e6ba9afd0f715aa413d025c0a94729a3a024e7a67f9a90c0ad99de4dbd",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a boy </p> [SEG:1] forms huge <p> circles </p> [SEG:0] while standing on a sandy beach near the water's edge. He appears to be playing or exploring the area. The boy is wearing shorts and seems to be enjoying his time at the beach.",
"reference_frame_id": 9,
"source_timestamps": [
22.58,
41.42
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_Mmdcsw_SEzc____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"circle"
],
"reference_frame_id": 9,
"source_bbox": [
264,
457,
596,
538
],
"source_mask_member": "activitynet_entities_gcg/masks/v_Mmdcsw_SEzc____1/00/mask.png",
"mask_path": "samples/v_Mmdcsw_SEzc____1/m0_mask.png",
"source_mask_sha256": "24885490a97ac2d5e1fbaf6c18ab3efc6e48eedc5ed1a962122a3219b3bbef28",
"mask_foreground_pixels": 18875,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"he"
],
"reference_frame_id": 9,
"source_bbox": [
294,
162,
510,
516
],
"source_mask_member": "activitynet_entities_gcg/masks/v_Mmdcsw_SEzc____1/01/mask.png",
"mask_path": "samples/v_Mmdcsw_SEzc____1/m1_mask.png",
"source_mask_sha256": "2d4c09391d178ca239585b7d5e9088b989ffcd5c393f3542b4f4ece15247d381",
"mask_foreground_pixels": 33087,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "boy playing in the sand",
"entity_kind": "thing",
"mentions": [
{
"text": "a boy",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
94,
96
]
},
{
"text": "The boy",
"occurrence": 1,
"model_occurrence": 1,
"span": [
142,
149
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
193,
196
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The person mask depicts the boy wearing red-and-blue shorts; all person references co-refer to him.",
"char_spans": [
[
14,
19
],
[
94,
96
],
[
142,
149
],
[
193,
196
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 9,
"mask_path": "samples/v_Mmdcsw_SEzc____1/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_Mmdcsw_SEzc____1/01/mask.png",
"source_mask_sha256": "2d4c09391d178ca239585b7d5e9088b989ffcd5c393f3542b4f4ece15247d381"
}
},
{
"group_id": "g2",
"entity_label": "circular sand formations",
"entity_kind": "collective",
"mentions": [
{
"text": "huge circles",
"occurrence": 1,
"model_occurrence": 1,
"span": [
26,
38
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The mask covers the raised, roughly circular sand formation around the boy's feet, which he shapes in the context frames. The unmarked plural remains one text group.",
"char_spans": [
[
26,
38
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 9,
"mask_path": "samples/v_Mmdcsw_SEzc____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_Mmdcsw_SEzc____1/00/mask.png",
"source_mask_sha256": "24885490a97ac2d5e1fbaf6c18ab3efc6e48eedc5ed1a962122a3219b3bbef28"
}
},
{
"group_id": "g3",
"entity_label": "sandy beach",
"entity_kind": "stuff",
"mentions": [
{
"text": "a sandy beach",
"occurrence": 1,
"model_occurrence": 1,
"span": [
57,
70
]
},
{
"text": "the beach",
"occurrence": 1,
"model_occurrence": 1,
"span": [
205,
214
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The beach is visible, but no beach mask is supplied. The localized sand-formation mask does not represent the whole beach.",
"char_spans": [
[
57,
70
],
[
205,
214
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "seawater",
"entity_kind": "stuff",
"mentions": [
{
"text": "the water's",
"occurrence": 1,
"model_occurrence": 1,
"span": [
76,
87
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The possessive noun refers to the visible seawater. No water mask is supplied; the spatial term 'edge' is excluded.",
"char_spans": [
[
76,
87
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "boy's red-and-blue shorts",
"entity_kind": "thing",
"mentions": [
{
"text": "shorts",
"occurrence": 1,
"model_occurrence": 1,
"span": [
161,
167
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The shorts are visible, but no separate clothing mask is supplied. The boy's mask is not a shorts mask.",
"char_spans": [
[
161,
167
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The beach, water, and shorts groups lack supplied masks and remain incomplete spatial annotations."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [
"g2:matched_but_not_high_confidence"
],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 3,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0b7b1d69a3e79919016ae4c5d9b60c87d1bce8d0430384b2c4",
"api_usage": {
"input_tokens": 16872,
"input_tokens_details": {
"cache_write_tokens": 7369,
"cached_tokens": 9500
},
"output_tokens": 1037,
"output_tokens_details": {
"reasoning_tokens": 225
},
"total_tokens": 17909
},
"request_custom_id": "gcg-object20::v_Mmdcsw_SEzc____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_Mmdcsw_SEzc____1",
"estimated_frame_times_s": [
23.522,
25.406,
27.29,
29.174,
31.058,
32.942,
34.826,
36.71,
38.594,
40.478
],
"new_caption_original": "<p> He </p> [SEG:1] forms huge <p> circles </p> [SEG:0] and stands up"
},
{
"sample_id": "v_BxzOUJbq28s____1",
"video_id": "v_BxzOUJbq28s",
"segment_id": "1",
"selection_index": 14,
"caption": "In the video, a man is playing the drums with his hands, focusing on incorporating a second drum into the mix. He is wearing headphones and is immersed in his performance.",
"caption_sha256": "342c84db9b72fd4190f0f8c71293f49a6bc4a1d40aa841d318f5ddd90c675f37",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:0] is playing the drums with his hands, focusing on incorporating a second drum into the mix. He is wearing headphones and is immersed in his performance.",
"reference_frame_id": 4,
"source_timestamps": [
49.59,
181.46
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_BxzOUJbq28s____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"man"
],
"reference_frame_id": 4,
"source_bbox": [
181,
22,
555,
537
],
"source_mask_member": "activitynet_entities_gcg/masks/v_BxzOUJbq28s____1/00/mask.png",
"mask_path": "samples/v_BxzOUJbq28s____1/m0_mask.png",
"source_mask_sha256": "33d62c3b62f82cdc78b9e7b5fc4c5e3826955d450df3be04e52ba04cae6e8bc6",
"mask_foreground_pixels": 99883,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "man playing drums",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
46,
49
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
111,
113
]
},
{
"text": "his",
"occurrence": 2,
"model_occurrence": 2,
"span": [
155,
158
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The supplied mask depicts the seated man playing the drums; both possessives and the pronoun refer to him.",
"char_spans": [
[
14,
19
],
[
46,
49
],
[
111,
113
],
[
155,
158
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 4,
"mask_path": "samples/v_BxzOUJbq28s____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_BxzOUJbq28s____1/00/mask.png",
"source_mask_sha256": "33d62c3b62f82cdc78b9e7b5fc4c5e3826955d450df3be04e52ba04cae6e8bc6"
}
},
{
"group_id": "g2",
"entity_label": "drums",
"entity_kind": "collective",
"mentions": [
{
"text": "the drums",
"occurrence": 1,
"model_occurrence": 1,
"span": [
31,
40
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Multiple drums are visible, but no drum masks are supplied. The plural mention remains one group.",
"char_spans": [
[
31,
40
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "man's hands",
"entity_kind": "body_part",
"mentions": [
{
"text": "his hands",
"occurrence": 1,
"model_occurrence": 1,
"span": [
46,
55
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Both hands are visible, but no separate hand masks are supplied; the person mask is not a hand mask.",
"char_spans": [
[
46,
55
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "second drum incorporated into playing",
"entity_kind": "thing",
"mentions": [
{
"text": "a second drum",
"occurrence": 1,
"model_occurrence": 1,
"span": [
83,
96
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The large pale-headed foreground drum is visible in the reference frame and is played alongside the central drum in context sample 05. No drum mask is supplied.",
"char_spans": [
[
83,
96
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "headphones worn by the man",
"entity_kind": "thing",
"mentions": [
{
"text": "headphones",
"occurrence": 1,
"model_occurrence": 1,
"span": [
125,
135
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Headphones are visible on the man's head, but no separate headphone mask is supplied.",
"char_spans": [
[
125,
135
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The drums, hands, second drum, and headphones lack supplied masks; these groups remain incomplete spatial annotations."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g3"
],
"spans": [
[
46,
49
],
[
46,
55
]
]
}
],
"unresolved_groups": 4,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_06454b48beab2077016ae4c5d8741c87d19527358f872d3df6",
"api_usage": {
"input_tokens": 17155,
"input_tokens_details": {
"cache_write_tokens": 7652,
"cached_tokens": 9500
},
"output_tokens": 908,
"output_tokens_details": {
"reasoning_tokens": 140
},
"total_tokens": 18063
},
"request_custom_id": "gcg-object20::v_BxzOUJbq28s____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_BxzOUJbq28s____1",
"estimated_frame_times_s": [
56.1835,
69.3705,
82.5575,
95.7445,
108.9315,
122.11850000000001,
135.3055,
148.4925,
161.67950000000002,
174.86650000000003
],
"new_caption_original": "The <p> man </p> [SEG:0] starts adding in a second drum to the mix"
},
{
"sample_id": "v_RrgUFU70PNQ____1",
"video_id": "v_RrgUFU70PNQ",
"segment_id": "1",
"selection_index": 15,
"caption": "In the video, two men are engaged in a martial arts routine, dressed in white, crouching in front of each other before springing onto the mat , fighting and kicking around each other. The performance captivates a group of onlookers, as the men demonstrate skillful and disciplined movements.",
"caption_sha256": "426315144846f16b4030183fb519310a30eb507f390c2452bde386765a0395dc",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, two men are engaged in a <p> martial arts </p> [SEG:0] routine, dressed in white, <p> crouching </p> in front of each other before springing onto the <p> mat </p> [SEG:0] , <p> fighting </p> and <p> kicking </p> around each other. The performance captivates a group of onlookers, as the men demonstrate skillful and disciplined movements.",
"reference_frame_id": 0,
"source_timestamps": [
11.34,
82.36
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_RrgUFU70PNQ____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"mat"
],
"reference_frame_id": 0,
"source_bbox": [
0,
186,
719,
404
],
"source_mask_member": "activitynet_entities_gcg/masks/v_RrgUFU70PNQ____1/00/mask.png",
"mask_path": "samples/v_RrgUFU70PNQ____1/m0_mask.png",
"source_mask_sha256": "7735991f7b5bedfc9fe858093a334e6d0f90faba42d0e8711556085d7eefb732",
"mask_foreground_pixels": 148891,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Two martial arts performers",
"entity_kind": "collective",
"mentions": [
{
"text": "two men",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
21
]
},
{
"text": "each other",
"occurrence": 1,
"model_occurrence": 1,
"span": [
101,
111
]
},
{
"text": "each other",
"occurrence": 2,
"model_occurrence": 2,
"span": [
172,
182
]
},
{
"text": "the men",
"occurrence": 1,
"model_occurrence": 1,
"span": [
236,
243
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Both performers are visible in the reference frame, but no person masks are supplied. The count without instance selectors remains one group; both reciprocal mentions refer to this pair.",
"char_spans": [
[
14,
21
],
[
101,
111
],
[
172,
182
],
[
236,
243
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "Martial arts mat",
"entity_kind": "thing",
"mentions": [
{
"text": "the mat",
"occurrence": 1,
"model_occurrence": 1,
"span": [
134,
141
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The overlay and binary mask identify the large mat beneath the performers.",
"char_spans": [
[
134,
141
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 0,
"mask_path": "samples/v_RrgUFU70PNQ____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_RrgUFU70PNQ____1/00/mask.png",
"source_mask_sha256": "7735991f7b5bedfc9fe858093a334e6d0f90faba42d0e8711556085d7eefb732"
}
},
{
"group_id": "g3",
"entity_label": "Group of onlookers surrounding the mat",
"entity_kind": "collective",
"mentions": [
{
"text": "a group of onlookers",
"occurrence": 1,
"model_occurrence": 1,
"span": [
211,
231
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The seated and standing audience is visible in the reference frame, but no audience masks are supplied. The counter-and-content phrase is retained as one span.",
"char_spans": [
[
211,
231
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The performers and onlookers are visible but lack supplied masks, leaving their spatial annotations incomplete."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 2,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_074a5ace3f94dd6d016ae4c5e30c5487d1b7b17339d2478443",
"api_usage": {
"input_tokens": 15352,
"input_tokens_details": {
"cache_write_tokens": 5849,
"cached_tokens": 9500
},
"output_tokens": 666,
"output_tokens_details": {
"reasoning_tokens": 65
},
"total_tokens": 16018
},
"request_custom_id": "gcg-object20::v_RrgUFU70PNQ____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_RrgUFU70PNQ____1",
"estimated_frame_times_s": [
14.891,
21.993000000000002,
29.095,
36.197,
43.29899999999999,
50.400999999999996,
57.503,
64.605,
71.707,
78.809
],
"new_caption_original": "Two crouch in front of each other before springing onto the <p> mat </p> [SEG:0] , fighting and kicking"
},
{
"sample_id": "v_OFn-c_XvIhU____4",
"video_id": "v_OFn-c_XvIhU",
"segment_id": "4",
"selection_index": 16,
"caption": "In the video, one of the girls who was on the trip gives the trip a thumbs up. She is standing outside, smiling and waving at the camera, enjoying the moment. The setting features some buildings, trees, and a clear sky. She is wearing casual clothing, a white shirt, and has black hair.",
"caption_sha256": "3f76a18d3f70c97bb86218b8ffd32ff0c67af6b2620bf10bcfb8de8b10c66624",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> one of the girls </p> [SEG:0] who was on the trip gives the trip a thumbs up. <p> She </p> [SEG:0] is standing outside, smiling and waving at the camera, enjoying the moment. The setting features some buildings, trees, and a clear sky. <p> She </p> [SEG:0] is wearing casual clothing, a white shirt, and has black hair. ",
"reference_frame_id": 1,
"source_timestamps": [
74.05,
74.5
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_OFn-c_XvIhU____4.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"girl"
],
"reference_frame_id": 1,
"source_bbox": [
55,
99,
463,
403
],
"source_mask_member": "activitynet_entities_gcg/masks/v_OFn-c_XvIhU____4/00/mask.png",
"mask_path": "samples/v_OFn-c_XvIhU____4/m0_mask.png",
"source_mask_sha256": "9287639cfde615b859187f0c42ccc156a9fdeb0999124aad3164ec7950d88160",
"mask_foreground_pixels": 73922,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Foreground girl",
"entity_kind": "thing",
"mentions": [
{
"text": "one",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
17
]
},
{
"text": "She",
"occurrence": 1,
"model_occurrence": 1,
"span": [
79,
82
]
},
{
"text": "She",
"occurrence": 2,
"model_occurrence": 2,
"span": [
220,
223
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The selected girl and both subsequent pronouns refer to the foreground person covered by m0.",
"char_spans": [
[
14,
17
],
[
79,
82
],
[
220,
223
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 1,
"mask_path": "samples/v_OFn-c_XvIhU____4/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_OFn-c_XvIhU____4/00/mask.png",
"source_mask_sha256": "9287639cfde615b859187f0c42ccc156a9fdeb0999124aad3164ec7950d88160"
}
},
{
"group_id": "g2",
"entity_label": "Girls on the trip",
"entity_kind": "collective",
"mentions": [
{
"text": "the girls",
"occurrence": 1,
"model_occurrence": 1,
"span": [
21,
30
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "uncertain",
"confidence": "high",
"reason": "The partitive requires a separate plural group. Only the selected member is visible; m0 does not represent the full group of girls.",
"char_spans": [
[
21,
30
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "Camera being addressed",
"entity_kind": "thing",
"mentions": [
{
"text": "the camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
126,
136
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The referenced filming camera is not visible in any supplied frame.",
"char_spans": [
[
126,
136
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "Background buildings",
"entity_kind": "collective",
"mentions": [
{
"text": "some buildings",
"occurrence": 1,
"model_occurrence": 1,
"span": [
180,
194
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Industrial buildings are visible behind the girl, but no building masks are supplied.",
"char_spans": [
[
180,
194
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "Background trees",
"entity_kind": "collective",
"mentions": [
{
"text": "trees",
"occurrence": 1,
"model_occurrence": 1,
"span": [
196,
201
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "Distant tree vegetation is visible at the far left; no tree masks are supplied.",
"char_spans": [
[
196,
201
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "Sky",
"entity_kind": "stuff",
"mentions": [
{
"text": "a clear sky",
"occurrence": 1,
"model_occurrence": 1,
"span": [
207,
218
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Sky is visible above the buildings, with no supplied sky mask.",
"char_spans": [
[
207,
218
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "Girl's white shirt",
"entity_kind": "thing",
"mentions": [
{
"text": "casual clothing",
"occurrence": 1,
"model_occurrence": 1,
"span": [
235,
250
]
},
{
"text": "a white shirt",
"occurrence": 1,
"model_occurrence": 1,
"span": [
252,
265
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The appositive white shirt identifies the casual clothing. It is visible but has no separate mask; the person mask is not a clothing mask.",
"char_spans": [
[
235,
250
],
[
252,
265
]
],
"mask_evidence": null
},
{
"group_id": "g8",
"entity_label": "Girl's black hair",
"entity_kind": "body_part",
"mentions": [
{
"text": "black hair",
"occurrence": 1,
"model_occurrence": 1,
"span": [
275,
285
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The girl's dark hair is visible but has no separate supplied mask.",
"char_spans": [
[
275,
285
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "caption_visual_mismatch",
"text": "a thumbs up",
"description": "The visible raised hand forms an OK sign rather than a thumbs-up. This phrase denotes a gesture, not a separate tangible entity."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The plural girls group cannot be spatially resolved from the single visible member.",
"The camera is not observed in the supplied frames.",
"Buildings, trees, sky, shirt, and hair lack separate source masks.",
"The stated thumbs-up gesture differs from the visible OK sign."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 7,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_07e9ef3ea9454c85016ae4c5db4cb487d1ae8122994ec4950f",
"api_usage": {
"input_tokens": 15358,
"input_tokens_details": {
"cache_write_tokens": 5855,
"cached_tokens": 9500
},
"output_tokens": 1397,
"output_tokens_details": {
"reasoning_tokens": 182
},
"total_tokens": 16755
},
"request_custom_id": "gcg-object20::v_OFn-c_XvIhU____4",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_OFn-c_XvIhU____4",
"estimated_frame_times_s": [
74.07249999999999,
74.11749999999999,
74.1625,
74.2075,
74.2525,
74.2975,
74.3425,
74.3875,
74.4325,
74.4775
],
"new_caption_original": "One of the <p> girls </p> [SEG:0] who was on the trip gives the trip a thumbs up"
},
{
"sample_id": "v_XLspZ5bof7o____1",
"video_id": "v_XLspZ5bof7o",
"segment_id": "1",
"selection_index": 17,
"caption": "In the video, a woman is seen turning around and grabbing a handful of her hair . She is then sitting in front of the camera, talking, brushing her hair, and braiding it.",
"caption_sha256": "224d65678a1962578d7c9836a6c22d3bfc434e345f51363b617e9cbd1c01736e",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a woman </p> [SEG:1] is seen turning around and grabbing a handful of <p> her hair </p> [SEG:0]. She is then sitting in front of the camera, talking, brushing her hair, and braiding it.",
"reference_frame_id": 3,
"source_timestamps": [
15.25,
28.8
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_XLspZ5bof7o____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"hair"
],
"reference_frame_id": 3,
"source_bbox": [
229,
42,
537,
400
],
"source_mask_member": "activitynet_entities_gcg/masks/v_XLspZ5bof7o____1/00/mask.png",
"mask_path": "samples/v_XLspZ5bof7o____1/m0_mask.png",
"source_mask_sha256": "4c485afba3b5ed56bca1e3b5130dc532a66a3519b744e22d5524889199979bac",
"mask_foreground_pixels": 66510,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"she"
],
"reference_frame_id": 3,
"source_bbox": [
99,
41,
661,
400
],
"source_mask_member": "activitynet_entities_gcg/masks/v_XLspZ5bof7o____1/01/mask.png",
"mask_path": "samples/v_XLspZ5bof7o____1/m1_mask.png",
"source_mask_sha256": "574223ea2f9879005f8d82c00e758e08cc961694e844a6ac34e67ab29d150c52",
"mask_foreground_pixels": 120748,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "woman arranging her hair",
"entity_kind": "thing",
"mentions": [
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
21
]
},
{
"text": "her",
"occurrence": 1,
"model_occurrence": 1,
"span": [
71,
74
]
},
{
"text": "She",
"occurrence": 1,
"model_occurrence": 1,
"span": [
82,
85
]
},
{
"text": "her",
"occurrence": 2,
"model_occurrence": 2,
"span": [
144,
147
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The reference-frame person mask covers the woman with raised arms; both possessives and She refer to her.",
"char_spans": [
[
14,
21
],
[
71,
74
],
[
82,
85
],
[
144,
147
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 3,
"mask_path": "samples/v_XLspZ5bof7o____1/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_XLspZ5bof7o____1/01/mask.png",
"source_mask_sha256": "574223ea2f9879005f8d82c00e758e08cc961694e844a6ac34e67ab29d150c52"
}
},
{
"group_id": "g2",
"entity_label": "woman's hair",
"entity_kind": "body_part",
"mentions": [
{
"text": "a handful of her hair",
"occurrence": 1,
"model_occurrence": 1,
"span": [
58,
79
]
},
{
"text": "her hair",
"occurrence": 2,
"model_occurrence": 2,
"span": [
144,
152
]
},
{
"text": "it",
"occurrence": 2,
"model_occurrence": 2,
"span": [
167,
169
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The hair mask identifies her long hair. The counter phrase stays intact, and the final it refers to the hair. Its substring occurrence is 2 because sitting contains the first it.",
"char_spans": [
[
58,
79
],
[
144,
152
],
[
167,
169
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 3,
"mask_path": "samples/v_XLspZ5bof7o____1/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_XLspZ5bof7o____1/00/mask.png",
"source_mask_sha256": "4c485afba3b5ed56bca1e3b5130dc532a66a3519b744e22d5524889199979bac"
}
},
{
"group_id": "g3",
"entity_label": "camera",
"entity_kind": "thing",
"mentions": [
{
"text": "the camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
114,
124
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The caption refers to a camera, but no camera is visible in the reference or context frames, and no camera mask is supplied.",
"char_spans": [
[
114,
124
]
],
"mask_evidence": null
}
],
"issues": [],
"model_needs_review": true,
"model_review_reasons": [
"The camera group has no observable entity or supplied mask and remains spatially unresolved."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g2"
],
"spans": [
[
71,
74
],
[
58,
79
]
]
},
{
"groups": [
"g1",
"g2"
],
"spans": [
[
144,
147
],
[
144,
152
]
]
}
],
"unresolved_groups": 1,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0c5b6ff19fb7b1e5016ae4c5dcf48487d1b09531810df07bfe",
"api_usage": {
"input_tokens": 15660,
"input_tokens_details": {
"cache_write_tokens": 6157,
"cached_tokens": 9500
},
"output_tokens": 851,
"output_tokens_details": {
"reasoning_tokens": 275
},
"total_tokens": 16511
},
"request_custom_id": "gcg-object20::v_XLspZ5bof7o____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_XLspZ5bof7o____1",
"estimated_frame_times_s": [
15.9275,
17.2825,
18.6375,
19.9925,
21.3475,
22.7025,
24.0575,
25.4125,
26.767500000000002,
28.1225
],
"new_caption_original": "<p> She </p> [SEG:1] turns around and grabs a handful of <p> hair </p> [SEG:0]"
},
{
"sample_id": "v_DPw74KWZzGY____1",
"video_id": "v_DPw74KWZzGY",
"segment_id": "1",
"selection_index": 18,
"caption": "In the video, a man is playing his accordion and singing to the camera. He speaks to the camera, ending with the two playing again.",
"caption_sha256": "242b1078ed1d5b90c6ca54b421898df91195580ef65f04616b76991e71fbb32b",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:0] is playing <p> his </p> [SEG:0] <p> accordion </p> [SEG:1] and singing to the camera. <p> He </p> [SEG:0] speaks to the camera, ending with the two playing again.",
"reference_frame_id": 7,
"source_timestamps": [
60.47,
128.66
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_DPw74KWZzGY____1.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"man",
"he"
],
"reference_frame_id": 7,
"source_bbox": [
37,
80,
314,
535
],
"source_mask_member": "activitynet_entities_gcg/masks/v_DPw74KWZzGY____1/00/mask.png",
"mask_path": "samples/v_DPw74KWZzGY____1/m0_mask.png",
"source_mask_sha256": "0ff41b7f345c02043f23d6a1eb2aa7c294f68f71d06a7b0040ff5ae6e1cbd1c1",
"mask_foreground_pixels": 59488,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"instrument"
],
"reference_frame_id": 7,
"source_bbox": [
87,
216,
290,
442
],
"source_mask_member": "activitynet_entities_gcg/masks/v_DPw74KWZzGY____1/01/mask.png",
"mask_path": "samples/v_DPw74KWZzGY____1/m1_mask.png",
"source_mask_sha256": "e4fa19da2ed82bed6b137953dd4d7d7b868df8271169e155be1e0881e7407481",
"mask_foreground_pixels": 21343,
"color": "#db2777"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Man playing the accordion",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "his",
"occurrence": 1,
"model_occurrence": 1,
"span": [
31,
34
]
},
{
"text": "He",
"occurrence": 1,
"model_occurrence": 1,
"span": [
72,
74
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m0"
],
"grounding_status": "mask_quality_issue",
"visibility": "reference_visible",
"confidence": "high",
"reason": "m0 identifies the accordion player, but its binary mask also includes substantial accordion pixels.",
"char_spans": [
[
14,
19
],
[
31,
34
],
[
72,
74
]
],
"model_mask_id": "m0",
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "Man's accordion",
"entity_kind": "thing",
"mentions": [
{
"text": "his accordion",
"occurrence": 1,
"model_occurrence": 1,
"span": [
31,
44
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "m1 corresponds to the accordion held by the man.",
"char_spans": [
[
31,
44
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 7,
"mask_path": "samples/v_DPw74KWZzGY____1/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_DPw74KWZzGY____1/01/mask.png",
"source_mask_sha256": "e4fa19da2ed82bed6b137953dd4d7d7b868df8271169e155be1e0881e7407481"
}
},
{
"group_id": "g3",
"entity_label": "Camera addressed by the man",
"entity_kind": "thing",
"mentions": [
{
"text": "the camera",
"occurrence": 1,
"model_occurrence": 1,
"span": [
60,
70
]
},
{
"text": "the camera",
"occurrence": 2,
"model_occurrence": 2,
"span": [
85,
95
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "not_observed",
"confidence": "high",
"reason": "The physical camera addressed by the man is not shown in any supplied frame; no camera mask is supplied.",
"char_spans": [
[
60,
70
],
[
85,
95
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "Accordion player and guitarist",
"entity_kind": "collective",
"mentions": [
{
"text": "the two",
"occurrence": 1,
"model_occurrence": 1,
"span": [
109,
116
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The two performers are visible together in the reference frame. The count-only reference remains one collective group, but no mask covers both people and no guitarist mask is supplied.",
"char_spans": [
[
109,
116
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "a man",
"description": "The supplied person mask includes substantial pixels belonging to the separately masked accordion; its segmentation needs review."
}
],
"model_needs_review": true,
"model_review_reasons": [
"The person mask has apparent accordion contamination.",
"The camera is not observed and has no supplied mask.",
"The collective reference to both performers lacks a corresponding mask."
],
"local_adjustments": [
{
"type": "unconfirmed_quality_mask_to_candidate",
"group_id": "g1",
"model_mask_id": "m0",
"resolved_mask_id": null,
"reason": "Preserve the proposed ID as a review candidate, not a confirmed correspondence."
}
],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g1",
"g2"
],
"spans": [
[
31,
34
],
[
31,
44
]
]
}
],
"unresolved_groups": 3,
"unused_source_mask_ids": [
"m0"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0638b2e4cd892811016ae4c5e68a4487d1973c556fc78832dc",
"api_usage": {
"input_tokens": 16860,
"input_tokens_details": {
"cache_write_tokens": 7357,
"cached_tokens": 9500
},
"output_tokens": 918,
"output_tokens_details": {
"reasoning_tokens": 219
},
"total_tokens": 17778
},
"request_custom_id": "gcg-object20::v_DPw74KWZzGY____1",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_DPw74KWZzGY____1",
"estimated_frame_times_s": [
63.8795,
70.6985,
77.5175,
84.3365,
91.1555,
97.97449999999999,
104.7935,
111.6125,
118.4315,
125.25049999999999
],
"new_caption_original": "The <p> man </p> [SEG:0] then sings to the camera while playing <p> his </p> [SEG:0] <p> instrument </p> [SEG:1] and speaks to the camera , ending with the two playing again"
},
{
"sample_id": "v_nm2sHNDN7Lo____2",
"video_id": "v_nm2sHNDN7Lo",
"segment_id": "2",
"selection_index": 19,
"caption": "In the video, a man and the two ladies laugh and cheer, and he high fives one lady at a casual outdoor social event. They are gathered around a table with a beer pong setup, engaging in friendly competition, and enjoying drinks from cups placed on the table. The atmosphere is relaxed and convivial, with participants interacting and having a good time together.",
"caption_sha256": "f9c55fec8b994abb82473391bc237948e776949be8fd98e6c0c590d8fc84f807",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, <p> a man </p> [SEG:1] and the two <p> ladies </p> [SEG:2] laugh and cheer, and <p> he </p> [SEG:1] high fives one <p> lady </p> [SEG:0] at a casual outdoor social event. They are gathered around a table with a beer pong setup, engaging in friendly competition, and enjoying drinks from cups placed on the table. The atmosphere is relaxed and convivial, with participants interacting and having a good time together.",
"reference_frame_id": 2,
"source_timestamps": [
5.08,
14.3
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_nm2sHNDN7Lo____2.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"lady",
"lady"
],
"reference_frame_id": 2,
"source_bbox": [
409,
142,
545,
403
],
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/00/mask.png",
"mask_path": "samples/v_nm2sHNDN7Lo____2/m0_mask.png",
"source_mask_sha256": "01656bae5d336f6a9fa75e48834ff00ff794dc156e2213b40211783d38796200",
"mask_foreground_pixels": 24597,
"color": "#2563eb"
},
{
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"source_classes": [
"man",
"he"
],
"reference_frame_id": 2,
"source_bbox": [
267,
114,
362,
404
],
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/01/mask.png",
"mask_path": "samples/v_nm2sHNDN7Lo____2/m1_mask.png",
"source_mask_sha256": "c0ba5f3623d6d1dd69a755855f9c91e7143b186c4e7bb8493cdab432480fdc62",
"mask_foreground_pixels": 14927,
"color": "#db2777"
},
{
"mask_id": "m2",
"source_seg_token": "[SEG:2]",
"source_classes": [
"lady"
],
"reference_frame_id": 2,
"source_bbox": [
589,
132,
718,
404
],
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/02/mask.png",
"mask_path": "samples/v_nm2sHNDN7Lo____2/m2_mask.png",
"source_mask_sha256": "69a4a8837969ab3b42dfdd6258c4ab2e61c4280dd94d6a2309da95eb8a6fa1d1",
"mask_foreground_pixels": 23709,
"color": "#16a34a"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Man wearing glasses and a white T-shirt",
"entity_kind": "thing",
"mentions": [
{
"text": "a man",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
19
]
},
{
"text": "he",
"occurrence": 4,
"model_occurrence": 1,
"span": [
60,
62
]
}
],
"mask_id": "m1",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The man is clearly visible at left; context frames show him high-fiving the blonde woman.",
"char_spans": [
[
14,
19
],
[
60,
62
]
],
"mask_evidence": {
"mask_id": "m1",
"source_seg_token": "[SEG:1]",
"reference_frame_id": 2,
"mask_path": "samples/v_nm2sHNDN7Lo____2/m1_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/01/mask.png",
"source_mask_sha256": "c0ba5f3623d6d1dd69a755855f9c91e7143b186c4e7bb8493cdab432480fdc62"
}
},
{
"group_id": "g2",
"entity_label": "Blonde woman in white who receives the high five",
"entity_kind": "thing",
"mentions": [
{
"text": "the two ladies",
"occurrence": 1,
"model_occurrence": 1,
"span": [
24,
38
]
},
{
"text": "one lady",
"occurrence": 1,
"model_occurrence": 1,
"span": [
74,
82
]
}
],
"mask_id": "m0",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The count phrase and instance-selecting mention identify the blonde woman as one of the two ladies. The high-five interaction is visible in the context frames.",
"char_spans": [
[
24,
38
],
[
74,
82
]
],
"mask_evidence": {
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"reference_frame_id": 2,
"mask_path": "samples/v_nm2sHNDN7Lo____2/m0_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/00/mask.png",
"source_mask_sha256": "01656bae5d336f6a9fa75e48834ff00ff794dc156e2213b40211783d38796200"
}
},
{
"group_id": "g3",
"entity_label": "Dark-haired woman in gray and denim shorts",
"entity_kind": "thing",
"mentions": [
{
"text": "the two ladies",
"occurrence": 1,
"model_occurrence": 1,
"span": [
24,
38
]
}
],
"mask_id": "m2",
"candidate_mask_ids": [],
"grounding_status": "matched",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The second woman is clearly visible at right and shares the counted type anchor with the blonde woman.",
"char_spans": [
[
24,
38
]
],
"mask_evidence": {
"mask_id": "m2",
"source_seg_token": "[SEG:2]",
"reference_frame_id": 2,
"mask_path": "samples/v_nm2sHNDN7Lo____2/m2_mask.png",
"source_mask_member": "activitynet_entities_gcg/masks/v_nm2sHNDN7Lo____2/02/mask.png",
"source_mask_sha256": "69a4a8837969ab3b42dfdd6258c4ab2e61c4280dd94d6a2309da95eb8a6fa1d1"
}
},
{
"group_id": "g4",
"entity_label": "The man and two women participating together",
"entity_kind": "collective",
"mentions": [
{
"text": "They",
"occurrence": 1,
"model_occurrence": 1,
"span": [
117,
121
]
},
{
"text": "participants",
"occurrence": 1,
"model_occurrence": 1,
"span": [
305,
317
]
}
],
"mask_id": null,
"candidate_mask_ids": [
"m0",
"m1",
"m2"
],
"grounding_status": "requires_mask_union",
"visibility": "reference_visible",
"confidence": "high",
"reason": "These plural references describe the three foreground participants together; their supplied masks are separate.",
"char_spans": [
[
117,
121
],
[
305,
317
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "Black beer-pong table",
"entity_kind": "thing",
"mentions": [
{
"text": "a table",
"occurrence": 1,
"model_occurrence": 1,
"span": [
142,
149
]
},
{
"text": "the table",
"occurrence": 1,
"model_occurrence": 1,
"span": [
248,
257
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "A small portion of the tabletop is visible along the lower-right reference-frame boundary, and context frames show it clearly. No table mask is supplied.",
"char_spans": [
[
142,
149
],
[
248,
257
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "Physical beer-pong arrangement",
"entity_kind": "collective",
"mentions": [
{
"text": "a beer pong setup",
"occurrence": 1,
"model_occurrence": 1,
"span": [
155,
172
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The arrangement is largely cropped out at the reference frame, with only a small tabletop and cup-rim portion visible; context frames establish the setup. No corresponding mask is supplied.",
"char_spans": [
[
155,
172
]
],
"mask_evidence": null
},
{
"group_id": "g7",
"entity_label": "Drinks contained in the cups",
"entity_kind": "stuff",
"mentions": [
{
"text": "drinks",
"occurrence": 1,
"model_occurrence": 1,
"span": [
221,
227
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "not_visible_in_reference",
"visibility": "context_only",
"confidence": "high",
"reason": "Amber liquid is visible inside cups in context frames, but is not discernible in the cropped reference view. No liquid mask is supplied.",
"char_spans": [
[
221,
227
]
],
"mask_evidence": null
},
{
"group_id": "g8",
"entity_label": "Plastic cups on the table",
"entity_kind": "collective",
"mentions": [
{
"text": "cups",
"occurrence": 1,
"model_occurrence": 1,
"span": [
233,
237
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "Only a small cup-rim portion appears at the lower reference-frame boundary; context frames clearly show the plural cups. No cup masks are supplied.",
"char_spans": [
[
233,
237
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "uncertain_coreference",
"text": "They",
"description": "Local review: inspect whether the shared plural pronoun should be included in the existing antecedent instance groups under section 3.4, rather than only in a new collective group. No automatic regrouping was performed.",
"origin": "local_review"
}
],
"model_needs_review": true,
"model_review_reasons": [
"The collective participant references require a union of three supplied person masks.",
"The table, beer-pong setup, and cups lack supplied masks and are mostly cropped out of the reference frame.",
"The drinks are visible only in context frames and have no supplied mask."
],
"local_adjustments": [
{
"type": "unique_whole_token_occurrence_resolution",
"group_id": "g1",
"text": "he",
"model_occurrence": 1,
"resolved_occurrence": 4,
"candidate_count": 1
}
],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [
{
"groups": [
"g2",
"g3"
],
"spans": [
[
24,
38
],
[
24,
38
]
]
}
],
"unresolved_groups": 5,
"unused_source_mask_ids": [],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0374579730e9a2f3016ae4c5eade0487d19a8c3a9ca0cb6288",
"api_usage": {
"input_tokens": 15998,
"input_tokens_details": {
"cache_write_tokens": 6495,
"cached_tokens": 9500
},
"output_tokens": 1648,
"output_tokens_details": {
"reasoning_tokens": 299
},
"total_tokens": 17646
},
"request_custom_id": "gcg-object20::v_nm2sHNDN7Lo____2",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_nm2sHNDN7Lo____2",
"estimated_frame_times_s": [
5.541,
6.463,
7.385,
8.307,
9.229,
10.151,
11.073,
11.995000000000001,
12.917000000000002,
13.839
],
"new_caption_original": "The <p> man </p> [SEG:1] and the two <p> ladies </p> [SEG:2] laugh and cheer and <p> he </p> [SEG:1] high fives one <p> lady </p> [SEG:0]"
},
{
"sample_id": "v_Dty6AUdBdrE____3",
"video_id": "v_Dty6AUdBdrE",
"segment_id": "3",
"selection_index": 20,
"caption": "In the video, a woman is raking leaves and adjusts the handle of the rake to make it shorter before handing it to the girl . The setting is outdoors with greenery around, suggesting a peaceful and leisurely environment.",
"caption_sha256": "58e4aadb454317e11f622d8631a90d10761c1924790f110baaa0faa3bb1a7aad",
"caption_source": "refined_caption",
"refined_caption_original": "In the video, a woman is raking leaves and adjusts the handle of the <p> rake </p> [SEG:0] to make it shorter before handing it to the <p> girl </p> [SEG:0]. The setting is outdoors with greenery around, suggesting a peaceful and leisurely environment.",
"reference_frame_id": 0,
"source_timestamps": [
88.69,
148.51
],
"source_annotation_member": "activitynet_entities_gcg/anns/v_Dty6AUdBdrE____3.json",
"source_masks": [
{
"mask_id": "m0",
"source_seg_token": "[SEG:0]",
"source_classes": [
"girl"
],
"reference_frame_id": 0,
"source_bbox": [
438,
81,
530,
218
],
"source_mask_member": "activitynet_entities_gcg/masks/v_Dty6AUdBdrE____3/00/mask.png",
"mask_path": "samples/v_Dty6AUdBdrE____3/m0_mask.png",
"source_mask_sha256": "92ed0f6e5fd2e94e985e5b65bb50e6a3b0fdb7a9c4aa340998d911b0ee5429e7",
"mask_foreground_pixels": 4429,
"color": "#2563eb"
}
],
"groups": [
{
"group_id": "g1",
"entity_label": "Woman crouching beside the girl",
"entity_kind": "thing",
"mentions": [
{
"text": "a woman",
"occurrence": 1,
"model_occurrence": 1,
"span": [
14,
21
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The woman is visible on the right, but no person mask is supplied.",
"char_spans": [
[
14,
21
]
],
"mask_evidence": null
},
{
"group_id": "g2",
"entity_label": "Fallen leaves being raked",
"entity_kind": "collective",
"mentions": [
{
"text": "leaves",
"occurrence": 1,
"model_occurrence": 1,
"span": [
32,
38
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "Fallen leaves are visible across the foreground; no matching mask is supplied.",
"char_spans": [
[
32,
38
]
],
"mask_evidence": null
},
{
"group_id": "g3",
"entity_label": "Rake handle",
"entity_kind": "thing",
"mentions": [
{
"text": "the handle",
"occurrence": 1,
"model_occurrence": 1,
"span": [
51,
61
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The rake shaft is visible between the people and rake head, but has no separate supplied mask.",
"char_spans": [
[
51,
61
]
],
"mask_evidence": null
},
{
"group_id": "g4",
"entity_label": "Rake being adjusted and handed to the girl",
"entity_kind": "thing",
"mentions": [
{
"text": "the rake",
"occurrence": 1,
"model_occurrence": 1,
"span": [
65,
73
]
},
{
"text": "it",
"occurrence": 1,
"model_occurrence": 1,
"span": [
82,
84
]
},
{
"text": "it",
"occurrence": 2,
"model_occurrence": 2,
"span": [
108,
110
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "medium",
"reason": "The rake is visible without a supplied mask. Both pronouns are provisionally read as referring to the rake; the first could instead refer specifically to its handle.",
"char_spans": [
[
65,
73
],
[
82,
84
],
[
108,
110
]
],
"mask_evidence": null
},
{
"group_id": "g5",
"entity_label": "Young girl beside the woman",
"entity_kind": "thing",
"mentions": [
{
"text": "the girl",
"occurrence": 1,
"model_occurrence": 1,
"span": [
114,
122
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The girl is visible near the center. Despite its category hint, m0 covers part of a background car rather than the girl.",
"char_spans": [
[
114,
122
]
],
"mask_evidence": null
},
{
"group_id": "g6",
"entity_label": "Surrounding green vegetation",
"entity_kind": "stuff",
"mentions": [
{
"text": "greenery",
"occurrence": 1,
"model_occurrence": 1,
"span": [
154,
162
]
}
],
"mask_id": null,
"candidate_mask_ids": [],
"grounding_status": "missing_mask",
"visibility": "reference_visible",
"confidence": "high",
"reason": "The lawn and shrubbery provide visible greenery, but no vegetation mask is supplied.",
"char_spans": [
[
154,
162
]
],
"mask_evidence": null
}
],
"issues": [
{
"type": "other",
"text": "the girl",
"description": "The only supplied mask, m0, is labeled girl but actually covers a portion of a background car. It cannot ground any caption entity."
},
{
"type": "uncertain_coreference",
"text": "make it shorter",
"description": "The first 'it' could refer to the rake or specifically to the handle. It is provisionally grouped with the rake and requires review."
}
],
"model_needs_review": true,
"model_review_reasons": [
"All six entity groups lack matching supplied masks.",
"The supplied m0 mask depicts part of a car rather than its hinted girl identity.",
"The antecedent of the first 'it' requires confirmation."
],
"local_adjustments": [],
"validation": {
"structurally_valid": true,
"errors": [],
"warnings": [],
"all_caption_wording_preserved": true,
"all_resolved_mentions_exact": true,
"cross_group_overlaps_allowed": [],
"unresolved_groups": 6,
"unused_source_mask_ids": [
"m0"
],
"complete_correspondence_candidate": false,
"human_reviewed": false
},
"model": "gpt-6.1-sol",
"response_id": "resp_0577b7242458c51e016ae4c5eac25887d1b6915a3fa5eec62c",
"api_usage": {
"input_tokens": 17163,
"input_tokens_details": {
"cache_write_tokens": 7660,
"cached_tokens": 9500
},
"output_tokens": 1224,
"output_tokens_details": {
"reasoning_tokens": 172
},
"total_tokens": 18387
},
"request_custom_id": "gcg-object20::v_Dty6AUdBdrE____3",
"prompt_sha256": "f4f2c370eab558f31c5e195fe1953a6232ee5bab77e1c89e5fed96768dd56213",
"status": "partial_or_needs_review",
"asset_dir": "assets/v_Dty6AUdBdrE____3",
"estimated_frame_times_s": [
91.681,
97.663,
103.645,
109.627,
115.609,
121.591,
127.573,
133.555,
139.53699999999998,
145.519
],
"new_caption_original": "The woman adjusted the handle of the rake to make it shorter and handed to the <p> girl </p> [SEG:0]"
}
]