{
 "model": "gemini-3.8-flash",
 "experiment": "shaina_fresh_test",
 "tasks": {
  "t1": {
   "scores": {
    "n": 10,
    "accuracy": 0.4,
    "ci95": [
     0.1682,
     0.6873
    ],
    "balanced_accuracy": 0.4167,
    "cohen_kappa": 0.2405
   },
   "items": [
    {
     "id": "t1_RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352",
     "clips": [
      "RVFA__seed_vc__153f__main__id00460_KCBOXmDu5Ks_00025__id08352.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "neither",
      "both",
      "the video",
      "the audio"
     ],
     "gt_answer": "the audio",
     "pred_answer": "neither",
     "correct": false,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_RVFA__seed_vc__77f__main__id01296_B7gb-0xbMn0_00024__id06068",
     "clips": [
      "RVFA__seed_vc__77f__main__id01296_B7gb-0xbMn0_00024__id06068.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "the audio",
      "both",
      "the video",
      "neither"
     ],
     "gt_answer": "the audio",
     "pred_answer": "the audio",
     "correct": true,
     "raw": "```json\n{\"answer\": \"the audio\"}\n```"
    },
    {
     "id": "t1_RVRA__real__77f__vocoded__id05623_HbnUHzNq8XI_00003__id00739",
     "clips": [
      "RVRA__real__77f__vocoded__id05623_HbnUHzNq8XI_00003__id00739.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "neither",
      "both",
      "the audio",
      "the video"
     ],
     "gt_answer": "neither",
     "pred_answer": "neither",
     "correct": true,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_RVRA__real__153f__vocoded__id02959_hp-EccEnxCQ_00026__id08803",
     "clips": [
      "RVRA__real__153f__vocoded__id02959_hp-EccEnxCQ_00026__id08803.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "the video",
      "both",
      "neither",
      "the audio"
     ],
     "gt_answer": "neither",
     "pred_answer": "neither",
     "correct": true,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_FVFA__s2v__153f__main__id03084_BQIuTs2QZUU_00021__id08748",
     "clips": [
      "FVFA__s2v__153f__main__id03084_BQIuTs2QZUU_00021__id08748.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "the video",
      "both",
      "neither",
      "the audio"
     ],
     "gt_answer": "both",
     "pred_answer": "the video",
     "correct": false,
     "raw": "```json\n{\"answer\": \"the video\"}\n```"
    },
    {
     "id": "t1_FVFA__animate__77f__main__id05162_NpL77H2Shgk_00043__id03606",
     "clips": [
      "FVFA__animate__77f__main__id05162_NpL77H2Shgk_00043__id03606.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "the audio",
      "neither",
      "both",
      "the video"
     ],
     "gt_answer": "both",
     "pred_answer": "neither",
     "correct": false,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_FVFA__s2v__77f__main__id06390_17l4_o3k29c_00007__id07683",
     "clips": [
      "FVFA__s2v__77f__main__id06390_17l4_o3k29c_00007__id07683.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "neither",
      "both",
      "the audio",
      "the video"
     ],
     "gt_answer": "both",
     "pred_answer": "neither",
     "correct": false,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_FVFA__facefusion__153f__main__id07786_bFZAH657cFs_00051__id07683",
     "clips": [
      "FVFA__facefusion__153f__main__id07786_bFZAH657cFs_00051__id07683.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "neither",
      "both",
      "the audio",
      "the video"
     ],
     "gt_answer": "both",
     "pred_answer": "the audio",
     "correct": false,
     "raw": "```json\n{\"answer\": \"the audio\"}\n```"
    },
    {
     "id": "t1_FVFA__facefusion__77f__main__id05670_8I_YYqRsihM_00041__id04900",
     "clips": [
      "FVFA__facefusion__77f__main__id05670_8I_YYqRsihM_00041__id04900.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "the video",
      "both",
      "neither",
      "the audio"
     ],
     "gt_answer": "both",
     "pred_answer": "neither",
     "correct": false,
     "raw": "```json\n{\"answer\": \"neither\"}\n```"
    },
    {
     "id": "t1_RVFA__seed_vc__77f__main__id01858_HBSDURviflg_00015__id01156",
     "clips": [
      "RVFA__seed_vc__77f__main__id01858_HBSDURviflg_00015__id01156.mp4"
     ],
     "part": null,
     "question": "Which part of this audio-visual clip, if any, is AI-generated or manipulated? A real recording that was only processed, without changing who is seen or heard, still counts as real.",
     "options": [
      "neither",
      "the audio",
      "both",
      "the video"
     ],
     "gt_answer": "the audio",
     "pred_answer": "the audio",
     "correct": true,
     "raw": "```json\n{\"answer\": \"the audio\"}\n```"
    }
   ]
  },
  "t2": {
   "scores": {
    "n": 10,
    "accuracy": 0.2,
    "ci95": [
     0.0567,
     0.5098
    ],
    "balanced_accuracy": 0.25,
    "cohen_kappa": 0.0
   },
   "items": [
    {
     "id": "t2_FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444",
     "clips": [
      "FVRA__s2v__77f__main__id08406_qr5lYg7ej40_00213__id05444.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "full-body swap",
      "fully generated",
      "real",
      "face swap"
     ],
     "gt_answer": "fully generated",
     "pred_answer": "real",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real\"}\n```"
    },
    {
     "id": "t2_FVRA__facefusion__77f__main__id08940_B5cvspWteMI_00009__id03946",
     "clips": [
      "FVRA__facefusion__77f__main__id08940_B5cvspWteMI_00009__id03946.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "face swap",
      "full-body swap",
      "fully generated",
      "real"
     ],
     "gt_answer": "face swap",
     "pred_answer": "face swap",
     "correct": true,
     "raw": "```json\n{\"answer\": \"face swap\"}\n```"
    },
    {
     "id": "t2_RVRA__real__77f__main__id05444_QIxNYNGaVg0_00040__id08406",
     "clips": [
      "RVRA__real__77f__main__id05444_QIxNYNGaVg0_00040__id08406.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "full-body swap",
      "face swap",
      "real",
      "fully generated"
     ],
     "gt_answer": "real",
     "pred_answer": "face swap",
     "correct": false,
     "raw": "{\"answer\": \"face swap\"}"
    },
    {
     "id": "t2_FVRA__animate__77f__main__id07054_gLIAqAQ40LQ_00053__id08610",
     "clips": [
      "FVRA__animate__77f__main__id07054_gLIAqAQ40LQ_00053__id08610.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "fully generated",
      "real",
      "full-body swap",
      "face swap"
     ],
     "gt_answer": "full-body swap",
     "pred_answer": "face swap",
     "correct": false,
     "raw": "{\"answer\": \"face swap\"}"
    },
    {
     "id": "t2_RVRA__real__77f__main__id00761_C_ZBK3E8bJk_00027__id07616",
     "clips": [
      "RVRA__real__77f__main__id00761_C_ZBK3E8bJk_00027__id07616.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "real",
      "fully generated",
      "face swap",
      "full-body swap"
     ],
     "gt_answer": "real",
     "pred_answer": "real",
     "correct": true,
     "raw": "```json\n{\"answer\": \"real\"}\n```"
    },
    {
     "id": "t2_FVRA__facefusion__77f__main__id06480_YQD4AsloKcY_00076__id01534",
     "clips": [
      "FVRA__facefusion__77f__main__id06480_YQD4AsloKcY_00076__id01534.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "real",
      "face swap",
      "fully generated",
      "full-body swap"
     ],
     "gt_answer": "face swap",
     "pred_answer": "real",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real\"}\n```"
    },
    {
     "id": "t2_FVRA__s2v__77f__main__id03854_yfQGoMSBDx0_00151__id00509",
     "clips": [
      "FVRA__s2v__77f__main__id03854_yfQGoMSBDx0_00151__id00509.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "face swap",
      "full-body swap",
      "real",
      "fully generated"
     ],
     "gt_answer": "fully generated",
     "pred_answer": "face swap",
     "correct": false,
     "raw": "{\"answer\": \"face swap\"}"
    },
    {
     "id": "t2_FVRA__animate__77f__main__id02959_h9Qwwv-DApg_00016__id03078",
     "clips": [
      "FVRA__animate__77f__main__id02959_h9Qwwv-DApg_00016__id03078.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "face swap",
      "fully generated",
      "full-body swap",
      "real"
     ],
     "gt_answer": "full-body swap",
     "pred_answer": "face swap",
     "correct": false,
     "raw": "```json\n{\"answer\": \"face swap\"}\n```"
    },
    {
     "id": "t2_FVRA__s2v__77f__main__id02359_hoPN5dCzSC8_00093__id06987",
     "clips": [
      "FVRA__s2v__77f__main__id02359_hoPN5dCzSC8_00093__id06987.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "fully generated",
      "full-body swap",
      "real",
      "face swap"
     ],
     "gt_answer": "fully generated",
     "pred_answer": "real",
     "correct": false,
     "raw": "{\"answer\": \"real\"}"
    },
    {
     "id": "t2_FVRA__animate__77f__main__id03681_Szvjxba6BTg_00066__id01664",
     "clips": [
      "FVRA__animate__77f__main__id03681_Szvjxba6BTg_00066__id01664.mp4"
     ],
     "part": null,
     "question": "Which part of this video, if any, was generated or altered by AI? Ignore normal processing such as brightness, colour or compression.",
     "options": [
      "real",
      "full-body swap",
      "face swap",
      "fully generated"
     ],
     "gt_answer": "full-body swap",
     "pred_answer": "real",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real\"}\n```"
    }
   ]
  },
  "t3": {
   "scores": {
    "n": 10,
    "accuracy": 0.5,
    "ci95": [
     0.2366,
     0.7634
    ],
    "balanced_accuracy": 0.3542,
    "cohen_kappa": 0.2647
   },
   "items": [
    {
     "id": "t3_voice_A_RVRA__real__77f__vocoded__id00948_PCy1KCIsNxo_00024__id04278",
     "clips": [
      "RVRA__real__77f__vocoded__id00948_PCy1KCIsNxo_00024__id04278.mp4",
      "RVRA__real__77f__main__id00948_z-P1KSBdNU8_00128__id08581.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "fake of a different person",
      "fake of this person",
      "real, same person",
      "real, different person"
     ],
     "gt_answer": "real, same person",
     "pred_answer": "real, same person",
     "correct": true,
     "raw": "```json\n{\"answer\": \"real, same person\"}\n```"
    },
    {
     "id": "t3_face_A_RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564",
     "clips": [
      "RVRA__real__77f__main__id05877_q_ZAQf6KfTQ_00093__id05564.mp4",
      "RVRA__real__77f__main__id05877_6QUcq_6B-LU_00011__id07768.mp4"
     ],
     "part": "face",
     "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.",
     "options": [
      "real, different person",
      "real, same person",
      "fake of this person",
      "fake of a different person"
     ],
     "gt_answer": "real, same person",
     "pred_answer": "real, same person",
     "correct": true,
     "raw": "```json\n{\"answer\": \"real, same person\"}\n```"
    },
    {
     "id": "t3_voice_B_RVRA__real__153f__vocoded__id00885_xiPMwAYISG0_00135__id05623",
     "clips": [
      "RVRA__real__153f__vocoded__id00885_xiPMwAYISG0_00135__id05623.mp4",
      "RVRA__real__153f__main__id05623_UXUHOpRirtg_00079__id03123.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "real, same person",
      "fake of this person",
      "real, different person",
      "fake of a different person"
     ],
     "gt_answer": "real, different person",
     "pred_answer": "real, same person",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real, same person\"}\n```"
    },
    {
     "id": "t3_face_B_RVRA__real__153f__main__id06713_5qTLKWiq73k_00074__id05042",
     "clips": [
      "RVRA__real__153f__main__id06713_5qTLKWiq73k_00074__id05042.mp4",
      "RVRA__real__153f__main__id05042_K1HMS1K0kWI_00010__id00144.mp4"
     ],
     "part": "face",
     "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.",
     "options": [
      "real, same person",
      "fake of a different person",
      "fake of this person",
      "real, different person"
     ],
     "gt_answer": "real, different person",
     "pred_answer": "real, different person",
     "correct": true,
     "raw": "```json\n{\"answer\": \"real, different person\"}\n```"
    },
    {
     "id": "t3_voice_A_RVFA__seed_vc__153f__main__id04049_JCDeI7UwWnE_00080__id08096",
     "clips": [
      "RVFA__seed_vc__153f__main__id04049_JCDeI7UwWnE_00080__id08096.mp4",
      "RVRA__real__153f__main__id04049_Flu_4soH0mo_00003__id04528.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "fake of a different person",
      "real, same person",
      "real, different person",
      "fake of this person"
     ],
     "gt_answer": "fake of a different person",
     "pred_answer": "real, different person",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real, different person\"}\n```"
    },
    {
     "id": "t3_voice_B_RVFA__seed_vc__77f__main__id07574_DbcyMh1l_Ng_00043__id06429",
     "clips": [
      "RVFA__seed_vc__77f__main__id07574_DbcyMh1l_Ng_00043__id06429.mp4",
      "RVRA__real__77f__main__id06429_0aTJ9yNMAx0_00002__id06427.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "fake of this person",
      "real, different person",
      "real, same person",
      "fake of a different person"
     ],
     "gt_answer": "fake of this person",
     "pred_answer": "fake of a different person",
     "correct": false,
     "raw": "```json\n{\"answer\": \"fake of a different person\"}\n```"
    },
    {
     "id": "t3_face_A_RVRA__real__153f__main__id00885_yMWJySOEuTI_00138__id00739",
     "clips": [
      "RVRA__real__153f__main__id00885_yMWJySOEuTI_00138__id00739.mp4",
      "RVRA__real__153f__main__id00885_xiPMwAYISG0_00135__id05623.mp4"
     ],
     "part": "face",
     "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.",
     "options": [
      "real, different person",
      "fake of this person",
      "real, same person",
      "fake of a different person"
     ],
     "gt_answer": "real, same person",
     "pred_answer": "real, different person",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real, different person\"}\n```"
    },
    {
     "id": "t3_face_B_RVRA__real__77f__main__id08808_vQV6tYM_-3M_00053__id07697",
     "clips": [
      "RVRA__real__77f__main__id08808_vQV6tYM_-3M_00053__id07697.mp4",
      "RVRA__real__77f__main__id07697_diNcpyVMCk8_00047__id07630.mp4"
     ],
     "part": "face",
     "question": "Clip 2 is a real recording of one person. Look only at the faces (the inner face: eyes, nose and mouth). Is the face in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose face it is, still counts as real.",
     "options": [
      "real, same person",
      "fake of this person",
      "fake of a different person",
      "real, different person"
     ],
     "gt_answer": "real, different person",
     "pred_answer": "real, different person",
     "correct": true,
     "raw": "```json\n{\"answer\": \"real, different person\"}\n```"
    },
    {
     "id": "t3_voice_B_RVFA__seed_vc__153f__main__id02359_QiCI0mefCx4_00030__id03078",
     "clips": [
      "RVFA__seed_vc__153f__main__id02359_QiCI0mefCx4_00030__id03078.mp4",
      "RVRA__real__153f__main__id03078_4XN_hNJCe9E_00031__id06987.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "real, same person",
      "real, different person",
      "fake of a different person",
      "fake of this person"
     ],
     "gt_answer": "fake of this person",
     "pred_answer": "real, different person",
     "correct": false,
     "raw": "```json\n{\"answer\": \"real, different person\"}\n```"
    },
    {
     "id": "t3_voice_A_RVRA__real__153f__vocoded__id01124_7Y-51ooBIWQ_00040__id09218",
     "clips": [
      "RVRA__real__153f__vocoded__id01124_7Y-51ooBIWQ_00040__id09218.mp4",
      "RVRA__real__153f__main__id01124_2hg7q9O8xLo_00015__id08748.mp4"
     ],
     "part": "voice",
     "question": "Clip 2 is a real recording of one person. Listen only to the voices. Is the voice in clip 1 real or fake, and is it the same person as in clip 2? A real recording that was only processed, without changing whose voice it is, still counts as real.",
     "options": [
      "real, same person",
      "fake of this person",
      "real, different person",
      "fake of a different person"
     ],
     "gt_answer": "real, same person",
     "pred_answer": "real, same person",
     "correct": true,
     "raw": "{\"answer\": \"real, same person\"}"
    }
   ]
  }
 }
}