evaluation/results.json

{
 "temperature": {
  "choice": 1.9567745923995972,
  "noul": 1.6382114887237549,
  "by_options": {
   "choice:2": 1.8836495876312256,
   "choice:3-5": 1.979034662246704,
   "noul:2": 1.6382114887237549
  }
 },
 "test_sets": {
  "test_exist": {
   "questions": 1120,
   "accuracy": 0.9384,
   "nll": 0.162,
   "ece": 0.0204,
   "accuracy_mismatched": 0.4929
  },
  "test_vqa_choice": {
   "questions": 1420,
   "accuracy": 0.8979,
   "nll": 0.2706,
   "ece": 0.0262,
   "accuracy_mismatched": 0.3993
  },
  "test_size": {
   "questions": 1100,
   "accuracy": 0.86,
   "nll": 0.3503,
   "ece": 0.036,
   "accuracy_mismatched": 0.5045
  },
  "bench_pope": {
   "questions": 8676,
   "accuracy": 0.8511,
   "nll": 0.3702,
   "ece": 0.0578,
   "accuracy_mismatched": 0.527
  },
  "test_gqa": {
   "questions": 992,
   "accuracy": 0.7843,
   "nll": 0.409,
   "ece": 0.0419,
   "accuracy_mismatched": 0.5323
  },
  "test_position": {
   "questions": 1274,
   "accuracy": 0.8721,
   "nll": 0.3045,
   "ece": 0.0251,
   "accuracy_mismatched": 0.4929
  },
  "test_vqa_yesno": {
   "questions": 1000,
   "accuracy": 0.71,
   "nll": 0.5768,
   "ece": 0.0223,
   "accuracy_mismatched": 0.518
  },
  "test_relation": {
   "questions": 1950,
   "accuracy": 0.7108,
   "nll": 0.5401,
   "ece": 0.0471,
   "accuracy_mismatched": 0.4841
  },
  "test_vsr": {
   "questions": 904,
   "accuracy": 0.6792,
   "nll": 0.6155,
   "ece": 0.0478,
   "accuracy_mismatched": 0.4812
  },
  "test_v7w": {
   "questions": 1000,
   "accuracy": 0.721,
   "nll": 0.6838,
   "ece": 0.0302,
   "accuracy_mismatched": 0.422
  },
  "test_count_fresh": {
   "questions": 600,
   "accuracy": 0.74,
   "nll": 0.5763,
   "ece": 0.0648,
   "accuracy_mismatched": 0.5067
  }
 },
 "laya_vision": {
  "vqav2_yesno": {
   "questions": 4887,
   "laya_vision": 0.7172,
   "devision": 0.7248,
   "difference": 0.0076,
   "ci95": [
    -0.0087,
    0.0233
   ]
  },
  "aokvqa": {
   "questions": 1138,
   "laya_vision": 0.5984,
   "devision": 0.6257,
   "difference": 0.0272,
   "ci95": [
    -0.0062,
    0.0598
   ]
  },
  "scienceqa": {
   "questions": 2097,
   "laya_vision": 0.824,
   "devision": 0.7663,
   "difference": -0.0577,
   "ci95": [
    -0.0792,
    -0.0362
   ]
  },
  "scienceqa_natural_needs_picture": {
   "questions": 323,
   "laya_vision": 0.6997,
   "devision": 0.4551,
   "difference": -0.2446,
   "ci95": [
    -0.3158,
    -0.1765
   ]
  },
  "pope": {
   "random": {
    "laya_vision_published": 0.836,
    "devision": 0.8913
   },
   "popular": {
    "laya_vision_published": 0.819,
    "devision": 0.868
   },
   "adversarial": {
    "laya_vision_published": 0.777,
    "devision": 0.7913
   }
  }
 },
 "cpu_latency": {
  "p50": 148,
  "p95": 161,
  "threads": 4,
  "machine": "Darwin arm64",
  "source": "runs/v12-ext/cpu/single.json"
 }
}