{
 "schema": "d1-omni-public-oracle/1",
 "what": "the publisher's model (AutoModel.from_pretrained(trust_remote_code=True), CPU, fp32) on every public record in text mode and, when it has media, in its media mode: per question the ids, the marker positions, the raw marker logits, the temperature, the probabilities, the response",
 "model": {
  "hf_id": "LiquidAI/d1-omni-600M",
  "revision": "414f8d6438174f5b2133a9c21a478fc42625e308",
  "source_sha256": {
   "encoder.py": "5f2e20319ee42a7367febf06c7653d7b3f9b28ef99a64cb9ec7739563dcfcffe",
   "modeling_d1.py": "2b71a5c909ec5d3aa6307378a41a0fd24af1e17d5c1c1f8d510582147051a686",
   "prompt.py": "a6b29a55ec8345f1fc1fdbfcc4b64d80d473dc4316f095a62c51cb6cce1194cf",
   "vision.py": "43ad71029f82e20fd26b97574627773d47639d7a3742c8b04fe5bf55f41c4c28",
   "audio.py": "c5ff09d52d6a079e79d0505c89dbd0582ba2bc9abf2b26595d8d5b1d0c347bdf",
   "config.json": "dfccae241822f81752426629c88c8b8d7efdd03dd19418b6645c604fad43d619"
  },
  "weights_sha256": "0713bb05270c2685ad106522f4092bceeeb3a93cf79b401f399a712296c911e1"
 },
 "versions": {
  "python": "3.12.11",
  "platform": "macOS-27.0-arm64-arm-64bit",
  "torch": "2.9.0",
  "transformers": "5.19.0",
  "numpy": "2.5.3",
  "tokenizers": "0.23.2",
  "safetensors": "0.8.0",
  "huggingface_hub": "1.33.0",
  "soundfile": "0.14.0",
  "torchvision": "0.24.0",
  "pillow": "12.3.0"
 },
 "reference": "AutoModel.from_pretrained(snapshot, trust_remote_code=True, dtype=torch.float32) (modeling_d1.D1OmniModel), CPU fp32, probabilities(state, questions, images, audio) per request",
 "oracle_files": [
  {
   "file": "records_ref_audio.json",
   "sha256": "c483f8c511cfcdb05a8211e495a7476b11f9318899a1125c387c9e722a6d8fbc",
   "schema": "d1-omni-reference/1",
   "written": "2026-10-08T11:20:06+0900",
   "fixtures_sha256": "18b4ddff73a8044b63f949c69245210e1af8a26c48c5fdb785fd91a7807ead5b"
  },
  {
   "file": "records_ref_images.json",
   "sha256": "681d2d10eef35ecb8980a3caf93d922a77a17e88ee58795468e78b915ac91b08",
   "schema": "d1-omni-reference/1",
   "written": "2026-10-08T08:38:10+0900",
   "fixtures_sha256": "07ea38a2c0f3d8da4afe4dd4ae79e083c46404cb8ab0ba150b9c0a3fca83ae62"
  },
  {
   "file": "records_ref.json",
   "sha256": "e7dba6f44d0452a6aea3e401746d417503056d6f468ff72b56c5511883436c5f",
   "schema": "d1-omni-reference/1",
   "written": "2026-10-08T04:10:36+0900",
   "fixtures_sha256": "c0781185cf9b86210024f94fa645f047af843bfaaf7d80c703c4272634030123"
  }
 ],
 "rows_checked": "every row's ids equal the v3 host rows (encode_rows.v3.json, the publisher's encode)",
 "records": [
  {
   "id": "aud_01",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 121,
   "usage_input_tokens": 502,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 121,
     "n_ids": 63,
     "positions": 184,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -2.8674635887145996,
      -11.332260131835938,
      -12.128717422485352,
      -0.14766040444374084
     ],
     "probs_raw": [
      0.06181371584534645,
      1.3027777640672866e-05,
      5.874532689631451e-06,
      0.93816739320755
     ],
     "probs": [
      0.06181371584534645,
      1.3027777640672866e-05,
      5.874532689631451e-06,
      0.93816739320755
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.8763536773622036,
     "near_tie": false,
     "gold": "delivery",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 121,
     "n_ids": 33,
     "positions": 154,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -1.4458585977554321,
      -0.055500663816928864
     ],
     "probs_raw": [
      0.19935061037540436,
      0.8006494045257568
     ],
     "probs": [
      0.8006494045257568,
      0.19935061037540436
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.6012987941503525,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 121,
     "n_ids": 43,
     "positions": 164,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -11.046607971191406,
      -1.6502888202667236,
      -6.171070098876953
     ],
     "probs_raw": [
      8.212868851842359e-05,
      0.9891554713249207,
      0.010762522928416729
     ],
     "probs": [
      8.212868851842359e-05,
      0.9891554713249207,
      0.010762522928416729
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.9783929483965039,
     "near_tie": false,
     "gold": "1",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.93816739320755,
      "probabilities": {
       "delivery": 0.06181371584534645,
       "billing": 1.3027777640672866e-05,
       "account": 5.874532689631451e-06,
       "feedback": 0.93816739320755
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.8006494045257568
     },
     "urgency": {
      "type": "score",
      "score": 1.0106805171817541,
      "confidence": 0.9891554713249207,
      "probabilities": {
       "0": 8.212868851842359e-05,
       "1": 0.9891554713249207,
       "2": 0.010762522928416729
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 502,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_01",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_02",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 110,
   "usage_input_tokens": 469,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 63,
     "positions": 173,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -11.529346466064453,
      -16.728370666503906,
      -14.800735473632812,
      0.060165852308273315
     ],
     "probs_raw": [
      9.262639650842175e-06,
      5.114783263593381e-08,
      3.5155119348928565e-07,
      0.9999903440475464
     ],
     "probs": [
      9.262639650842175e-06,
      5.114783263593381e-08,
      3.5155119348928565e-07,
      0.9999903440475464
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9999810814078955,
     "near_tie": false,
     "gold": "feedback",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 33,
     "positions": 143,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      0.2793175280094147,
      -3.478671073913574
     ],
     "probs_raw": [
      0.9772013425827026,
      0.02279871143400669
     ],
     "probs": [
      0.02279871143400669,
      0.9772013425827026
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.954402631148696,
     "near_tie": false,
     "gold": "false",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 43,
     "positions": 153,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -12.738178253173828,
      -5.769467830657959,
      1.4846340417861938
     ],
     "probs_raw": [
      6.64972333197511e-07,
      0.0007067669648677111,
      0.9992924928665161
     ],
     "probs": [
      6.64972333197511e-07,
      0.0007067669648677111,
      0.9992924928665161
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9985857259016484,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 5.820766091346741e-11,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9999903440475464,
      "probabilities": {
       "delivery": 9.262639650842175e-06,
       "billing": 5.114783263593381e-08,
       "account": 3.5155119348928565e-07,
       "feedback": 0.9999903440475464
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.02279871143400669
     },
     "urgency": {
      "type": "score",
      "score": 1.9992917526979,
      "confidence": 0.9992924928665161,
      "probabilities": {
       "0": 6.64972333197511e-07,
       "1": 0.0007067669648677111,
       "2": 0.9992924928665161
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 469,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_02",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_03",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 110,
   "usage_input_tokens": 451,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 57,
     "positions": 167,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -8.26899528503418,
      -5.348294258117676,
      -9.974102020263672,
      -1.3370122909545898
     ],
     "probs_raw": [
      0.0009576162556186318,
      0.0177678894251585,
      0.0001740496081765741,
      0.9811004400253296
     ],
     "probs": [
      0.0009576162556186318,
      0.0177678894251585,
      0.0001740496081765741,
      0.9811004400253296
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9633325506001711,
     "near_tie": false,
     "gold": "account",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4551915228366852e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 27,
     "positions": 137,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -1.357582926750183,
      -1.9369168281555176
     ],
     "probs_raw": [
      0.6409140825271606,
      0.3590858578681946
     ],
     "probs": [
      0.3590858578681946,
      0.6409140825271606
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.28182822465896606,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 110,
     "n_ids": 37,
     "positions": 147,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -13.345606803894043,
      -3.927008628845215,
      1.5194247961044312
     ],
     "probs_raw": [
      3.4860235587075294e-07,
      0.004293142817914486,
      0.9957064986228943
     ],
     "probs": [
      3.4860235587075294e-07,
      0.004293142817914486,
      0.9957064986228943
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9914133558049798,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9811004400253296,
      "probabilities": {
       "delivery": 0.0009576162556186318,
       "billing": 0.0177678894251585,
       "account": 0.0001740496081765741,
       "feedback": 0.9811004400253296
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.3590858578681946
     },
     "urgency": {
      "type": "score",
      "score": 1.995706140063703,
      "confidence": 0.9957064986228943,
      "probabilities": {
       "0": 3.4860235587075294e-07,
       "1": 0.004293142817914486,
       "2": 0.9957064986228943
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 451,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_03",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "aud_04",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 35,
   "usage_input_tokens": 244,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 35,
     "n_ids": 63,
     "positions": 98,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -11.180398941040039,
      -14.908483505249023,
      -14.098936080932617,
      -0.7033980488777161
     ],
     "probs_raw": [
      2.8176236810395494e-05,
      6.773240102120326e-07,
      1.5218730595734087e-06,
      0.9999696016311646
     ],
     "probs": [
      2.8176236810395494e-05,
      6.773240102120326e-07,
      1.5218730595734087e-06,
      0.9999696016311646
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9999414253943542,
     "near_tie": false,
     "gold": "account",
     "correct": false,
     "host_softmax_max_abs_dp": 5.684341886080802e-14,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 35,
     "n_ids": 33,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -0.5335029363632202,
      -3.4489498138427734
     ],
     "probs_raw": [
      0.9486048221588135,
      0.0513952411711216
     ],
     "probs": [
      0.0513952411711216,
      0.9486048221588135
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.8972095809876919,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 35,
     "n_ids": 43,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -13.666671752929688,
      -7.450804710388184,
      2.360034465789795
     ],
     "probs_raw": [
      1.0956349427715395e-07,
      5.4850806918693706e-05,
      0.9999450445175171
     ],
     "probs": [
      1.0956349427715395e-07,
      5.4850806918693706e-05,
      0.9999450445175171
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9998901937105984,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9999696016311646,
      "probabilities": {
       "delivery": 2.8176236810395494e-05,
       "billing": 6.773240102120326e-07,
       "account": 1.5218730595734087e-06,
       "feedback": 0.9999696016311646
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.0513952411711216
     },
     "urgency": {
      "type": "score",
      "score": 1.9999449398419529,
      "confidence": 0.9999450445175171,
      "probabilities": {
       "0": 1.0956349427715395e-07,
       "1": 5.4850806918693706e-05,
       "2": 0.9999450445175171
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 244,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_04",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_05",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 36,
   "usage_input_tokens": 229,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 36,
     "n_ids": 57,
     "positions": 93,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -11.728607177734375,
      -14.39934253692627,
      -4.546878814697266,
      -2.9778451919555664
     ],
     "probs_raw": [
      0.00013103160017635673,
      9.067559403774794e-06,
      0.17233005166053772,
      0.8275298476219177
     ],
     "probs": [
      0.00013103160017635673,
      9.067559403774794e-06,
      0.17233005166053772,
      0.8275298476219177
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.65519979596138,
     "near_tie": false,
     "gold": "billing",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 36,
     "n_ids": 27,
     "positions": 63,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -0.6051132678985596,
      -0.5525320768356323
     ],
     "probs_raw": [
      0.48685774207115173,
      0.5131422877311707
     ],
     "probs": [
      0.5131422877311707,
      0.48685774207115173
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.02628454566001892,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 36,
     "n_ids": 37,
     "positions": 73,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -13.145566940307617,
      -6.944738388061523,
      -7.77335262298584
     ],
     "probs_raw": [
      0.0014094498474150896,
      0.6950807571411133,
      0.30350977182388306
     ],
     "probs": [
      0.0014094498474150896,
      0.6950807571411133,
      0.30350977182388306
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.3915709853172302,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8275298476219177,
      "probabilities": {
       "delivery": 0.00013103160017635673,
       "billing": 9.067559403774794e-06,
       "account": 0.17233005166053772,
       "feedback": 0.8275298476219177
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.5131422877311707
     },
     "urgency": {
      "type": "score",
      "score": 1.3021003007888794,
      "confidence": 0.6950807571411133,
      "probabilities": {
       "0": 0.0014094498474150896,
       "1": 0.6950807571411133,
       "2": 0.30350977182388306
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 229,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_05",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_06",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 101,
   "usage_input_tokens": 442,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 101,
     "n_ids": 63,
     "positions": 164,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -5.219973564147949,
      -7.757868766784668,
      -11.41282844543457,
      0.0007779449224472046
     ],
     "probs_raw": [
      0.005371889099478722,
      0.00042455419315956533,
      1.0980044862662908e-05,
      0.9941925406455994
     ],
     "probs": [
      0.005371889099478722,
      0.00042455419315956533,
      1.0980044862662908e-05,
      0.9941925406455994
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9888206515461206,
     "near_tie": false,
     "gold": "delivery",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9103830456733704e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 101,
     "n_ids": 33,
     "positions": 134,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -0.22706714272499084,
      -3.0375657081604004
     ],
     "probs_raw": [
      0.9432405829429626,
      0.05675949156284332
     ],
     "probs": [
      0.05675949156284332,
      0.9432405829429626
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.8864810913801193,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 101,
     "n_ids": 43,
     "positions": 144,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -13.540458679199219,
      -7.2855424880981445,
      -5.484748840332031
     ],
     "probs_raw": [
      0.0002722343779169023,
      0.1417158991098404,
      0.8580119013786316
     ],
     "probs": [
      0.0002722343779169023,
      0.1417158991098404,
      0.8580119013786316
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.7162960022687912,
     "near_tie": false,
     "gold": "1",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9941925406455994,
      "probabilities": {
       "delivery": 0.005371889099478722,
       "billing": 0.00042455419315956533,
       "account": 1.0980044862662908e-05,
       "feedback": 0.9941925406455994
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.05675949156284332
     },
     "urgency": {
      "type": "score",
      "score": 1.8577397018671036,
      "confidence": 0.8580119013786316,
      "probabilities": {
       "0": 0.0002722343779169023,
       "1": 0.1417158991098404,
       "2": 0.8580119013786316
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 442,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_06",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_07",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 106,
   "usage_input_tokens": 439,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 106,
     "n_ids": 57,
     "positions": 163,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -13.036137580871582,
      -11.367012023925781,
      -12.230705261230469,
      -2.8472213745117188
     ],
     "probs_raw": [
      3.757253216463141e-05,
      0.00019941713253501803,
      8.407468703808263e-05,
      0.9996789693832397
     ],
     "probs": [
      3.757253216463141e-05,
      0.00019941713253501803,
      8.407468703808263e-05,
      0.9996789693832397
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9994795522507047,
     "near_tie": false,
     "gold": "feedback",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 106,
     "n_ids": 27,
     "positions": 133,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      0.03376114368438721,
      -3.4692351818084717
     ],
     "probs_raw": [
      0.970772922039032,
      0.02922709286212921
     ],
     "probs": [
      0.02922709286212921,
      0.970772922039032
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9415458291769028,
     "near_tie": false,
     "gold": "false",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 106,
     "n_ids": 37,
     "positions": 143,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -9.906222343444824,
      -5.438223838806152,
      1.249060869216919
     ],
     "probs_raw": [
      1.4281534276960883e-05,
      0.0012450935319066048,
      0.9987406134605408
     ],
     "probs": [
      1.4281534276960883e-05,
      0.0012450935319066048,
      0.9987406134605408
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9974955199286342,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 1.1641532182693481e-10,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9996789693832397,
      "probabilities": {
       "delivery": 3.757253216463141e-05,
       "billing": 0.00019941713253501803,
       "account": 8.407468703808263e-05,
       "feedback": 0.9996789693832397
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.02922709286212921
     },
     "urgency": {
      "type": "score",
      "score": 1.9987263204529881,
      "confidence": 0.9987406134605408,
      "probabilities": {
       "0": 1.4281534276960883e-05,
       "1": 0.0012450935319066048,
       "2": 0.9987406134605408
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 439,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_07",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_08",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 199,
   "usage_input_tokens": 736,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 199,
     "n_ids": 63,
     "positions": 262,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -8.704425811767578,
      -3.4039201736450195,
      -11.297805786132812,
      -0.0038675889372825623
     ],
     "probs_raw": [
      0.00016108817362692207,
      0.03228823095560074,
      1.204406453325646e-05,
      0.967538595199585
     ],
     "probs": [
      0.00016108817362692207,
      0.03228823095560074,
      1.204406453325646e-05,
      0.967538595199585
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9352503642439842,
     "near_tie": false,
     "gold": "billing",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 199,
     "n_ids": 33,
     "positions": 232,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -1.890669584274292,
      1.8680564165115356
     ],
     "probs_raw": [
      0.022782286629080772,
      0.9772177338600159
     ],
     "probs": [
      0.9772177338600159,
      0.022782286629080772
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9544354472309351,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 199,
     "n_ids": 43,
     "positions": 242,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -13.792383193969727,
      -4.434501647949219,
      -5.2392988204956055
     ],
     "probs_raw": [
      5.961775968899019e-05,
      0.6909584999084473,
      0.30898186564445496
     ],
     "probs": [
      5.961775968899019e-05,
      0.6909584999084473,
      0.30898186564445496
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.3819766342639923,
     "near_tie": false,
     "gold": "1",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.967538595199585,
      "probabilities": {
       "delivery": 0.00016108817362692207,
       "billing": 0.03228823095560074,
       "account": 1.204406453325646e-05,
       "feedback": 0.967538595199585
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9772177338600159
     },
     "urgency": {
      "type": "score",
      "score": 1.3089222311973572,
      "confidence": 0.6909584999084473,
      "probabilities": {
       "0": 5.961775968899019e-05,
       "1": 0.6909584999084473,
       "2": 0.30898186564445496
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 736,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_08",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_09",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 172,
   "usage_input_tokens": 637,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 172,
     "n_ids": 57,
     "positions": 229,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -10.309013366699219,
      -12.59838581085205,
      -9.959497451782227,
      -2.7299599647521973
     ],
     "probs_raw": [
      0.0005103875882923603,
      5.1717615860980004e-05,
      0.0007239239639602602,
      0.9987140893936157
     ],
     "probs": [
      0.0005103875882923603,
      5.1717615860980004e-05,
      0.0007239239639602602,
      0.9987140893936157
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9979901654296555,
     "near_tie": false,
     "gold": "account",
     "correct": false,
     "host_softmax_max_abs_dp": 1.1920928955078125e-07,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 172,
     "n_ids": 27,
     "positions": 199,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -1.9508432149887085,
      -2.6538851261138916
     ],
     "probs_raw": [
      0.6688618659973145,
      0.33113816380500793
     ],
     "probs": [
      0.33113816380500793,
      0.6688618659973145
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.3377237021923065,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 172,
     "n_ids": 37,
     "positions": 209,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -12.979911804199219,
      -4.641338348388672,
      3.067082166671753
     ],
     "probs_raw": [
      1.0732090771625735e-07,
      0.00044882844667881727,
      0.999550998210907
     ],
     "probs": [
      1.0732090771625735e-07,
      0.00044882844667881727,
      0.999550998210907
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9991021697642282,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 7.105427357601002e-15,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9987140893936157,
      "probabilities": {
       "delivery": 0.0005103875882923603,
       "billing": 5.1717615860980004e-05,
       "account": 0.0007239239639602602,
       "feedback": 0.9987140893936157
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.33113816380500793
     },
     "urgency": {
      "type": "score",
      "score": 1.9995508248684928,
      "confidence": 0.999550998210907,
      "probabilities": {
       "0": 1.0732090771625735e-07,
       "1": 0.00044882844667881727,
       "2": 0.999550998210907
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 637,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_09",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_10",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 269,
   "usage_input_tokens": 946,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 269,
     "n_ids": 63,
     "positions": 332,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -6.047428131103516,
      -11.045299530029297,
      -10.394746780395508,
      -0.31905287504196167
     ],
     "probs_raw": [
      0.003241606056690216,
      2.1888312403461896e-05,
      4.195113797322847e-05,
      0.9966945648193359
     ],
     "probs": [
      0.003241606056690216,
      2.1888312403461896e-05,
      4.195113797322847e-05,
      0.9966945648193359
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9934529587626457,
     "near_tie": false,
     "gold": "feedback",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 269,
     "n_ids": 33,
     "positions": 302,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -0.8847565054893494,
      -3.556145191192627
     ],
     "probs_raw": [
      0.9353170394897461,
      0.06468290090560913
     ],
     "probs": [
      0.06468290090560913,
      0.9353170394897461
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.870634138584137,
     "near_tie": false,
     "gold": "false",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 269,
     "n_ids": 43,
     "positions": 312,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -12.225658416748047,
      -3.0319995880126953,
      -1.7885916233062744
     ],
     "probs_raw": [
      2.27603468374582e-05,
      0.22383823990821838,
      0.776138961315155
     ],
     "probs": [
      2.27603468374582e-05,
      0.22383823990821838,
      0.776138961315155
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.5523007214069366,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9966945648193359,
      "probabilities": {
       "delivery": 0.003241606056690216,
       "billing": 2.1888312403461896e-05,
       "account": 4.195113797322847e-05,
       "feedback": 0.9966945648193359
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.06468290090560913
     },
     "urgency": {
      "type": "score",
      "score": 1.7761161625385284,
      "confidence": 0.776138961315155,
      "probabilities": {
       "0": 2.27603468374582e-05,
       "1": 0.22383823990821838,
       "2": 0.776138961315155
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 946,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_10",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_11",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 263,
   "usage_input_tokens": 910,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 263,
     "n_ids": 57,
     "positions": 320,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -5.378786087036133,
      -6.809049606323242,
      -13.527311325073242,
      -2.2166378498077393
     ],
     "probs_raw": [
      0.04022393375635147,
      0.009623407386243343,
      1.1631209417828359e-05,
      0.9501410126686096
     ],
     "probs": [
      0.04022393375635147,
      0.009623407386243343,
      1.1631209417828359e-05,
      0.9501410126686096
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9099170789122581,
     "near_tie": false,
     "gold": "delivery",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 263,
     "n_ids": 27,
     "positions": 290,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -2.169802665710449,
      -2.4261226654052734
     ],
     "probs_raw": [
      0.5637314319610596,
      0.43626853823661804
     ],
     "probs": [
      0.43626853823661804,
      0.5637314319610596
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.12746289372444153,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 263,
     "n_ids": 37,
     "positions": 300,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -13.344354629516602,
      -3.0399885177612305,
      -0.4444863498210907
     ],
     "probs_raw": [
      2.3249160676641623e-06,
      0.06942828744649887,
      0.9305693507194519
     ],
     "probs": [
      2.3249160676641623e-06,
      0.06942828744649887,
      0.9305693507194519
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.861141063272953,
     "near_tie": false,
     "gold": "1",
     "correct": false,
     "host_softmax_max_abs_dp": 2.2737367544323206e-13,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9501410126686096,
      "probabilities": {
       "delivery": 0.04022393375635147,
       "billing": 0.009623407386243343,
       "account": 1.1631209417828359e-05,
       "feedback": 0.9501410126686096
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.43626853823661804
     },
     "urgency": {
      "type": "score",
      "score": 1.9305669888854027,
      "confidence": 0.9305693507194519,
      "probabilities": {
       "0": 2.3249160676641623e-06,
       "1": 0.06942828744649887,
       "2": 0.9305693507194519
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 910,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_11",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_12",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 350,
   "usage_input_tokens": 1189,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 350,
     "n_ids": 63,
     "positions": 413,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -8.041118621826172,
      -6.483062744140625,
      -13.2620849609375,
      -0.41955888271331787
     ],
     "probs_raw": [
      0.0004884006921201944,
      0.0023196975234895945,
      2.638391379150562e-06,
      0.9971892237663269
     ],
     "probs": [
      0.0004884006921201944,
      0.0023196975234895945,
      2.638391379150562e-06,
      0.9971892237663269
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9948695262428373,
     "near_tie": false,
     "gold": "billing",
     "correct": false,
     "host_softmax_max_abs_dp": 2.2737367544323206e-13,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 350,
     "n_ids": 33,
     "positions": 383,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -0.8726661205291748,
      -2.7932546138763428
     ],
     "probs_raw": [
      0.8722040057182312,
      0.12779594957828522
     ],
     "probs": [
      0.12779594957828522,
      0.8722040057182312
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.744408056139946,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 350,
     "n_ids": 43,
     "positions": 393,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -10.186288833618164,
      -1.5687068700790405,
      0.817966103553772
     ],
     "probs_raw": [
      1.5230326425808016e-05,
      0.08419332653284073,
      0.915791392326355
     ],
     "probs": [
      1.5230326425808016e-05,
      0.08419332653284073,
      0.915791392326355
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.8315980657935143,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 9.094947017729282e-13,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9971892237663269,
      "probabilities": {
       "delivery": 0.0004884006921201944,
       "billing": 0.0023196975234895945,
       "account": 2.638391379150562e-06,
       "feedback": 0.9971892237663269
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.12779594957828522
     },
     "urgency": {
      "type": "score",
      "score": 1.9157761111855507,
      "confidence": 0.915791392326355,
      "probabilities": {
       "0": 1.5230326425808016e-05,
       "1": 0.08419332653284073,
       "2": 0.915791392326355
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 1189,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_12",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_13",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 356,
   "usage_input_tokens": 1189,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 356,
     "n_ids": 57,
     "positions": 413,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -10.233248710632324,
      -8.533600807189941,
      -11.744644165039062,
      -1.8605585098266602
     ],
     "probs_raw": [
      0.00023073617194313556,
      0.0012625929666683078,
      5.0900838687084615e-05,
      0.9984557628631592
     ],
     "probs": [
      0.00023073617194313556,
      0.0012625929666683078,
      5.0900838687084615e-05,
      0.9984557628631592
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9971931698964909,
     "near_tie": false,
     "gold": "account",
     "correct": false,
     "host_softmax_max_abs_dp": 3.637978807091713e-12,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 356,
     "n_ids": 27,
     "positions": 383,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -1.111641764640808,
      -1.4849694967269897
     ],
     "probs_raw": [
      0.5922628045082092,
      0.4077371656894684
     ],
     "probs": [
      0.4077371656894684,
      0.5922628045082092
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.18452563881874084,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 356,
     "n_ids": 37,
     "positions": 393,
     "bucket": 512,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -8.093282699584961,
      -3.591269016265869,
      1.4334510564804077
     ],
     "probs_raw": [
      7.239612750709057e-05,
      0.006530025042593479,
      0.99339759349823
     ],
     "probs": [
      7.239612750709057e-05,
      0.006530025042593479,
      0.99339759349823
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9868675684556365,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9984557628631592,
      "probabilities": {
       "delivery": 0.00023073617194313556,
       "billing": 0.0012625929666683078,
       "account": 5.0900838687084615e-05,
       "feedback": 0.9984557628631592
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.4077371656894684
     },
     "urgency": {
      "type": "score",
      "score": 1.9933252120390534,
      "confidence": 0.99339759349823,
      "probabilities": {
       "0": 7.239612750709057e-05,
       "1": 0.006530025042593479,
       "2": 0.99339759349823
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 1189,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_13",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_14",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 7,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 7,
     "n_ids": 57,
     "positions": 64,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      12,
      25,
      36,
      45
     ],
     "logits_raw": [
      -14.164146423339844,
      -10.380624771118164,
      -15.470945358276367,
      -2.3223929405212402
     ],
     "probs_raw": [
      7.1953281803871505e-06,
      0.0003163830260746181,
      1.9476683519314975e-06,
      0.9996744394302368
     ],
     "probs": [
      7.1953281803871505e-06,
      0.0003163830260746181,
      1.9476683519314975e-06,
      0.9996744394302368
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9993580564041622,
     "near_tie": false,
     "gold": "billing",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 7,
     "n_ids": 27,
     "positions": 34,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      15,
      21
     ],
     "logits_raw": [
      -1.1729061603546143,
      -0.18968117237091064
     ],
     "probs_raw": [
      0.27225232124328613,
      0.7277476191520691
     ],
     "probs": [
      0.7277476191520691,
      0.27225232124328613
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.45549529790878296,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 7,
     "n_ids": 37,
     "positions": 44,
     "bucket": 256,
     "ids": [
      1,
      17,
      24373,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      11,
      20,
      28
     ],
     "logits_raw": [
      -10.382744789123535,
      -6.77346658706665,
      -5.851489067077637
     ],
     "probs_raw": [
      0.007644413039088249,
      0.2823798954486847,
      0.7099757194519043
     ],
     "probs": [
      0.007644413039088249,
      0.2823798954486847,
      0.7099757194519043
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.4275958240032196,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.9996744394302368,
      "probabilities": {
       "delivery": 7.1953281803871505e-06,
       "billing": 0.0003163830260746181,
       "account": 1.9476683519314975e-06,
       "feedback": 0.9996744394302368
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.7277476191520691
     },
     "urgency": {
      "type": "score",
      "score": 1.7023313343524933,
      "confidence": 0.7099757194519043,
      "probabilities": {
       "0": 0.007644413039088249,
       "1": 0.2823798954486847,
       "2": 0.7099757194519043
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_14",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 121,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 49,
     "positions": 49,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32,
      39
     ],
     "logits_raw": [
      -4.030765533447266,
      -2.516629695892334,
      -4.948546409606934,
      -1.2068599462509155
     ],
     "probs_raw": [
      0.04388390854001045,
      0.19947391748428345,
      0.017527427524328232,
      0.7391147613525391
     ],
     "probs": [
      0.08343102037906647,
      0.24607208371162415,
      0.043311357498168945,
      0.6271855235099792
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.3811134397983551,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      14,
      26
     ],
     "logits_raw": [
      -0.19816279411315918,
      3.9951019287109375
     ],
     "probs_raw": [
      0.014872386120259762,
      0.9851276278495789
     ],
     "probs": [
      0.9252889752388,
      0.07471100240945816
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8505779728293419,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 36,
     "positions": 36,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      10,
      19,
      27
     ],
     "logits_raw": [
      -7.304863452911377,
      -6.445466995239258,
      -5.352657318115234
     ],
     "probs_raw": [
      0.09609891474246979,
      0.22696013748645782,
      0.6769409775733948
     ],
     "probs": [
      0.17439991235733032,
      0.28659766912460327,
      0.5390024185180664
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.25240474939346313,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.6271855235099792,
      "probabilities": {
       "delivery": 0.08343102037906647,
       "billing": 0.24607208371162415,
       "account": 0.043311357498168945,
       "feedback": 0.6271855235099792
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.9252889752388
     },
     "urgency": {
      "type": "score",
      "score": 1.364602506160736,
      "confidence": 0.5390024185180664,
      "probabilities": {
       "0": 0.17439991235733032,
       "1": 0.28659766912460327,
       "2": 0.5390024185180664
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 121,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_15",
   "mode": "audio",
   "source": "own_audio",
   "native": true,
   "prefix": 375,
   "usage_input_tokens": 1264,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 375,
     "n_ids": 63,
     "positions": 438,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      7832,
      572,
      1873,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      7832,
      572,
      7778,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      7832,
      572,
      12992,
      535,
      21040,
      3197,
      20,
      19,
      16,
      7832,
      572,
      15296,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      31,
      42,
      51
     ],
     "logits_raw": [
      -7.3204827308654785,
      -7.056801795959473,
      -11.541250228881836,
      -1.0992538928985596
     ],
     "probs_raw": [
      0.001977700972929597,
      0.0025743974838405848,
      2.9047207135590725e-05,
      0.995418906211853
     ],
     "probs": [
      0.001977700972929597,
      0.0025743974838405848,
      2.9047207135590725e-05,
      0.995418906211853
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.9928445087280124,
     "near_tie": false,
     "gold": "delivery",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 375,
     "n_ids": 33,
     "positions": 408,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      21,
      27
     ],
     "logits_raw": [
      -1.6256054639816284,
      0.10680553317070007
     ],
     "probs_raw": [
      0.1502794474363327,
      0.8497205972671509
     ],
     "probs": [
      0.8497205972671509,
      0.1502794474363327
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.6994411498308182,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 375,
     "n_ids": 43,
     "positions": 418,
     "bucket": 512,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -12.868496894836426,
      -3.0056636333465576,
      4.093652725219727
     ],
     "probs_raw": [
      4.2960873969377644e-08,
      0.0008249877719208598,
      0.9991748929023743
     ],
     "probs": [
      4.2960873969377644e-08,
      0.0008249877719208598,
      0.9991748929023743
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9983499051304534,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 5.820766091346741e-11,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.995418906211853,
      "probabilities": {
       "delivery": 0.001977700972929597,
       "billing": 0.0025743974838405848,
       "account": 2.9047207135590725e-05,
       "feedback": 0.995418906211853
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.8497205972671509
     },
     "urgency": {
      "type": "score",
      "score": 1.9991747735766694,
      "confidence": 0.9991748929023743,
      "probabilities": {
       "0": 4.2960873969377644e-08,
       "1": 0.0008249877719208598,
       "2": 0.9991748929023743
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 1264,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "aud_15",
   "mode": "text",
   "source": "own_audio",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 142,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      3493,
      856,
      779,
      21009,
      10789,
      1337,
      540,
      19,
      16,
      11746,
      535,
      835,
      11746,
      933,
      768,
      15073,
      8688,
      20,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      810,
      23325,
      20,
      19,
      16,
      3863,
      535,
      21040,
      3197,
      20,
      19,
      16,
      12779,
      535,
      36843,
      3321,
      1337,
      779,
      3714,
      20,
      21
     ],
     "markers": [
      18,
      29,
      39,
      46
     ],
     "logits_raw": [
      -2.1100118160247803,
      -1.3073279857635498,
      -3.290978193283081,
      1.7141824960708618
     ],
     "probs_raw": [
      0.020269835367798805,
      0.04523257911205292,
      0.0062224725261330605,
      0.9282751679420471
     ],
     "probs": [
      0.05386649817228317,
      0.0955735445022583,
      0.023170966655015945,
      0.8273889422416687
     ],
     "argmax_index": 3,
     "argmax": "feedback",
     "top2_margin": 0.7318153977394104,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "wants",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4940,
      779,
      21009,
      14065,
      875,
      3503,
      811,
      874,
      4007,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      21,
      33
     ],
     "logits_raw": [
      2.693018674850464,
      -4.27215051651001
     ],
     "probs_raw": [
      0.9990566372871399,
      0.0009433126542717218
     ],
     "probs": [
      0.015068508684635162,
      0.984931468963623
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9698629602789879,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 43,
     "positions": 43,
     "bucket": 256,
     "ids": [
      1,
      17,
      27377,
      1139,
      6634,
      988,
      768,
      5196,
      523,
      18,
      4023,
      31149,
      856,
      779,
      5940,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      32238,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      17231,
      2166,
      20,
      21
     ],
     "markers": [
      17,
      26,
      34
     ],
     "logits_raw": [
      -8.044438362121582,
      -6.318297863006592,
      -8.428503036499023
     ],
     "probs_raw": [
      0.13698606193065643,
      0.7697144746780396,
      0.09329947084188461
     ],
     "probs": [
      0.22158345580101013,
      0.6009449362754822,
      0.17747168242931366
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.37936148047447205,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "feedback",
      "confidence": 0.8273889422416687,
      "probabilities": {
       "delivery": 0.05386649817228317,
       "billing": 0.0955735445022583,
       "account": 0.023170966655015945,
       "feedback": 0.8273889422416687
      }
     },
     "wants": {
      "type": "noul",
      "noul": 0.015068508684635162
     },
     "urgency": {
      "type": "score",
      "score": 0.9558883011341095,
      "confidence": 0.6009449362754822,
      "probabilities": {
       "0": 0.22158345580101013,
       "1": 0.6009449362754822,
       "2": 0.17747168242931366
      },
      "legend": {
       "0": "Can wait",
       "1": "Soon",
       "2": "Right now"
      }
     }
    },
    "usage": {
     "input_tokens": 142,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_audio.json"
  },
  {
   "id": "card_batch_00",
   "mode": "text",
   "source": "card",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 53,
   "questions": [
    {
     "qid": "team",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 53,
     "positions": 53,
     "bucket": 256,
     "ids": [
      1,
      17,
      15090,
      856,
      1727,
      61026,
      540,
      1311,
      953,
      2888,
      17980,
      523,
      18,
      28828,
      3410,
      1753,
      9936,
      1033,
      540,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      521,
      53467,
      592,
      521,
      60827,
      1864,
      20,
      19,
      16,
      10017,
      535,
      5135,
      933,
      3475,
      50022,
      20,
      19,
      16,
      24744,
      535,
      11631,
      7937,
      59006,
      2851,
      1517,
      20,
      21
     ],
     "markers": [
      20,
      34,
      43
     ],
     "logits_raw": [
      -4.570059776306152,
      -1.822887659072876,
      -7.762394428253174
     ],
     "probs_raw": [
      0.060097839683294296,
      0.937433660030365,
      0.0024685710668563843
     ],
     "probs": [
      0.12167388945817947,
      0.865885853767395,
      0.012440279126167297
     ],
     "argmax_index": 1,
     "argmax": "technical",
     "top2_margin": 0.7442119643092155,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "team": {
      "type": "choice",
      "choice": "technical",
      "confidence": 0.865885853767395,
      "probabilities": {
       "billing": 0.12167388945817947,
       "technical": 0.865885853767395,
       "fraud": 0.012440279126167297
      }
     }
    },
    "usage": {
     "input_tokens": 53,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "card_batch_01",
   "mode": "text",
   "source": "card",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 51,
   "questions": [
    {
     "qid": "team",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 51,
     "positions": 51,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1495,
      49085,
      1403,
      859,
      2714,
      11063,
      523,
      18,
      28828,
      3410,
      1753,
      9936,
      1033,
      540,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      521,
      53467,
      592,
      521,
      60827,
      1864,
      20,
      19,
      16,
      10017,
      535,
      5135,
      933,
      3475,
      50022,
      20,
      19,
      16,
      24744,
      535,
      11631,
      7937,
      59006,
      2851,
      1517,
      20,
      21
     ],
     "markers": [
      18,
      32,
      41
     ],
     "logits_raw": [
      -4.752628326416016,
      4.058452606201172,
      -4.859061241149902
     ],
     "probs_raw": [
      0.00014902983093634248,
      0.9997169375419617,
      0.00013398309238255024
     ],
     "probs": [
      0.0018406440503895283,
      0.9964535236358643,
      0.0017058887751773
     ],
     "argmax_index": 1,
     "argmax": "technical",
     "top2_margin": 0.9946128795854747,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "team": {
      "type": "choice",
      "choice": "technical",
      "confidence": 0.9964535236358643,
      "probabilities": {
       "billing": 0.0018406440503895283,
       "technical": 0.9964535236358643,
       "fraud": 0.0017058887751773
      }
     }
    },
    "usage": {
     "input_tokens": 51,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "card_text",
   "mode": "text",
   "source": "card",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 154,
   "questions": [
    {
     "qid": "refund",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 47,
     "positions": 47,
     "bucket": 256,
     "ids": [
      1,
      17,
      550,
      953,
      15418,
      11126,
      1033,
      5805,
      521,
      8694,
      53467,
      1235,
      803,
      1377,
      523,
      18,
      4940,
      779,
      13608,
      14065,
      875,
      768,
      53467,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      25,
      37
     ],
     "logits_raw": [
      -2.0755720138549805,
      8.723648071289062
     ],
     "probs_raw": [
      2.0415003746165894e-05,
      0.9999796152114868
     ],
     "probs": [
      0.998469889163971,
      0.001530130160972476
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9969397590029985,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "team",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 56,
     "positions": 56,
     "bucket": 256,
     "ids": [
      1,
      17,
      550,
      953,
      15418,
      11126,
      1033,
      5805,
      521,
      8694,
      53467,
      1235,
      803,
      1377,
      523,
      18,
      28828,
      3410,
      1753,
      9936,
      1033,
      540,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      521,
      53467,
      592,
      521,
      60827,
      1864,
      20,
      19,
      16,
      10017,
      535,
      5135,
      933,
      3475,
      50022,
      20,
      19,
      16,
      24744,
      535,
      11631,
      7937,
      59006,
      2851,
      1517,
      20,
      21
     ],
     "markers": [
      23,
      37,
      46
     ],
     "logits_raw": [
      6.0775146484375,
      -1.2129172086715698,
      -2.832930088043213
     ],
     "probs_raw": [
      0.999183714389801,
      0.0006814766093157232,
      0.00013486169336829334
     ],
     "probs": [
      0.9928572177886963,
      0.005434439517557621,
      0.0017083368729799986
     ],
     "argmax_index": 0,
     "argmax": "billing",
     "top2_margin": 0.9874227782711387,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "urgency",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 51,
     "positions": 51,
     "bucket": 256,
     "ids": [
      1,
      17,
      550,
      953,
      15418,
      11126,
      1033,
      5805,
      521,
      8694,
      53467,
      1235,
      803,
      1377,
      523,
      18,
      4023,
      31149,
      856,
      1033,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      2577,
      7498,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      11268,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      18578,
      800,
      779,
      13608,
      2166,
      20,
      21
     ],
     "markers": [
      22,
      31,
      39
     ],
     "logits_raw": [
      -12.788542747497559,
      -6.237281322479248,
      -3.7804148197174072
     ],
     "probs_raw": [
      0.00011273521522525698,
      0.07892896980047226,
      0.9209583401679993
     ],
     "probs": [
      0.004393829498440027,
      0.19379673898220062,
      0.8018094301223755
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.6080126911401749,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "refund": {
      "type": "noul",
      "noul": 0.998469889163971
     },
     "team": {
      "type": "choice",
      "choice": "billing",
      "confidence": 0.9928572177886963,
      "probabilities": {
       "billing": 0.9928572177886963,
       "technical": 0.005434439517557621,
       "fraud": 0.0017083368729799986
      }
     },
     "urgency": {
      "type": "score",
      "score": 1.7974155992269516,
      "confidence": 0.8018094301223755,
      "probabilities": {
       "0": 0.004393829498440027,
       "1": 0.19379673898220062,
       "2": 0.8018094301223755
      },
      "legend": {
       "0": "Can wait",
       "1": "Today",
       "2": "Blocking the customer now"
      }
     }
    },
    "usage": {
     "input_tokens": 154,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "img_01",
   "mode": "image",
   "source": "own_image",
   "native": true,
   "prefix": 144,
   "usage_input_tokens": 530,
   "questions": [
    {
     "qid": "room",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 144,
     "n_ids": 39,
     "positions": 183,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      3746,
      803,
      5362,
      856,
      1033,
      540,
      19,
      16,
      15936,
      535,
      835,
      15936,
      20,
      19,
      16,
      32874,
      535,
      835,
      32874,
      20,
      19,
      16,
      6756,
      535,
      1592,
      6756,
      20,
      19,
      16,
      32062,
      535,
      835,
      32062,
      20,
      21
     ],
     "markers": [
      11,
      18,
      25,
      32
     ],
     "logits_raw": [
      -13.372673034667969,
      2.3188087940216064,
      -10.872926712036133,
      -5.046749114990234
     ],
     "probs_raw": [
      1.5310895662423718e-07,
      0.9993657469749451,
      1.8647758679435356e-06,
      0.0006322711706161499
     ],
     "probs": [
      1.5310895662423718e-07,
      0.9993657469749451,
      1.8647758679435356e-06,
      0.0006322711706161499
     ],
     "argmax_index": 1,
     "argmax": "bedroom",
     "top2_margin": 0.9987334758043289,
     "near_tie": false,
     "gold": "bedroom",
     "correct": true,
     "host_softmax_max_abs_dp": 5.820766091346741e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "suitcase",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 144,
     "n_ids": 25,
     "positions": 169,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      1413,
      768,
      14503,
      10511,
      797,
      779,
      5362,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      13,
      19
     ],
     "logits_raw": [
      -3.17026948928833,
      11.013753890991211
     ],
     "probs_raw": [
      6.91762181759259e-07,
      0.9999992847442627
     ],
     "probs": [
      0.9999992847442627,
      6.91762181759259e-07
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9999985929820809,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "brightness",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 144,
     "n_ids": 34,
     "positions": 178,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      9899,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      18003,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      19640,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      32989,
      20,
      21
     ],
     "markers": [
      10,
      18,
      26
     ],
     "logits_raw": [
      -0.548412561416626,
      -6.016256332397461,
      -5.538527011871338
     ],
     "probs_raw": [
      0.9890949130058289,
      0.004174298141151667,
      0.006730679422616959
     ],
     "probs": [
      0.9890949130058289,
      0.004174298141151667,
      0.006730679422616959
     ],
     "argmax_index": 0,
     "argmax": "0",
     "top2_margin": 0.9823642335832119,
     "near_tie": false,
     "gold": "2",
     "correct": false,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "room": {
      "type": "choice",
      "choice": "bedroom",
      "confidence": 0.9993657469749451,
      "probabilities": {
       "kitchen": 1.5310895662423718e-07,
       "bedroom": 0.9993657469749451,
       "office": 1.8647758679435356e-06,
       "bathroom": 0.0006322711706161499
      }
     },
     "suitcase": {
      "type": "noul",
      "noul": 0.9999992847442627
     },
     "brightness": {
      "type": "score",
      "score": 0.017635656986385584,
      "confidence": 0.9890949130058289,
      "probabilities": {
       "0": 0.9890949130058289,
       "1": 0.004174298141151667,
       "2": 0.006730679422616959
      },
      "legend": {
       "0": "Dark",
       "1": "Dim",
       "2": "Bright"
      }
     }
    },
    "usage": {
     "input_tokens": 530,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "img_01",
   "mode": "text",
   "source": "own_image",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 108,
   "questions": [
    {
     "qid": "room",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 39,
     "positions": 39,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      3493,
      3746,
      803,
      5362,
      856,
      1033,
      540,
      19,
      16,
      15936,
      535,
      835,
      15936,
      20,
      19,
      16,
      32874,
      535,
      835,
      32874,
      20,
      19,
      16,
      6756,
      535,
      1592,
      6756,
      20,
      19,
      16,
      32062,
      535,
      835,
      32062,
      20,
      21
     ],
     "markers": [
      11,
      18,
      25,
      32
     ],
     "logits_raw": [
      -0.9235021471977234,
      1.1862629652023315,
      -0.5316015481948853,
      0.6049119234085083
     ],
     "probs_raw": [
      0.0652020052075386,
      0.5376754999160767,
      0.09648530185222626,
      0.3006371855735779
     ],
     "probs": [
      0.1018717959523201,
      0.4598037302494049,
      0.13478277623653412,
      0.3035416603088379
     ],
     "argmax_index": 1,
     "argmax": "bedroom",
     "top2_margin": 0.15626206994056702,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "suitcase",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 35,
     "positions": 35,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      1413,
      768,
      14503,
      10511,
      797,
      779,
      5362,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      13,
      25
     ],
     "logits_raw": [
      -0.22425425052642822,
      1.6848607063293457
     ],
     "probs_raw": [
      0.12908032536506653,
      0.8709197044372559
     ],
     "probs": [
      0.7587257027626038,
      0.24127434194087982
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.5174513608217239,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "brightness",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 34,
     "positions": 34,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      9899,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      18003,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      19640,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      32989,
      20,
      21
     ],
     "markers": [
      10,
      18,
      26
     ],
     "logits_raw": [
      -2.512366533279419,
      -0.007647328078746796,
      1.206966519355774
     ],
     "probs_raw": [
      0.01835636794567108,
      0.2246842086315155,
      0.7569594383239746
     ],
     "probs": [
      0.07227402925491333,
      0.30741074681282043,
      0.6203151941299438
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.3129044473171234,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "room": {
      "type": "choice",
      "choice": "bedroom",
      "confidence": 0.4598037302494049,
      "probabilities": {
       "kitchen": 0.1018717959523201,
       "bedroom": 0.4598037302494049,
       "office": 0.13478277623653412,
       "bathroom": 0.3035416603088379
      }
     },
     "suitcase": {
      "type": "noul",
      "noul": 0.7587257027626038
     },
     "brightness": {
      "type": "score",
      "score": 1.5480411350727081,
      "confidence": 0.6203151941299438,
      "probabilities": {
       "0": 0.07227402925491333,
       "1": 0.30741074681282043,
       "2": 0.6203151941299438
      },
      "legend": {
       "0": "Dark",
       "1": "Dim",
       "2": "Bright"
      }
     }
    },
    "usage": {
     "input_tokens": 108,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "img_02",
   "mode": "image",
   "source": "own_image",
   "native": true,
   "prefix": 108,
   "usage_input_tokens": 437,
   "questions": [
    {
     "qid": "room",
     "type": "choice",
     "K": 4,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 108,
     "n_ids": 45,
     "positions": 153,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      3493,
      3746,
      803,
      5362,
      856,
      1033,
      540,
      19,
      16,
      15936,
      535,
      835,
      15936,
      20,
      19,
      16,
      32874,
      535,
      835,
      32874,
      20,
      19,
      16,
      6756,
      535,
      1592,
      6756,
      20,
      19,
      16,
      32062,
      535,
      835,
      32062,
      20,
      21
     ],
     "markers": [
      17,
      24,
      31,
      38
     ],
     "logits_raw": [
      -13.370351791381836,
      4.531220436096191,
      -10.323932647705078,
      -0.2855755388736725
     ],
     "probs_raw": [
      1.6670359315185124e-08,
      0.9919719696044922,
      3.5074251059086237e-07,
      0.008027708157896996
     ],
     "probs": [
      1.6670359315185124e-08,
      0.9919719696044922,
      3.5074251059086237e-07,
      0.008027708157896996
     ],
     "argmax_index": 1,
     "argmax": "bedroom",
     "top2_margin": 0.9839442614465952,
     "near_tie": false,
     "gold": "bedroom",
     "correct": true,
     "host_softmax_max_abs_dp": 2.842170943040401e-14,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "lamp",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 108,
     "n_ids": 28,
     "positions": 136,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      4940,
      768,
      29114,
      35589,
      884,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      16,
      22
     ],
     "logits_raw": [
      1.247429609298706,
      -0.5311287641525269
     ],
     "probs_raw": [
      0.8555187582969666,
      0.14448124170303345
     ],
     "probs": [
      0.14448124170303345,
      0.8555187582969666
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.7110375165939331,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "brightness",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 108,
     "n_ids": 40,
     "positions": 148,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      4023,
      9899,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      18003,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      19640,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      32989,
      20,
      21
     ],
     "markers": [
      16,
      24,
      32
     ],
     "logits_raw": [
      -2.5174143314361572,
      -0.759251594543457,
      -3.9761927127838135
     ],
     "probs_raw": [
      0.1421608030796051,
      0.824783980846405,
      0.033055249601602554
     ],
     "probs": [
      0.1421608030796051,
      0.824783980846405,
      0.033055249601602554
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.6826231777667999,
     "near_tie": false,
     "gold": "1",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "room": {
      "type": "choice",
      "choice": "bedroom",
      "confidence": 0.9919719696044922,
      "probabilities": {
       "kitchen": 1.6670359315185124e-08,
       "bedroom": 0.9919719696044922,
       "office": 3.5074251059086237e-07,
       "bathroom": 0.008027708157896996
      }
     },
     "lamp": {
      "type": "noul",
      "noul": 0.14448124170303345
     },
     "brightness": {
      "type": "score",
      "score": 0.8908944800496101,
      "confidence": 0.824783980846405,
      "probabilities": {
       "0": 0.1421608030796051,
       "1": 0.824783980846405,
       "2": 0.033055249601602554
      },
      "legend": {
       "0": "Dark",
       "1": "Dim",
       "2": "Bright"
      }
     }
    },
    "usage": {
     "input_tokens": 437,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "img_02",
   "mode": "text",
   "source": "own_image",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 123,
   "questions": [
    {
     "qid": "room",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 45,
     "positions": 45,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      3493,
      3746,
      803,
      5362,
      856,
      1033,
      540,
      19,
      16,
      15936,
      535,
      835,
      15936,
      20,
      19,
      16,
      32874,
      535,
      835,
      32874,
      20,
      19,
      16,
      6756,
      535,
      1592,
      6756,
      20,
      19,
      16,
      32062,
      535,
      835,
      32062,
      20,
      21
     ],
     "markers": [
      17,
      24,
      31,
      38
     ],
     "logits_raw": [
      -2.2342491149902344,
      1.2450541257858276,
      -1.7813855409622192,
      -1.1000628471374512
     ],
     "probs_raw": [
      0.026233933866024017,
      0.8509531617164612,
      0.041260987520217896,
      0.0815519317984581
     ],
     "probs": [
      0.0601099468767643,
      0.7216733694076538,
      0.0830691009759903,
      0.1351475715637207
     ],
     "argmax_index": 1,
     "argmax": "bedroom",
     "top2_margin": 0.5865257978439331,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "lamp",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 38,
     "positions": 38,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      4940,
      768,
      29114,
      35589,
      884,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      16,
      28
     ],
     "logits_raw": [
      1.744201421737671,
      -1.7224774360656738
     ],
     "probs_raw": [
      0.9697246551513672,
      0.030275333672761917
     ],
     "probs": [
      0.11101281642913818,
      0.8889872431755066
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.7779744267463684,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "brightness",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 40,
     "positions": 40,
     "bucket": 256,
     "ids": [
      1,
      17,
      26252,
      988,
      768,
      5362,
      3999,
      523,
      18,
      4023,
      9899,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      18003,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      19640,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      32989,
      20,
      21
     ],
     "markers": [
      16,
      24,
      32
     ],
     "logits_raw": [
      -1.4011543989181519,
      0.5800229907035828,
      0.1053333580493927
     ],
     "probs_raw": [
      0.0783567875623703,
      0.5681866407394409,
      0.3534564673900604
     ],
     "probs": [
      0.15310442447662354,
      0.48117661476135254,
      0.3657189905643463
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.11545762419700623,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "room": {
      "type": "choice",
      "choice": "bedroom",
      "confidence": 0.7216733694076538,
      "probabilities": {
       "kitchen": 0.0601099468767643,
       "bedroom": 0.7216733694076538,
       "office": 0.0830691009759903,
       "bathroom": 0.1351475715637207
      }
     },
     "lamp": {
      "type": "noul",
      "noul": 0.11101281642913818
     },
     "brightness": {
      "type": "score",
      "score": 1.2126145958900452,
      "confidence": 0.48117661476135254,
      "probabilities": {
       "0": 0.15310442447662354,
       "1": 0.48117661476135254,
       "2": 0.3657189905643463
      },
      "legend": {
       "0": "Dark",
       "1": "Dim",
       "2": "Bright"
      }
     }
    },
    "usage": {
     "input_tokens": 123,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "img_03",
   "mode": "image",
   "source": "own_image",
   "native": true,
   "prefix": 1770,
   "usage_input_tokens": 5411,
   "questions": [
    {
     "qid": "beds",
     "type": "choice",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 1770,
     "n_ids": 35,
     "positions": 1805,
     "bucket": 2048,
     "ids": [
      1,
      17,
      18,
      4023,
      1697,
      25893,
      938,
      797,
      779,
      5362,
      540,
      19,
      16,
      1235,
      535,
      3732,
      6100,
      20,
      19,
      16,
      1547,
      535,
      8726,
      25893,
      20,
      19,
      16,
      1209,
      535,
      13016,
      933,
      1209,
      25893,
      20,
      21
     ],
     "markers": [
      12,
      19,
      26
     ],
     "logits_raw": [
      1.1830202341079712,
      3.3079018592834473,
      -7.577269554138184
     ],
     "probs_raw": [
      0.10670007765293121,
      0.8932831883430481,
      1.673474434937816e-05
     ],
     "probs": [
      0.10670007765293121,
      0.8932831883430481,
      1.673474434937816e-05
     ],
     "argmax_index": 1,
     "argmax": "two",
     "top2_margin": 0.7865831106901169,
     "near_tie": false,
     "gold": "two",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "towel",
     "type": "noul",
     "K": 2,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 1770,
     "n_ids": 25,
     "positions": 1795,
     "bucket": 2048,
     "ids": [
      1,
      17,
      18,
      4940,
      1413,
      768,
      41478,
      50624,
      884,
      768,
      6100,
      540,
      19,
      16,
      4831,
      535,
      1295,
      20,
      19,
      16,
      3347,
      535,
      18672,
      20,
      21
     ],
     "markers": [
      13,
      19
     ],
     "logits_raw": [
      3.5223562717437744,
      -2.9612913131713867
     ],
     "probs_raw": [
      0.9984740614891052,
      0.0015258945059031248
     ],
     "probs": [
      0.0015258945059031248,
      0.9984740614891052
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.9969481669832021,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 1.1641532182693481e-10,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "tidy",
     "type": "score",
     "K": 3,
     "calibrate": false,
     "temperature_key": null,
     "T": null,
     "prefix": 1770,
     "n_ids": 41,
     "positions": 1811,
     "bucket": 2048,
     "ids": [
      1,
      17,
      18,
      4023,
      18584,
      598,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      25913,
      1064,
      11119,
      791,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      4260,
      1064,
      11119,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      837,
      48307,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32
     ],
     "logits_raw": [
      -4.784424304962158,
      -1.4155879020690918,
      3.0489602088928223
     ],
     "probs_raw": [
      0.00039161930908448994,
      0.01137446891516447,
      0.9882339239120483
     ],
     "probs": [
      0.00039161930908448994,
      0.01137446891516447,
      0.9882339239120483
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.9768594549968839,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "beds": {
      "type": "choice",
      "choice": "two",
      "confidence": 0.8932831883430481,
      "probabilities": {
       "one": 0.10670007765293121,
       "two": 0.8932831883430481,
       "more": 1.673474434937816e-05
      }
     },
     "towel": {
      "type": "noul",
      "noul": 0.0015258945059031248
     },
     "tidy": {
      "type": "score",
      "score": 1.9878423167392612,
      "confidence": 0.9882339239120483,
      "probabilities": {
       "0": 0.00039161930908448994,
       "1": 0.01137446891516447,
       "2": 0.9882339239120483
      },
      "legend": {
       "0": "Very cluttered",
       "1": "Some clutter",
       "2": "Tidy"
      }
     }
    },
    "usage": {
     "input_tokens": 5411,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "img_03",
   "mode": "text",
   "source": "own_image",
   "native": false,
   "prefix": 0,
   "usage_input_tokens": 111,
   "questions": [
    {
     "qid": "beds",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 35,
     "positions": 35,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      1697,
      25893,
      938,
      797,
      779,
      5362,
      540,
      19,
      16,
      1235,
      535,
      3732,
      6100,
      20,
      19,
      16,
      1547,
      535,
      8726,
      25893,
      20,
      19,
      16,
      1209,
      535,
      13016,
      933,
      1209,
      25893,
      20,
      21
     ],
     "markers": [
      12,
      19,
      26
     ],
     "logits_raw": [
      -0.25700244307518005,
      0.2678792476654053,
      1.0923843383789062
     ],
     "probs_raw": [
      0.1527809202671051,
      0.2582393288612366,
      0.5889797806739807
     ],
     "probs": [
      0.19697225093841553,
      0.2865765392780304,
      0.5164511799812317
     ],
     "argmax_index": 2,
     "argmax": "more",
     "top2_margin": 0.2298746407032013,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "towel",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 35,
     "positions": 35,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4940,
      1413,
      768,
      41478,
      50624,
      884,
      768,
      6100,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      13,
      25
     ],
     "logits_raw": [
      -0.3175913095474243,
      7.188986301422119
     ],
     "probs_raw": [
      0.0005491566262207925,
      0.9994508624076843
     ],
     "probs": [
      0.9890658855438232,
      0.0109340725466609
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9781318129971623,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "tidy",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 41,
     "positions": 41,
     "bucket": 256,
     "ids": [
      1,
      17,
      18,
      4023,
      18584,
      598,
      856,
      779,
      5362,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      25913,
      1064,
      11119,
      791,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      4260,
      1064,
      11119,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      837,
      48307,
      20,
      21
     ],
     "markers": [
      11,
      22,
      32
     ],
     "logits_raw": [
      -3.0232226848602295,
      -0.7932671308517456,
      3.672762393951416
     ],
     "probs_raw": [
      0.0012203309452161193,
      0.011348405852913857,
      0.9874312877655029
     ],
     "probs": [
      0.01901734061539173,
      0.06901048868894577,
      0.911972165107727
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.8429616764187813,
     "near_tie": false,
     "gold": null,
     "correct": null,
     "host_softmax_max_abs_dp": 1.862645149230957e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "beds": {
      "type": "choice",
      "choice": "more",
      "confidence": 0.5164511799812317,
      "probabilities": {
       "one": 0.19697225093841553,
       "two": 0.2865765392780304,
       "more": 0.5164511799812317
      }
     },
     "towel": {
      "type": "noul",
      "noul": 0.9890658855438232
     },
     "tidy": {
      "type": "score",
      "score": 1.8929548189043999,
      "confidence": 0.911972165107727,
      "probabilities": {
       "0": 0.01901734061539173,
       "1": 0.06901048868894577,
       "2": 0.911972165107727
      },
      "legend": {
       "0": "Very cluttered",
       "1": "Some clutter",
       "2": "Tidy"
      }
     }
    },
    "usage": {
     "input_tokens": 111,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref_images.json"
  },
  {
   "id": "long_3400",
   "mode": "text",
   "source": "own_long_extended",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 13867,
   "questions": [
    {
     "qid": "component",
     "type": "choice",
     "K": 5,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 3467,
     "positions": 3467,
     "bucket": 4096,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      708,
      4972,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4972,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4972,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4972,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4972,
      535,
      7058,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      22218,
      1532,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      4528,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      1374,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      533,
      514,
      7614,
      708,
      4528,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      898,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      28701,
      1155,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      2925,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3878,
      8061,
      1015,
      4528,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      2108,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      3167,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      26465,
      4972,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      7568,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      2999,
      8061,
      1015,
      4905,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      1056,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      530,
      514,
      7614,
      708,
      4905,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      1443,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      32522,
      6326,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      898,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3092,
      8061,
      1015,
      4905,
      535,
      898,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      525,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4905,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4905,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4905,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4905,
      535,
      3337,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      33231,
      4541,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      3167,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      532,
      514,
      7614,
      708,
      4905,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      1155,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      1155,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      1155,
      535,
      1056,
      535,
      2108,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      20425,
      968,
      4582,
      1055,
      843,
      802,
      13573,
      797,
      779,
      730,
      4972,
      535,
      2625,
      8501,
      6634,
      3271,
      2638,
      35036,
      536,
      1295,
      32215,
      4602,
      708,
      1155,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      1155,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      1155,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      1155,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      1155,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      24656,
      3361,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      1155,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      1155,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      1155,
      535,
      2197,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      28828,
      8150,
      5457,
      779,
      878,
      4318,
      1047,
      540,
      19,
      16,
      49883,
      535,
      3492,
      3955,
      49883,
      20,
      19,
      16,
      43497,
      572,
      532,
      535,
      2020,
      4674,
      777,
      43497,
      730,
      532,
      20,
      19,
      16,
      49574,
      886,
      535,
      889,
      1991,
      886,
      5716,
      20,
      19,
      16,
      2410,
      535,
      9642,
      7018,
      20,
      19,
      16,
      5901,
      535,
      10478,
      5901,
      20,
      21
     ],
     "markers": [
      3422,
      3430,
      3443,
      3453,
      3460
     ],
     "logits_raw": [
      -2.666999101638794,
      -0.14830780029296875,
      0.5828009843826294,
      -1.5615222454071045,
      -2.4719622135162354
     ],
     "probs_raw": [
      0.023023677989840508,
      0.2857777774333954,
      0.5936698317527771,
      0.06954680383205414,
      0.027981961145997047
     ],
     "probs": [
      0.048573244363069534,
      0.29361486434936523,
      0.4949849545955658,
      0.10699225217103958,
      0.05583466589450836
     ],
     "argmax_index": 2,
     "argmax": "diverter",
     "top2_margin": 0.20137009024620056,
     "near_tie": false,
     "gold": "lane_7",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 4
     }
    },
    {
     "qid": "normal_at_end",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 3449,
     "positions": 3449,
     "bucket": 4096,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      708,
      4972,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4972,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4972,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4972,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4972,
      535,
      7058,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      22218,
      1532,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      4528,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      1374,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      533,
      514,
      7614,
      708,
      4528,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      898,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      28701,
      1155,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      2925,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3878,
      8061,
      1015,
      4528,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      2108,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      3167,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      26465,
      4972,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      7568,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      2999,
      8061,
      1015,
      4905,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      1056,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      530,
      514,
      7614,
      708,
      4905,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      1443,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      32522,
      6326,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      898,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3092,
      8061,
      1015,
      4905,
      535,
      898,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      525,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4905,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4905,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4905,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4905,
      535,
      3337,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      33231,
      4541,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      3167,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      532,
      514,
      7614,
      708,
      4905,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      1155,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      1155,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      1155,
      535,
      1056,
      535,
      2108,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      20425,
      968,
      4582,
      1055,
      843,
      802,
      13573,
      797,
      779,
      730,
      4972,
      535,
      2625,
      8501,
      6634,
      3271,
      2638,
      35036,
      536,
      1295,
      32215,
      4602,
      708,
      1155,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      1155,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      1155,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      1155,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      1155,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      24656,
      3361,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      1155,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      1155,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      1155,
      535,
      2197,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      28519,
      779,
      8411,
      886,
      6840,
      12465,
      963,
      779,
      1764,
      803,
      779,
      3446,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      3427,
      3439
     ],
     "logits_raw": [
      0.9065542221069336,
      1.8131377696990967
     ],
     "probs_raw": [
      0.28769946098327637,
      0.7123005390167236
     ],
     "probs": [
      0.6327569484710693,
      0.36724305152893066
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.26551389694213867,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 3,
      "B": 4
     }
    },
    {
     "qid": "stoppage",
     "type": "score",
     "K": 6,
     "calibrate": true,
     "temperature_key": "score:6-10",
     "T": 1.0,
     "prefix": 0,
     "n_ids": 3497,
     "positions": 3497,
     "bucket": 4096,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      708,
      4972,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4972,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4972,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4972,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4972,
      535,
      7058,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      22218,
      1532,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      4528,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      1374,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      533,
      514,
      7614,
      708,
      4528,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      898,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      28701,
      1155,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      2925,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3878,
      8061,
      1015,
      4528,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      2108,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      3167,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      26465,
      4972,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      7568,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      2999,
      8061,
      1015,
      4905,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      1056,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      530,
      514,
      7614,
      708,
      4905,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      1443,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      32522,
      6326,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      898,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3092,
      8061,
      1015,
      4905,
      535,
      898,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      525,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4905,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4905,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4905,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4905,
      535,
      3337,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      33231,
      4541,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      3167,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      532,
      514,
      7614,
      708,
      4905,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      1155,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      1155,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      1155,
      535,
      1056,
      535,
      2108,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      20425,
      968,
      4582,
      1055,
      843,
      802,
      13573,
      797,
      779,
      730,
      4972,
      535,
      2625,
      8501,
      6634,
      3271,
      2638,
      35036,
      536,
      1295,
      32215,
      4602,
      708,
      1155,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      1155,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      1155,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      1155,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      1155,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      24656,
      3361,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      1155,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      1155,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      1155,
      535,
      2197,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      4023,
      1787,
      953,
      779,
      8411,
      886,
      7408,
      13525,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      1490,
      730,
      527,
      5112,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      730,
      527,
      811,
      730,
      530,
      5112,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      730,
      530,
      811,
      730,
      1155,
      5112,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      730,
      1155,
      811,
      730,
      898,
      5112,
      20,
      19,
      16,
      2119,
      730,
      529,
      535,
      730,
      898,
      811,
      730,
      2108,
      5112,
      20,
      19,
      16,
      2119,
      730,
      530,
      535,
      1391,
      730,
      2108,
      5112,
      20,
      21
     ],
     "markers": [
      3423,
      3434,
      3447,
      3460,
      3473,
      3486
     ],
     "logits_raw": [
      -5.387547016143799,
      -3.613497495651245,
      -2.67159366607666,
      -1.864549160003662,
      0.024332530796527863,
      -3.945512294769287
     ],
     "probs_raw": [
      0.00351888220757246,
      0.020742669701576233,
      0.05320204421877861,
      0.11924036592245102,
      0.7884137034416199,
      0.014882382936775684
     ],
     "probs": [
      0.00351888220757246,
      0.020742669701576233,
      0.05320204421877861,
      0.11924036592245102,
      0.7884137034416199,
      0.014882382936775684
     ],
     "argmax_index": 4,
     "argmax": "4",
     "top2_margin": 0.6691733375191689,
     "near_tie": false,
     "gold": "4",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 4
     }
    },
    {
     "qid": "tension_10",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 3454,
     "positions": 3454,
     "bucket": 4096,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      708,
      4972,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4972,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4972,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4972,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4972,
      535,
      7058,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      22218,
      1532,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      4528,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      1374,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      533,
      514,
      7614,
      708,
      4528,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      898,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      28701,
      1155,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4528,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4528,
      535,
      2925,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3878,
      8061,
      1015,
      4528,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4528,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4528,
      535,
      2108,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4528,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4528,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4528,
      535,
      3167,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      26465,
      4972,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4528,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4528,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4528,
      535,
      7568,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      2999,
      8061,
      1015,
      4905,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      1056,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      530,
      514,
      7614,
      708,
      4905,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      1443,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      32522,
      6326,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      898,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3092,
      8061,
      1015,
      4905,
      535,
      898,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      525,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4905,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4905,
      535,
      2138,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      4905,
      535,
      2925,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      4905,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4905,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      4905,
      535,
      3337,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4905,
      535,
      3337,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      4613,
      8061,
      1015,
      4905,
      535,
      3337,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      33231,
      4541,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4905,
      535,
      3167,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      4905,
      535,
      3167,
      535,
      1838,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      532,
      514,
      7614,
      708,
      4905,
      535,
      7058,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      527,
      12079,
      4905,
      535,
      7568,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      1155,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      529,
      12079,
      1155,
      535,
      1056,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      533,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      1155,
      535,
      1056,
      535,
      2108,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      20425,
      968,
      4582,
      1055,
      843,
      802,
      13573,
      797,
      779,
      730,
      4972,
      535,
      2625,
      8501,
      6634,
      3271,
      2638,
      35036,
      536,
      1295,
      32215,
      4602,
      708,
      1155,
      535,
      6326,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      1155,
      535,
      4528,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      1155,
      535,
      4528,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3756,
      8061,
      1015,
      1155,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      1155,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      24656,
      3361,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      1155,
      535,
      1443,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      1155,
      535,
      898,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      1155,
      535,
      2197,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      5257,
      779,
      730,
      1155,
      535,
      1056,
      3999,
      521,
      953,
      779,
      43497,
      730,
      532,
      23961,
      17828,
      2638,
      35036,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      3432,
      3444
     ],
     "logits_raw": [
      0.22075581550598145,
      0.5800595879554749
     ],
     "probs_raw": [
      0.41112810373306274,
      0.5888718366622925
     ],
     "probs": [
      0.5536988377571106,
      0.4463012218475342
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.10739761590957642,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 4
     }
    }
   ],
   "response": {
    "answers": {
     "component": {
      "type": "choice",
      "choice": "diverter",
      "confidence": 0.4949849545955658,
      "probabilities": {
       "scanner": 0.048573244363069534,
       "lane_7": 0.29361486434936523,
       "diverter": 0.4949849545955658,
       "power": 0.10699225217103958,
       "software": 0.05583466589450836
      }
     },
     "normal_at_end": {
      "type": "noul",
      "noul": 0.6327569484710693
     },
     "stoppage": {
      "type": "score",
      "score": 3.7129345843568444,
      "confidence": 0.7884137034416199,
      "probabilities": {
       "0": 0.00351888220757246,
       "1": 0.020742669701576233,
       "2": 0.05320204421877861,
       "3": 0.11924036592245102,
       "4": 0.7884137034416199,
       "5": 0.014882382936775684
      },
      "legend": {
       "0": "under 2 minutes",
       "1": "2 to 5 minutes",
       "2": "5 to 10 minutes",
       "3": "10 to 20 minutes",
       "4": "20 to 40 minutes",
       "5": "over 40 minutes"
      }
     },
     "tension_10": {
      "type": "noul",
      "noul": 0.5536988377571106
     }
    },
    "usage": {
     "input_tokens": 13867,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_L01",
   "mode": "text",
   "source": "own_long",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 4830,
   "questions": [
    {
     "qid": "component",
     "type": "choice",
     "K": 5,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 1606,
     "positions": 1606,
     "bucket": 2048,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      28828,
      8150,
      5457,
      779,
      878,
      4318,
      1047,
      540,
      19,
      16,
      49883,
      535,
      3492,
      3955,
      49883,
      20,
      19,
      16,
      43497,
      572,
      532,
      535,
      2020,
      4674,
      777,
      43497,
      730,
      532,
      20,
      19,
      16,
      49574,
      886,
      535,
      889,
      1991,
      886,
      5716,
      20,
      19,
      16,
      2410,
      535,
      9642,
      7018,
      20,
      19,
      16,
      5901,
      535,
      10478,
      5901,
      20,
      21
     ],
     "markers": [
      1561,
      1569,
      1582,
      1592,
      1599
     ],
     "logits_raw": [
      -0.840394139289856,
      1.6203656196594238,
      0.25618529319763184,
      -2.4982593059539795,
      -3.887787103652954
     ],
     "probs_raw": [
      0.06271307170391083,
      0.7346024513244629,
      0.18775710463523865,
      0.011949683539569378,
      0.0029777754098176956
     ],
     "probs": [
      0.10629419982433319,
      0.6164789795875549,
      0.23265084624290466,
      0.032522596418857574,
      0.012053357437252998
     ],
     "argmax_index": 1,
     "argmax": "lane_7",
     "top2_margin": 0.38382813334465027,
     "near_tie": false,
     "gold": "lane_7",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "normal_at_end",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 1588,
     "positions": 1588,
     "bucket": 2048,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      28519,
      779,
      8411,
      886,
      6840,
      12465,
      963,
      779,
      1764,
      803,
      779,
      3446,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      1566,
      1578
     ],
     "logits_raw": [
      0.6751614809036255,
      1.5158710479736328
     ],
     "probs_raw": [
      0.3013853430747986,
      0.6986145973205566
     ],
     "probs": [
      0.6235232949256897,
      0.3764767050743103
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.2470465898513794,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "stoppage",
     "type": "score",
     "K": 6,
     "calibrate": true,
     "temperature_key": "score:6-10",
     "T": 1.0,
     "prefix": 0,
     "n_ids": 1636,
     "positions": 1636,
     "bucket": 2048,
     "ids": [
      1,
      17,
      15991,
      23707,
      3613,
      4425,
      4857,
      561,
      33371,
      12292,
      38620,
      1141,
      61026,
      17824,
      730,
      527,
      521,
      5008,
      1985,
      2647,
      8411,
      886,
      826,
      526,
      521,
      22694,
      3446,
      843,
      1513,
      31225,
      1015,
      27464,
      535,
      2960,
      843,
      6468,
      535,
      1056,
      522,
      1536,
      535,
      1056,
      518,
      767,
      52574,
      884,
      15774,
      535,
      1267,
      868,
      2290,
      767,
      12003,
      884,
      1868,
      535,
      4582,
      1055,
      509,
      6468,
      535,
      1056,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      6135,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      4905,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      3188,
      8061,
      1015,
      6468,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      1374,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      33118,
      3375,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      531,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      527,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      530,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      532,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2142,
      535,
      4536,
      7174,
      17261,
      730,
      49574,
      886,
      889,
      529,
      41172,
      3999,
      7410,
      843,
      22500,
      730,
      525,
      523,
      5489,
      8061,
      1015,
      6468,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      534,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      530,
      12079,
      6468,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10173,
      528,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      532,
      12079,
      6468,
      535,
      2625,
      535,
      1417,
      923,
      32535,
      730,
      49883,
      837,
      527,
      1295,
      49321,
      884,
      61026,
      730,
      30664,
      2108,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      6468,
      535,
      3024,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      533,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      526,
      12079,
      6468,
      535,
      3188,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      8380,
      525,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      2108,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      11233,
      529,
      19331,
      1706,
      10012,
      536,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      6468,
      535,
      4536,
      535,
      1374,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      533,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      530,
      775,
      708,
      6468,
      535,
      4536,
      535,
      5046,
      923,
      32535,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      534,
      523,
      529,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      518,
      875,
      730,
      1374,
      775,
      708,
      6468,
      535,
      4344,
      535,
      2108,
      923,
      32535,
      730,
      43497,
      730,
      532,
      23961,
      6586,
      730,
      526,
      523,
      5370,
      796,
      5266,
      843,
      2390,
      5723,
      730,
      526,
      523,
      2346,
      796,
      5266,
      1015,
      6468,
      535,
      4613,
      535,
      3654,
      59687,
      2850,
      43497,
      730,
      532,
      16017,
      12365,
      535,
      11697,
      38993,
      578,
      27493,
      532,
      522,
      528,
      26781,
      875,
      1209,
      1399,
      730,
      528,
      775,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      59687,
      2850,
      8411,
      886,
      826,
      526,
      13525,
      968,
      6155,
      22694,
      843,
      7188,
      535,
      16017,
      43497,
      730,
      532,
      4032,
      1113,
      50027,
      23993,
      791,
      708,
      6468,
      535,
      4613,
      535,
      6468,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      60433,
      536,
      19331,
      1706,
      4908,
      884,
      24163,
      57506,
      708,
      6468,
      535,
      4613,
      535,
      2108,
      7174,
      17261,
      730,
      16753,
      28984,
      968,
      1267,
      868,
      2290,
      963,
      779,
      2669,
      26862,
      708,
      6468,
      535,
      3337,
      535,
      1386,
      7174,
      17261,
      730,
      12003,
      1868,
      9388,
      811,
      4582,
      1055,
      843,
      589,
      60088,
      730,
      526,
      1015,
      6468,
      535,
      4195,
      535,
      5843,
      7174,
      17261,
      730,
      49883,
      837,
      526,
      810,
      837,
      527,
      54126,
      843,
      2744,
      19331,
      1706,
      4032,
      2515,
      29067,
      7410,
      708,
      6468,
      535,
      3745,
      535,
      1664,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      2752,
      514,
      7614,
      708,
      6468,
      535,
      1838,
      535,
      1374,
      7174,
      17261,
      730,
      4582,
      1055,
      884,
      3475,
      963,
      43497,
      730,
      532,
      536,
      11023,
      17652,
      6944,
      811,
      43497,
      730,
      532,
      8323,
      708,
      6468,
      535,
      7058,
      535,
      3167,
      7174,
      17261,
      730,
      24163,
      57506,
      963,
      730,
      9731,
      514,
      7614,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      8557,
      811,
      1361,
      19082,
      871,
      708,
      6468,
      535,
      3878,
      535,
      898,
      7174,
      17261,
      730,
      16017,
      3360,
      4468,
      968,
      4582,
      1055,
      535,
      38169,
      8227,
      772,
      28782,
      4563,
      1490,
      779,
      43497,
      730,
      532,
      23961,
      963,
      779,
      2341,
      46129,
      708,
      6468,
      535,
      5046,
      535,
      4536,
      7174,
      17261,
      730,
      8227,
      772,
      8365,
      988,
      43497,
      730,
      532,
      536,
      46129,
      909,
      532,
      522,
      527,
      2083,
      7937,
      521,
      1295,
      5896,
      1833,
      708,
      4972,
      535,
      1056,
      535,
      3654,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      8744,
      4174,
      35036,
      536,
      32215,
      4984,
      708,
      4972,
      535,
      6135,
      535,
      1374,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      23961,
      17828,
      23848,
      536,
      1927,
      2685,
      963,
      2920,
      6586,
      730,
      525,
      523,
      530,
      796,
      5266,
      875,
      730,
      2198,
      775,
      7410,
      708,
      4972,
      535,
      6326,
      535,
      1698,
      7174,
      17261,
      730,
      11023,
      17652,
      8365,
      988,
      43497,
      730,
      532,
      8323,
      708,
      4972,
      535,
      3654,
      535,
      1664,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      2280,
      33480,
      968,
      1267,
      868,
      2290,
      536,
      878,
      4318,
      1047,
      730,
      2019,
      1750,
      730,
      1826,
      775,
      708,
      4972,
      535,
      3654,
      535,
      2999,
      7174,
      17261,
      730,
      1641,
      13053,
      19461,
      1189,
      49826,
      536,
      1641,
      13053,
      18483,
      730,
      528,
      810,
      730,
      529,
      840,
      59188,
      708,
      4972,
      535,
      6468,
      535,
      1056,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      531,
      523,
      526,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      1015,
      4972,
      535,
      4905,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4787,
      525,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      1374,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      3818,
      531,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      1386,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      4420,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      1386,
      535,
      3337,
      923,
      32535,
      730,
      49883,
      837,
      526,
      1295,
      49321,
      884,
      61026,
      730,
      32928,
      2476,
      536,
      61026,
      3567,
      811,
      16687,
      43497,
      708,
      4972,
      535,
      1133,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      6583,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2197,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      9887,
      533,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2197,
      535,
      4905,
      7174,
      17261,
      730,
      24163,
      57506,
      1892,
      4174,
      730,
      1155,
      514,
      7614,
      708,
      4972,
      535,
      1826,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10556,
      526,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      531,
      12079,
      4972,
      535,
      2142,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10520,
      527,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      525,
      12079,
      4972,
      535,
      2142,
      535,
      1664,
      7174,
      17261,
      730,
      43497,
      730,
      532,
      8323,
      7925,
      2835,
      730,
      530,
      523,
      534,
      835,
      843,
      35436,
      730,
      533,
      523,
      525,
      835,
      4032,
      23961,
      6586,
      730,
      526,
      523,
      2346,
      796,
      5266,
      708,
      4972,
      535,
      1664,
      535,
      1056,
      7174,
      17261,
      730,
      63688,
      730,
      10426,
      530,
      19331,
      1706,
      10012,
      843,
      3321,
      3600,
      38689,
      4032,
      50027,
      730,
      526,
      522,
      1374,
      5592,
      536,
      49883,
      2062,
      4002,
      730,
      2962,
      523,
      528,
      12079,
      4972,
      535,
      2625,
      535,
      1056,
      7174,
      17261,
      730,
      8501,
      6634,
      968,
      1267,
      868,
      2290,
      535,
      43497,
      730,
      532,
      23961,
      17828,
      811,
      874,
      20425,
      2030,
      963,
      730,
      1155,
      535,
      1056,
      536,
      8227,
      772,
      42806,
      811,
      874,
      5689,
      1337,
      8233,
      20842,
      708,
      4972,
      535,
      2625,
      535,
      3654,
      7174,
      17261,
      730,
      8411,
      886,
      826,
      526,
      6840,
      12465,
      536,
      1295,
      5592,
      16753,
      592,
      18,
      4023,
      1787,
      953,
      779,
      8411,
      886,
      7408,
      13525,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      1490,
      730,
      527,
      5112,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      730,
      527,
      811,
      730,
      530,
      5112,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      730,
      530,
      811,
      730,
      1155,
      5112,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      730,
      1155,
      811,
      730,
      898,
      5112,
      20,
      19,
      16,
      2119,
      730,
      529,
      535,
      730,
      898,
      811,
      730,
      2108,
      5112,
      20,
      19,
      16,
      2119,
      730,
      530,
      535,
      1391,
      730,
      2108,
      5112,
      20,
      21
     ],
     "markers": [
      1562,
      1573,
      1586,
      1599,
      1612,
      1625
     ],
     "logits_raw": [
      -6.378607273101807,
      -3.3338048458099365,
      -2.862976551055908,
      -1.375028133392334,
      0.9950432181358337,
      -0.7478825449943542
     ],
     "probs_raw": [
      0.00048148856149055064,
      0.010114093311131,
      0.01619589701294899,
      0.07171545922756195,
      0.7672231793403625,
      0.13426990807056427
     ],
     "probs": [
      0.00048148856149055064,
      0.010114093311131,
      0.01619589701294899,
      0.07171545922756195,
      0.7672231793403625,
      0.13426990807056427
     ],
     "argmax_index": 4,
     "argmax": "4",
     "top2_margin": 0.6329532712697983,
     "near_tie": false,
     "gold": "4",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "component": {
      "type": "choice",
      "choice": "lane_7",
      "confidence": 0.6164789795875549,
      "probabilities": {
       "scanner": 0.10629419982433319,
       "lane_7": 0.6164789795875549,
       "diverter": 0.23265084624290466,
       "power": 0.032522596418857574,
       "software": 0.012053357437252998
      }
     },
     "normal_at_end": {
      "type": "noul",
      "noul": 0.6235232949256897
     },
     "stoppage": {
      "type": "score",
      "score": 3.9978945227339864,
      "confidence": 0.7672231793403625,
      "probabilities": {
       "0": 0.00048148856149055064,
       "1": 0.010114093311131,
       "2": 0.01619589701294899,
       "3": 0.07171545922756195,
       "4": 0.7672231793403625,
       "5": 0.13426990807056427
      },
      "legend": {
       "0": "under 2 minutes",
       "1": "2 to 5 minutes",
       "2": "5 to 10 minutes",
       "3": "10 to 20 minutes",
       "4": "20 to 40 minutes",
       "5": "over 40 minutes"
      }
     }
    },
    "usage": {
     "input_tokens": 4830,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_L03",
   "mode": "text",
   "source": "own_long",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 5815,
   "questions": [
    {
     "qid": "fee_motion",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 1463,
     "positions": 1463,
     "bucket": 2048,
     "ids": [
      1,
      17,
      564,
      17769,
      3613,
      4425,
      12411,
      34632,
      2730,
      41891,
      516,
      31289,
      7272,
      550,
      10877,
      708,
      13635,
      3962,
      803,
      779,
      27845,
      5525,
      35590,
      4908,
      884,
      22045,
      730,
      1374,
      4690,
      730,
      1718,
      531,
      963,
      730,
      913,
      535,
      1664,
      797,
      33864,
      54898,
      2176,
      24579,
      7329,
      509,
      41678,
      535,
      730,
      3745,
      3967,
      843,
      1615,
      959,
      1143,
      13975,
      12930,
      518,
      810,
      730,
      528,
      24753,
      521,
      3136,
      1377,
      779,
      29780,
      12548,
      50804,
      777,
      521,
      2100,
      3852,
      1543,
      523,
      1030,
      779,
      12377,
      535,
      918,
      54069,
      523,
      3617,
      3962,
      535,
      9413,
      598,
      19718,
      843,
      52709,
      1126,
      1635,
      10412,
      6650,
      535,
      730,
      1632,
      3967,
      3567,
      1149,
      1514,
      6650,
      523,
      509,
      526,
      523,
      56975,
      810,
      967,
      24339,
      708,
      1098,
      12377,
      41975,
      3967,
      810,
      24753,
      810,
      14954,
      791,
      779,
      11917,
      15671,
      875,
      779,
      1517,
      803,
      779,
      5362,
      523,
      941,
      12778,
      6835,
      730,
      1664,
      3967,
      811,
      874,
      2251,
      875,
      768,
      3177,
      8324,
      811,
      874,
      967,
      777,
      931,
      523,
      3877,
      730,
      3745,
      3967,
      2251,
      521,
      779,
      8324,
      953,
      13096,
      967,
      777,
      931,
      523,
      509,
      527,
      523,
      3617,
      3962,
      803,
      779,
      2659,
      8900,
      3177,
      8324,
      708,
      1098,
      5112,
      803,
      779,
      8324,
      4908,
      884,
      730,
      1532,
      4690,
      730,
      1718,
      530,
      1166,
      12043,
      906,
      768,
      3347,
      3084,
      521,
      7885,
      968,
      9213,
      833,
      810,
      2341,
      791,
      968,
      28682,
      523,
      2479,
      1166,
      1295,
      14332,
      35517,
      896,
      1166,
      1014,
      4001,
      884,
      779,
      26168,
      523,
      509,
      528,
      523,
      21640,
      1090,
      2312,
      708,
      548,
      54069,
      17422,
      2851,
      779,
      1423,
      523,
      941,
      10039,
      9657,
      779,
      6844,
      7405,
      521,
      1144,
      9388,
      730,
      527,
      521,
      9300,
      875,
      779,
      11917,
      12562,
      3549,
      521,
      810,
      35130,
      1547,
      27297,
      45546,
      3256,
      779,
      8008,
      3509,
      521,
      1895,
      916,
      1209,
      1399,
      11583,
      20470,
      523,
      941,
      7982,
      875,
      768,
      55097,
      22212,
      884,
      8995,
      27784,
      6806,
      521,
      810,
      779,
      17677,
      1178,
      2166,
      11860,
      811,
      6715,
      1230,
      768,
      11013,
      8153,
      797,
      3354,
      523,
      49427,
      5313,
      14118,
      988,
      730,
      18833,
      811,
      730,
      22989,
      16671,
      523,
      1405,
      14954,
      791,
      779,
      15671,
      810,
      779,
      20470,
      1316,
      5514,
      779,
      38666,
      811,
      1973,
      3622,
      2759,
      3217,
      768,
      1423,
      523,
      509,
      529,
      523,
      9312,
      802,
      9233,
      1090,
      2312,
      708,
      1098,
      1230,
      9932,
      3185,
      802,
      9233,
      521,
      2345,
      586,
      521,
      6892,
      779,
      12324,
      875,
      779,
      1423,
      523,
      53813,
      953,
      730,
      531,
      521,
      21957,
      535,
      22398,
      23023,
      730,
      527,
      521,
      26547,
      521,
      779,
      6844,
      7405,
      730,
      527,
      521,
      9300,
      810,
      41634,
      730,
      526,
      521,
      30542,
      523,
      2315,
      2124,
      11761,
      953,
      730,
      529,
      521,
      25803,
      521,
      8787,
      38666,
      17236,
      843,
      526,
      521,
      21405,
      1109,
      12235,
      843,
      23617,
      1109,
      11917,
      33265,
      843,
      7537,
      518,
      810,
      7494,
      875,
      779,
      8008,
      3509,
      5158,
      6688,
      843,
      526,
      521,
      26668,
      1191,
      941,
      1423,
      8557,
      916,
      768,
      41938,
      803,
      730,
      526,
      521,
      32647,
      810,
      768,
      7493,
      8386,
      803,
      730,
      534,
      521,
      19222,
      523,
      941,
      12324,
      1320,
      1381,
      20425,
      968,
      902,
      7054,
      1924,
      5725,
      521,
      1316,
      1833,
      1295,
      4662,
      523,
      32826,
      12043,
      779,
      12324,
      968,
      768,
      1914,
      803,
      6108,
      521,
      916,
      1295,
      20402,
      2723,
      523,
      509,
      530,
      523,
      32964,
      988,
      779,
      3822,
      3892,
      708,
      26691,
      2514,
      843,
      549,
      37986,
      3271,
      779,
      2514,
      21440,
      730,
      5411,
      8267,
      35304,
      884,
      779,
      1375,
      977,
      5329,
      779,
      2254,
      810,
      50398,
      1547,
      803,
      779,
      2759,
      5854,
      20842,
      884,
      779,
      5253,
      523,
      1311,
      1499,
      1672,
      811,
      8153,
      779,
      8008,
      3509,
      875,
      21275,
      4476,
      797,
      779,
      6844,
      810,
      856,
      5400,
      875,
      1547,
      1209,
      20470,
      819,
      30193,
      2514,
      843,
      545,
      6310,
      793,
      3271,
      29504,
      779,
      6844,
      7405,
      521,
      779,
      2514,
      9657,
      768,
      30780,
      4915,
      797,
      4843,
      896,
      17322,
      1919,
      523,
      835,
      3316,
      4915,
      797,
      779,
      11917,
      856,
      13573,
      875,
      4471,
      1288,
      768,
      35563,
      1011,
      874,
      20560,
      963,
      768,
      16336,
      7264,
      819,
      35798,
      4977,
      810,
      5600,
      2514,
      843,
      549,
      831,
      833,
      3271,
      10325,
      988,
      779,
      22212,
      7982,
      521,
      779,
      2514,
      4468,
      2204,
      15073,
      11513,
      15489,
      810,
      768,
      26781,
      15761,
      884,
      13878,
      9690,
      811,
      779,
      17677,
      536,
      1547,
      803,
      779,
      15489,
      1052,
      1381,
      50398,
      523,
      509,
      531,
      523,
      7329,
      12562,
      3549,
      708,
      559,
      4182,
      4468,
      875,
      779,
      11917,
      15671,
      896,
      779,
      12562,
      12940,
      1178,
      1381,
      29878,
      963,
      730,
      1133,
      521,
      6019,
      523,
      3877,
      779,
      730,
      527,
      521,
      9300,
      988,
      779,
      6844,
      7405,
      521,
      779,
      3549,
      2166,
      12973,
      963,
      730,
      1632,
      521,
      14671,
      523,
      941,
      15671,
      1178,
      6944,
      811,
      1547,
      56302,
      8810,
      592,
      875,
      779,
      2280,
      810,
      45777,
      1149,
      13041,
      968,
      779,
      1764,
      803,
      3354,
      523,
      6162,
      4584,
      2155,
      2076,
      4526,
      1929,
      30541,
      1232,
      1000,
      797,
      779,
      12562,
      3494,
      2013,
      779,
      6844,
      521,
      810,
      902,
      8535,
      8923,
      1188,
      3999,
      779,
      12562,
      2076,
      1523,
      1336,
      10309,
      523,
      32826,
      5751,
      3984,
      779,
      10039,
      1499,
      3644,
      1523,
      3290,
      8372,
      536,
      28682,
      1845,
      896,
      779,
      15671,
      1499,
      2743,
      1892,
      811,
      768,
      3177,
      8324,
      2076,
      54526,
      1523,
      803,
      779,
      10039,
      1090,
      4636,
      523,
      3732,
      5484,
      9303,
      811,
      49904,
      768,
      32673,
      5043,
      875,
      779,
      3549,
      797,
      4963,
      521,
      810,
      2993,
      3967,
      12547,
      3501,
      811,
      1801,
      523,
      509,
      532,
      523,
      53776,
      803,
      12900,
      708,
      548,
      54069,
      11860,
      811,
      2472,
      2030,
      906,
      12377,
      810,
      953,
      840,
      5508,
      3800,
      946,
      920,
      4346,
      523,
      9413,
      598,
      19718,
      953,
      840,
      5508,
      3800,
      906,
      27246,
      946,
      920,
      4346,
      523,
      2345,
      586,
      13682,
      2208,
      906,
      3185,
      802,
      9233,
      1720,
      4250,
      1779,
      523,
      8726,
      3967,
      1166,
      33256,
      875,
      779,
      2375,
      535,
      10593,
      3102,
      810,
      15642,
      523,
      4162,
      3290,
      15450,
      988,
      2059,
      16397,
      521,
      768,
      7155,
      48506,
      953,
      4908,
      523,
      10593,
      3102,
      5068,
      730,
      2235,
      20402,
      810,
      15642,
      730,
      1443,
      521,
      916,
      730,
      527,
      20069,
      9244,
      12452,
      523,
      10593,
      3102,
      953,
      13096,
      11510,
      906,
      3185,
      802,
      9233,
      523,
      941,
      8324,
      14954,
      791,
      2345,
      586,
      6248,
      876,
      875,
      1210,
      1779,
      803,
      3714,
      523,
      509,
      533,
      523,
      41453,
      730,
      526,
      535,
      39391,
      1465,
      14122,
      884,
      779,
      7314,
      5253,
      708,
      2364,
      4346,
      968,
      1030,
      774,
      521,
      2341,
      791,
      968,
      15821,
      793,
      535,
      997,
      7464,
      779,
      10039,
      47882,
      1287,
      811,
      730,
      526,
      521,
      1086,
      988,
      1352,
      25511,
      884,
      2759,
      39391,
      1465,
      14122,
      963,
      779,
      9028,
      803,
      779,
      7314,
      5253,
      521,
      3910,
      811,
      17677,
      15001,
      2927,
      1030,
      774,
      10201,
      896,
      28285,
      5419,
      938,
      6654,
      979,
      3213,
      811,
      779,
      8746,
      1327,
      521,
      1144,
      30809,
      779,
      5799,
      810,
      12785,
      779,
      3789,
      8034,
      875,
      10415,
      824,
      5569,
      523,
      3732,
      5484,
      5751,
      3984,
      779,
      1465,
      14122,
      1499,
      8524,
      38883,
      536,
      1030,
      774,
      25723,
      896,
      1186,
      1499,
      2472,
      797,
      3160,
      803,
      779,
      15565,
      1090,
      11404,
      523,
      1019,
      2176,
      535,
      730,
      2999,
      875,
      521,
      730,
      534,
      2723,
      521,
      730,
      532,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      7818,
      523,
      509,
      534,
      523,
      41453,
      730,
      527,
      535,
      8900,
      22398,
      22819,
      708,
      2364,
      4346,
      968,
      779,
      15671,
      535,
      997,
      7464,
      779,
      8900,
      22819,
      27870,
      988,
      730,
      1374,
      811,
      730,
      1386,
      1094,
      8461,
      988,
      730,
      526,
      4471,
      730,
      1718,
      531,
      2927,
      941,
      3185,
      802,
      9233,
      59339,
      1845,
      896,
      1033,
      1423,
      1090,
      41938,
      4201,
      902,
      3761,
      27759,
      875,
      2166,
      523,
      17857,
      3967,
      1845,
      896,
      768,
      3585,
      22819,
      1499,
      3151,
      1612,
      1361,
      16671,
      521,
      4155,
      797,
      779,
      1361,
      1434,
      1872,
      884,
      13878,
      9690,
      523,
      1019,
      2176,
      535,
      730,
      1133,
      875,
      521,
      730,
      2117,
      2723,
      521,
      730,
      528,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      17602,
      521,
      810,
      779,
      22819,
      32660,
      963,
      730,
      1374,
      523,
      509,
      1155,
      523,
      41453,
      730,
      528,
      535,
      3681,
      9041,
      708,
      2364,
      4346,
      968,
      38992,
      1068,
      521,
      2341,
      791,
      968,
      43842,
      535,
      997,
      7464,
      779,
      10039,
      12881,
      768,
      3681,
      9041,
      884,
      779,
      52352,
      13646,
      5598,
      779,
      11917,
      2927,
      32826,
      7951,
      779,
      4588,
      797,
      11618,
      1203,
      5751,
      1316,
      1499,
      2144,
      1720,
      779,
      9041,
      810,
      3984,
      779,
      11917,
      15671,
      40917,
      779,
      13646,
      523,
      1311,
      953,
      11860,
      811,
      2021,
      1287,
      768,
      3822,
      2514,
      803,
      38992,
      1068,
      521,
      43842,
      810,
      15642,
      811,
      2312,
      811,
      779,
      3280,
      3177,
      8324,
      523,
      3253,
      11386,
      953,
      4281,
      523,
      509,
      1632,
      523,
      10972,
      1312,
      3436,
      708,
      13792,
      518,
      7223,
      4729,
      884,
      8995,
      27784,
      535,
      2993,
      3967,
      4468,
      11242,
      10978,
      1714,
      3537,
      779,
      5630,
      963,
      2254,
      3217,
      523,
      941,
      27246,
      1188,
      5773,
      811,
      779,
      17677,
      14065,
      875,
      779,
      3354,
      8153,
      811,
      2957,
      6586,
      1818,
      819,
      10388,
      518,
      54894,
      535,
      20470,
      938,
      4602,
      811,
      5514,
      884,
      779,
      9168,
      3392,
      803,
      779,
      7314,
      819,
      10120,
      518,
      941,
      29780,
      12548,
      50804,
      777,
      1845,
      896,
      779,
      17677,
      1090,
      8825,
      14018,
      875,
      3681,
      7092,
      21981,
      797,
      4963,
      810,
      896,
      779,
      9041,
      2724,
      33460,
      819,
      9285,
      518,
      1416,
      48053,
      535,
      779,
      17677,
      1188,
      5840,
      7980,
      23878,
      988,
      5105,
      536,
      779,
      6111,
      3378,
      1188,
      874,
      16022,
      797,
      779,
      3280,
      38666,
      819,
      11634,
      518,
      8528,
      3711,
      937,
      1888,
      535,
      9213,
      833,
      4468,
      896,
      779,
      1629,
      3711,
      937,
      1888,
      5329,
      779,
      15565,
      953,
      20425,
      797,
      6060,
      810,
      896,
      1352,
      46967,
      2716,
      1046,
      797,
      5277,
      536,
      779,
      15671,
      1188,
      2565,
      8297,
      1667,
      523,
      509,
      1374,
      523,
      13887,
      803,
      779,
      3280,
      8324,
      708,
      1098,
      3280,
      3177,
      8324,
      1188,
      874,
      4908,
      797,
      4526,
      536,
      779,
      5562,
      1188,
      874,
      9139,
      797,
      779,
      6844,
      38666,
      523,
      941,
      8324,
      8557,
      963,
      730,
      2197,
      535,
      1155,
      523,
      18,
      3493,
      9862,
      811,
      779,
      9557,
      884,
      779,
      8900,
      22819,
      540,
      19,
      16,
      12043,
      20,
      19,
      16,
      20691,
      20,
      19,
      16,
      825,
      25745,
      535,
      6953,
      2939,
      791,
      811,
      768,
      2919,
      8324,
      933,
      768,
      3822,
      2514,
      20,
      19,
      16,
      47927,
      535,
      3877,
      577,
      29058,
      2076,
      779,
      11386,
      20,
      21
     ],
     "markers": [
      1427,
      1431,
      1435,
      1452
     ],
     "logits_raw": [
      -1.615821123123169,
      -2.7980968952178955,
      0.5164243578910828,
      -2.294139862060547
     ],
     "probs_raw": [
      0.09758160263299942,
      0.029916591942310333,
      0.8229821920394897,
      0.04951968789100647
     ],
     "probs": [
      0.15077616274356842,
      0.0647965595126152,
      0.6915529370307922,
      0.09287437796592712
     ],
     "argmax_index": 2,
     "argmax": "deferred",
     "top2_margin": 0.5407767742872238,
     "near_tie": false,
     "gold": "rejected",
     "correct": false,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 4
     }
    },
    {
     "qid": "quorate",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 1446,
     "positions": 1446,
     "bucket": 2048,
     "ids": [
      1,
      17,
      564,
      17769,
      3613,
      4425,
      12411,
      34632,
      2730,
      41891,
      516,
      31289,
      7272,
      550,
      10877,
      708,
      13635,
      3962,
      803,
      779,
      27845,
      5525,
      35590,
      4908,
      884,
      22045,
      730,
      1374,
      4690,
      730,
      1718,
      531,
      963,
      730,
      913,
      535,
      1664,
      797,
      33864,
      54898,
      2176,
      24579,
      7329,
      509,
      41678,
      535,
      730,
      3745,
      3967,
      843,
      1615,
      959,
      1143,
      13975,
      12930,
      518,
      810,
      730,
      528,
      24753,
      521,
      3136,
      1377,
      779,
      29780,
      12548,
      50804,
      777,
      521,
      2100,
      3852,
      1543,
      523,
      1030,
      779,
      12377,
      535,
      918,
      54069,
      523,
      3617,
      3962,
      535,
      9413,
      598,
      19718,
      843,
      52709,
      1126,
      1635,
      10412,
      6650,
      535,
      730,
      1632,
      3967,
      3567,
      1149,
      1514,
      6650,
      523,
      509,
      526,
      523,
      56975,
      810,
      967,
      24339,
      708,
      1098,
      12377,
      41975,
      3967,
      810,
      24753,
      810,
      14954,
      791,
      779,
      11917,
      15671,
      875,
      779,
      1517,
      803,
      779,
      5362,
      523,
      941,
      12778,
      6835,
      730,
      1664,
      3967,
      811,
      874,
      2251,
      875,
      768,
      3177,
      8324,
      811,
      874,
      967,
      777,
      931,
      523,
      3877,
      730,
      3745,
      3967,
      2251,
      521,
      779,
      8324,
      953,
      13096,
      967,
      777,
      931,
      523,
      509,
      527,
      523,
      3617,
      3962,
      803,
      779,
      2659,
      8900,
      3177,
      8324,
      708,
      1098,
      5112,
      803,
      779,
      8324,
      4908,
      884,
      730,
      1532,
      4690,
      730,
      1718,
      530,
      1166,
      12043,
      906,
      768,
      3347,
      3084,
      521,
      7885,
      968,
      9213,
      833,
      810,
      2341,
      791,
      968,
      28682,
      523,
      2479,
      1166,
      1295,
      14332,
      35517,
      896,
      1166,
      1014,
      4001,
      884,
      779,
      26168,
      523,
      509,
      528,
      523,
      21640,
      1090,
      2312,
      708,
      548,
      54069,
      17422,
      2851,
      779,
      1423,
      523,
      941,
      10039,
      9657,
      779,
      6844,
      7405,
      521,
      1144,
      9388,
      730,
      527,
      521,
      9300,
      875,
      779,
      11917,
      12562,
      3549,
      521,
      810,
      35130,
      1547,
      27297,
      45546,
      3256,
      779,
      8008,
      3509,
      521,
      1895,
      916,
      1209,
      1399,
      11583,
      20470,
      523,
      941,
      7982,
      875,
      768,
      55097,
      22212,
      884,
      8995,
      27784,
      6806,
      521,
      810,
      779,
      17677,
      1178,
      2166,
      11860,
      811,
      6715,
      1230,
      768,
      11013,
      8153,
      797,
      3354,
      523,
      49427,
      5313,
      14118,
      988,
      730,
      18833,
      811,
      730,
      22989,
      16671,
      523,
      1405,
      14954,
      791,
      779,
      15671,
      810,
      779,
      20470,
      1316,
      5514,
      779,
      38666,
      811,
      1973,
      3622,
      2759,
      3217,
      768,
      1423,
      523,
      509,
      529,
      523,
      9312,
      802,
      9233,
      1090,
      2312,
      708,
      1098,
      1230,
      9932,
      3185,
      802,
      9233,
      521,
      2345,
      586,
      521,
      6892,
      779,
      12324,
      875,
      779,
      1423,
      523,
      53813,
      953,
      730,
      531,
      521,
      21957,
      535,
      22398,
      23023,
      730,
      527,
      521,
      26547,
      521,
      779,
      6844,
      7405,
      730,
      527,
      521,
      9300,
      810,
      41634,
      730,
      526,
      521,
      30542,
      523,
      2315,
      2124,
      11761,
      953,
      730,
      529,
      521,
      25803,
      521,
      8787,
      38666,
      17236,
      843,
      526,
      521,
      21405,
      1109,
      12235,
      843,
      23617,
      1109,
      11917,
      33265,
      843,
      7537,
      518,
      810,
      7494,
      875,
      779,
      8008,
      3509,
      5158,
      6688,
      843,
      526,
      521,
      26668,
      1191,
      941,
      1423,
      8557,
      916,
      768,
      41938,
      803,
      730,
      526,
      521,
      32647,
      810,
      768,
      7493,
      8386,
      803,
      730,
      534,
      521,
      19222,
      523,
      941,
      12324,
      1320,
      1381,
      20425,
      968,
      902,
      7054,
      1924,
      5725,
      521,
      1316,
      1833,
      1295,
      4662,
      523,
      32826,
      12043,
      779,
      12324,
      968,
      768,
      1914,
      803,
      6108,
      521,
      916,
      1295,
      20402,
      2723,
      523,
      509,
      530,
      523,
      32964,
      988,
      779,
      3822,
      3892,
      708,
      26691,
      2514,
      843,
      549,
      37986,
      3271,
      779,
      2514,
      21440,
      730,
      5411,
      8267,
      35304,
      884,
      779,
      1375,
      977,
      5329,
      779,
      2254,
      810,
      50398,
      1547,
      803,
      779,
      2759,
      5854,
      20842,
      884,
      779,
      5253,
      523,
      1311,
      1499,
      1672,
      811,
      8153,
      779,
      8008,
      3509,
      875,
      21275,
      4476,
      797,
      779,
      6844,
      810,
      856,
      5400,
      875,
      1547,
      1209,
      20470,
      819,
      30193,
      2514,
      843,
      545,
      6310,
      793,
      3271,
      29504,
      779,
      6844,
      7405,
      521,
      779,
      2514,
      9657,
      768,
      30780,
      4915,
      797,
      4843,
      896,
      17322,
      1919,
      523,
      835,
      3316,
      4915,
      797,
      779,
      11917,
      856,
      13573,
      875,
      4471,
      1288,
      768,
      35563,
      1011,
      874,
      20560,
      963,
      768,
      16336,
      7264,
      819,
      35798,
      4977,
      810,
      5600,
      2514,
      843,
      549,
      831,
      833,
      3271,
      10325,
      988,
      779,
      22212,
      7982,
      521,
      779,
      2514,
      4468,
      2204,
      15073,
      11513,
      15489,
      810,
      768,
      26781,
      15761,
      884,
      13878,
      9690,
      811,
      779,
      17677,
      536,
      1547,
      803,
      779,
      15489,
      1052,
      1381,
      50398,
      523,
      509,
      531,
      523,
      7329,
      12562,
      3549,
      708,
      559,
      4182,
      4468,
      875,
      779,
      11917,
      15671,
      896,
      779,
      12562,
      12940,
      1178,
      1381,
      29878,
      963,
      730,
      1133,
      521,
      6019,
      523,
      3877,
      779,
      730,
      527,
      521,
      9300,
      988,
      779,
      6844,
      7405,
      521,
      779,
      3549,
      2166,
      12973,
      963,
      730,
      1632,
      521,
      14671,
      523,
      941,
      15671,
      1178,
      6944,
      811,
      1547,
      56302,
      8810,
      592,
      875,
      779,
      2280,
      810,
      45777,
      1149,
      13041,
      968,
      779,
      1764,
      803,
      3354,
      523,
      6162,
      4584,
      2155,
      2076,
      4526,
      1929,
      30541,
      1232,
      1000,
      797,
      779,
      12562,
      3494,
      2013,
      779,
      6844,
      521,
      810,
      902,
      8535,
      8923,
      1188,
      3999,
      779,
      12562,
      2076,
      1523,
      1336,
      10309,
      523,
      32826,
      5751,
      3984,
      779,
      10039,
      1499,
      3644,
      1523,
      3290,
      8372,
      536,
      28682,
      1845,
      896,
      779,
      15671,
      1499,
      2743,
      1892,
      811,
      768,
      3177,
      8324,
      2076,
      54526,
      1523,
      803,
      779,
      10039,
      1090,
      4636,
      523,
      3732,
      5484,
      9303,
      811,
      49904,
      768,
      32673,
      5043,
      875,
      779,
      3549,
      797,
      4963,
      521,
      810,
      2993,
      3967,
      12547,
      3501,
      811,
      1801,
      523,
      509,
      532,
      523,
      53776,
      803,
      12900,
      708,
      548,
      54069,
      11860,
      811,
      2472,
      2030,
      906,
      12377,
      810,
      953,
      840,
      5508,
      3800,
      946,
      920,
      4346,
      523,
      9413,
      598,
      19718,
      953,
      840,
      5508,
      3800,
      906,
      27246,
      946,
      920,
      4346,
      523,
      2345,
      586,
      13682,
      2208,
      906,
      3185,
      802,
      9233,
      1720,
      4250,
      1779,
      523,
      8726,
      3967,
      1166,
      33256,
      875,
      779,
      2375,
      535,
      10593,
      3102,
      810,
      15642,
      523,
      4162,
      3290,
      15450,
      988,
      2059,
      16397,
      521,
      768,
      7155,
      48506,
      953,
      4908,
      523,
      10593,
      3102,
      5068,
      730,
      2235,
      20402,
      810,
      15642,
      730,
      1443,
      521,
      916,
      730,
      527,
      20069,
      9244,
      12452,
      523,
      10593,
      3102,
      953,
      13096,
      11510,
      906,
      3185,
      802,
      9233,
      523,
      941,
      8324,
      14954,
      791,
      2345,
      586,
      6248,
      876,
      875,
      1210,
      1779,
      803,
      3714,
      523,
      509,
      533,
      523,
      41453,
      730,
      526,
      535,
      39391,
      1465,
      14122,
      884,
      779,
      7314,
      5253,
      708,
      2364,
      4346,
      968,
      1030,
      774,
      521,
      2341,
      791,
      968,
      15821,
      793,
      535,
      997,
      7464,
      779,
      10039,
      47882,
      1287,
      811,
      730,
      526,
      521,
      1086,
      988,
      1352,
      25511,
      884,
      2759,
      39391,
      1465,
      14122,
      963,
      779,
      9028,
      803,
      779,
      7314,
      5253,
      521,
      3910,
      811,
      17677,
      15001,
      2927,
      1030,
      774,
      10201,
      896,
      28285,
      5419,
      938,
      6654,
      979,
      3213,
      811,
      779,
      8746,
      1327,
      521,
      1144,
      30809,
      779,
      5799,
      810,
      12785,
      779,
      3789,
      8034,
      875,
      10415,
      824,
      5569,
      523,
      3732,
      5484,
      5751,
      3984,
      779,
      1465,
      14122,
      1499,
      8524,
      38883,
      536,
      1030,
      774,
      25723,
      896,
      1186,
      1499,
      2472,
      797,
      3160,
      803,
      779,
      15565,
      1090,
      11404,
      523,
      1019,
      2176,
      535,
      730,
      2999,
      875,
      521,
      730,
      534,
      2723,
      521,
      730,
      532,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      7818,
      523,
      509,
      534,
      523,
      41453,
      730,
      527,
      535,
      8900,
      22398,
      22819,
      708,
      2364,
      4346,
      968,
      779,
      15671,
      535,
      997,
      7464,
      779,
      8900,
      22819,
      27870,
      988,
      730,
      1374,
      811,
      730,
      1386,
      1094,
      8461,
      988,
      730,
      526,
      4471,
      730,
      1718,
      531,
      2927,
      941,
      3185,
      802,
      9233,
      59339,
      1845,
      896,
      1033,
      1423,
      1090,
      41938,
      4201,
      902,
      3761,
      27759,
      875,
      2166,
      523,
      17857,
      3967,
      1845,
      896,
      768,
      3585,
      22819,
      1499,
      3151,
      1612,
      1361,
      16671,
      521,
      4155,
      797,
      779,
      1361,
      1434,
      1872,
      884,
      13878,
      9690,
      523,
      1019,
      2176,
      535,
      730,
      1133,
      875,
      521,
      730,
      2117,
      2723,
      521,
      730,
      528,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      17602,
      521,
      810,
      779,
      22819,
      32660,
      963,
      730,
      1374,
      523,
      509,
      1155,
      523,
      41453,
      730,
      528,
      535,
      3681,
      9041,
      708,
      2364,
      4346,
      968,
      38992,
      1068,
      521,
      2341,
      791,
      968,
      43842,
      535,
      997,
      7464,
      779,
      10039,
      12881,
      768,
      3681,
      9041,
      884,
      779,
      52352,
      13646,
      5598,
      779,
      11917,
      2927,
      32826,
      7951,
      779,
      4588,
      797,
      11618,
      1203,
      5751,
      1316,
      1499,
      2144,
      1720,
      779,
      9041,
      810,
      3984,
      779,
      11917,
      15671,
      40917,
      779,
      13646,
      523,
      1311,
      953,
      11860,
      811,
      2021,
      1287,
      768,
      3822,
      2514,
      803,
      38992,
      1068,
      521,
      43842,
      810,
      15642,
      811,
      2312,
      811,
      779,
      3280,
      3177,
      8324,
      523,
      3253,
      11386,
      953,
      4281,
      523,
      509,
      1632,
      523,
      10972,
      1312,
      3436,
      708,
      13792,
      518,
      7223,
      4729,
      884,
      8995,
      27784,
      535,
      2993,
      3967,
      4468,
      11242,
      10978,
      1714,
      3537,
      779,
      5630,
      963,
      2254,
      3217,
      523,
      941,
      27246,
      1188,
      5773,
      811,
      779,
      17677,
      14065,
      875,
      779,
      3354,
      8153,
      811,
      2957,
      6586,
      1818,
      819,
      10388,
      518,
      54894,
      535,
      20470,
      938,
      4602,
      811,
      5514,
      884,
      779,
      9168,
      3392,
      803,
      779,
      7314,
      819,
      10120,
      518,
      941,
      29780,
      12548,
      50804,
      777,
      1845,
      896,
      779,
      17677,
      1090,
      8825,
      14018,
      875,
      3681,
      7092,
      21981,
      797,
      4963,
      810,
      896,
      779,
      9041,
      2724,
      33460,
      819,
      9285,
      518,
      1416,
      48053,
      535,
      779,
      17677,
      1188,
      5840,
      7980,
      23878,
      988,
      5105,
      536,
      779,
      6111,
      3378,
      1188,
      874,
      16022,
      797,
      779,
      3280,
      38666,
      819,
      11634,
      518,
      8528,
      3711,
      937,
      1888,
      535,
      9213,
      833,
      4468,
      896,
      779,
      1629,
      3711,
      937,
      1888,
      5329,
      779,
      15565,
      953,
      20425,
      797,
      6060,
      810,
      896,
      1352,
      46967,
      2716,
      1046,
      797,
      5277,
      536,
      779,
      15671,
      1188,
      2565,
      8297,
      1667,
      523,
      509,
      1374,
      523,
      13887,
      803,
      779,
      3280,
      8324,
      708,
      1098,
      3280,
      3177,
      8324,
      1188,
      874,
      4908,
      797,
      4526,
      536,
      779,
      5562,
      1188,
      874,
      9139,
      797,
      779,
      6844,
      38666,
      523,
      941,
      8324,
      8557,
      963,
      730,
      2197,
      535,
      1155,
      523,
      18,
      28519,
      779,
      8324,
      967,
      777,
      931,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      1424,
      1436
     ],
     "logits_raw": [
      0.2696681618690491,
      5.161313533782959
     ],
     "probs_raw": [
      0.007453090511262417,
      0.9925468564033508
     ],
     "probs": [
      0.9495782256126404,
      0.0504218228161335
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8991564027965069,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 4
     }
    },
    {
     "qid": "treasurer",
     "type": "choice",
     "K": 2,
     "calibrate": true,
     "temperature_key": "choice:2",
     "T": 1.7465145587921143,
     "prefix": 0,
     "n_ids": 1440,
     "positions": 1440,
     "bucket": 2048,
     "ids": [
      1,
      17,
      564,
      17769,
      3613,
      4425,
      12411,
      34632,
      2730,
      41891,
      516,
      31289,
      7272,
      550,
      10877,
      708,
      13635,
      3962,
      803,
      779,
      27845,
      5525,
      35590,
      4908,
      884,
      22045,
      730,
      1374,
      4690,
      730,
      1718,
      531,
      963,
      730,
      913,
      535,
      1664,
      797,
      33864,
      54898,
      2176,
      24579,
      7329,
      509,
      41678,
      535,
      730,
      3745,
      3967,
      843,
      1615,
      959,
      1143,
      13975,
      12930,
      518,
      810,
      730,
      528,
      24753,
      521,
      3136,
      1377,
      779,
      29780,
      12548,
      50804,
      777,
      521,
      2100,
      3852,
      1543,
      523,
      1030,
      779,
      12377,
      535,
      918,
      54069,
      523,
      3617,
      3962,
      535,
      9413,
      598,
      19718,
      843,
      52709,
      1126,
      1635,
      10412,
      6650,
      535,
      730,
      1632,
      3967,
      3567,
      1149,
      1514,
      6650,
      523,
      509,
      526,
      523,
      56975,
      810,
      967,
      24339,
      708,
      1098,
      12377,
      41975,
      3967,
      810,
      24753,
      810,
      14954,
      791,
      779,
      11917,
      15671,
      875,
      779,
      1517,
      803,
      779,
      5362,
      523,
      941,
      12778,
      6835,
      730,
      1664,
      3967,
      811,
      874,
      2251,
      875,
      768,
      3177,
      8324,
      811,
      874,
      967,
      777,
      931,
      523,
      3877,
      730,
      3745,
      3967,
      2251,
      521,
      779,
      8324,
      953,
      13096,
      967,
      777,
      931,
      523,
      509,
      527,
      523,
      3617,
      3962,
      803,
      779,
      2659,
      8900,
      3177,
      8324,
      708,
      1098,
      5112,
      803,
      779,
      8324,
      4908,
      884,
      730,
      1532,
      4690,
      730,
      1718,
      530,
      1166,
      12043,
      906,
      768,
      3347,
      3084,
      521,
      7885,
      968,
      9213,
      833,
      810,
      2341,
      791,
      968,
      28682,
      523,
      2479,
      1166,
      1295,
      14332,
      35517,
      896,
      1166,
      1014,
      4001,
      884,
      779,
      26168,
      523,
      509,
      528,
      523,
      21640,
      1090,
      2312,
      708,
      548,
      54069,
      17422,
      2851,
      779,
      1423,
      523,
      941,
      10039,
      9657,
      779,
      6844,
      7405,
      521,
      1144,
      9388,
      730,
      527,
      521,
      9300,
      875,
      779,
      11917,
      12562,
      3549,
      521,
      810,
      35130,
      1547,
      27297,
      45546,
      3256,
      779,
      8008,
      3509,
      521,
      1895,
      916,
      1209,
      1399,
      11583,
      20470,
      523,
      941,
      7982,
      875,
      768,
      55097,
      22212,
      884,
      8995,
      27784,
      6806,
      521,
      810,
      779,
      17677,
      1178,
      2166,
      11860,
      811,
      6715,
      1230,
      768,
      11013,
      8153,
      797,
      3354,
      523,
      49427,
      5313,
      14118,
      988,
      730,
      18833,
      811,
      730,
      22989,
      16671,
      523,
      1405,
      14954,
      791,
      779,
      15671,
      810,
      779,
      20470,
      1316,
      5514,
      779,
      38666,
      811,
      1973,
      3622,
      2759,
      3217,
      768,
      1423,
      523,
      509,
      529,
      523,
      9312,
      802,
      9233,
      1090,
      2312,
      708,
      1098,
      1230,
      9932,
      3185,
      802,
      9233,
      521,
      2345,
      586,
      521,
      6892,
      779,
      12324,
      875,
      779,
      1423,
      523,
      53813,
      953,
      730,
      531,
      521,
      21957,
      535,
      22398,
      23023,
      730,
      527,
      521,
      26547,
      521,
      779,
      6844,
      7405,
      730,
      527,
      521,
      9300,
      810,
      41634,
      730,
      526,
      521,
      30542,
      523,
      2315,
      2124,
      11761,
      953,
      730,
      529,
      521,
      25803,
      521,
      8787,
      38666,
      17236,
      843,
      526,
      521,
      21405,
      1109,
      12235,
      843,
      23617,
      1109,
      11917,
      33265,
      843,
      7537,
      518,
      810,
      7494,
      875,
      779,
      8008,
      3509,
      5158,
      6688,
      843,
      526,
      521,
      26668,
      1191,
      941,
      1423,
      8557,
      916,
      768,
      41938,
      803,
      730,
      526,
      521,
      32647,
      810,
      768,
      7493,
      8386,
      803,
      730,
      534,
      521,
      19222,
      523,
      941,
      12324,
      1320,
      1381,
      20425,
      968,
      902,
      7054,
      1924,
      5725,
      521,
      1316,
      1833,
      1295,
      4662,
      523,
      32826,
      12043,
      779,
      12324,
      968,
      768,
      1914,
      803,
      6108,
      521,
      916,
      1295,
      20402,
      2723,
      523,
      509,
      530,
      523,
      32964,
      988,
      779,
      3822,
      3892,
      708,
      26691,
      2514,
      843,
      549,
      37986,
      3271,
      779,
      2514,
      21440,
      730,
      5411,
      8267,
      35304,
      884,
      779,
      1375,
      977,
      5329,
      779,
      2254,
      810,
      50398,
      1547,
      803,
      779,
      2759,
      5854,
      20842,
      884,
      779,
      5253,
      523,
      1311,
      1499,
      1672,
      811,
      8153,
      779,
      8008,
      3509,
      875,
      21275,
      4476,
      797,
      779,
      6844,
      810,
      856,
      5400,
      875,
      1547,
      1209,
      20470,
      819,
      30193,
      2514,
      843,
      545,
      6310,
      793,
      3271,
      29504,
      779,
      6844,
      7405,
      521,
      779,
      2514,
      9657,
      768,
      30780,
      4915,
      797,
      4843,
      896,
      17322,
      1919,
      523,
      835,
      3316,
      4915,
      797,
      779,
      11917,
      856,
      13573,
      875,
      4471,
      1288,
      768,
      35563,
      1011,
      874,
      20560,
      963,
      768,
      16336,
      7264,
      819,
      35798,
      4977,
      810,
      5600,
      2514,
      843,
      549,
      831,
      833,
      3271,
      10325,
      988,
      779,
      22212,
      7982,
      521,
      779,
      2514,
      4468,
      2204,
      15073,
      11513,
      15489,
      810,
      768,
      26781,
      15761,
      884,
      13878,
      9690,
      811,
      779,
      17677,
      536,
      1547,
      803,
      779,
      15489,
      1052,
      1381,
      50398,
      523,
      509,
      531,
      523,
      7329,
      12562,
      3549,
      708,
      559,
      4182,
      4468,
      875,
      779,
      11917,
      15671,
      896,
      779,
      12562,
      12940,
      1178,
      1381,
      29878,
      963,
      730,
      1133,
      521,
      6019,
      523,
      3877,
      779,
      730,
      527,
      521,
      9300,
      988,
      779,
      6844,
      7405,
      521,
      779,
      3549,
      2166,
      12973,
      963,
      730,
      1632,
      521,
      14671,
      523,
      941,
      15671,
      1178,
      6944,
      811,
      1547,
      56302,
      8810,
      592,
      875,
      779,
      2280,
      810,
      45777,
      1149,
      13041,
      968,
      779,
      1764,
      803,
      3354,
      523,
      6162,
      4584,
      2155,
      2076,
      4526,
      1929,
      30541,
      1232,
      1000,
      797,
      779,
      12562,
      3494,
      2013,
      779,
      6844,
      521,
      810,
      902,
      8535,
      8923,
      1188,
      3999,
      779,
      12562,
      2076,
      1523,
      1336,
      10309,
      523,
      32826,
      5751,
      3984,
      779,
      10039,
      1499,
      3644,
      1523,
      3290,
      8372,
      536,
      28682,
      1845,
      896,
      779,
      15671,
      1499,
      2743,
      1892,
      811,
      768,
      3177,
      8324,
      2076,
      54526,
      1523,
      803,
      779,
      10039,
      1090,
      4636,
      523,
      3732,
      5484,
      9303,
      811,
      49904,
      768,
      32673,
      5043,
      875,
      779,
      3549,
      797,
      4963,
      521,
      810,
      2993,
      3967,
      12547,
      3501,
      811,
      1801,
      523,
      509,
      532,
      523,
      53776,
      803,
      12900,
      708,
      548,
      54069,
      11860,
      811,
      2472,
      2030,
      906,
      12377,
      810,
      953,
      840,
      5508,
      3800,
      946,
      920,
      4346,
      523,
      9413,
      598,
      19718,
      953,
      840,
      5508,
      3800,
      906,
      27246,
      946,
      920,
      4346,
      523,
      2345,
      586,
      13682,
      2208,
      906,
      3185,
      802,
      9233,
      1720,
      4250,
      1779,
      523,
      8726,
      3967,
      1166,
      33256,
      875,
      779,
      2375,
      535,
      10593,
      3102,
      810,
      15642,
      523,
      4162,
      3290,
      15450,
      988,
      2059,
      16397,
      521,
      768,
      7155,
      48506,
      953,
      4908,
      523,
      10593,
      3102,
      5068,
      730,
      2235,
      20402,
      810,
      15642,
      730,
      1443,
      521,
      916,
      730,
      527,
      20069,
      9244,
      12452,
      523,
      10593,
      3102,
      953,
      13096,
      11510,
      906,
      3185,
      802,
      9233,
      523,
      941,
      8324,
      14954,
      791,
      2345,
      586,
      6248,
      876,
      875,
      1210,
      1779,
      803,
      3714,
      523,
      509,
      533,
      523,
      41453,
      730,
      526,
      535,
      39391,
      1465,
      14122,
      884,
      779,
      7314,
      5253,
      708,
      2364,
      4346,
      968,
      1030,
      774,
      521,
      2341,
      791,
      968,
      15821,
      793,
      535,
      997,
      7464,
      779,
      10039,
      47882,
      1287,
      811,
      730,
      526,
      521,
      1086,
      988,
      1352,
      25511,
      884,
      2759,
      39391,
      1465,
      14122,
      963,
      779,
      9028,
      803,
      779,
      7314,
      5253,
      521,
      3910,
      811,
      17677,
      15001,
      2927,
      1030,
      774,
      10201,
      896,
      28285,
      5419,
      938,
      6654,
      979,
      3213,
      811,
      779,
      8746,
      1327,
      521,
      1144,
      30809,
      779,
      5799,
      810,
      12785,
      779,
      3789,
      8034,
      875,
      10415,
      824,
      5569,
      523,
      3732,
      5484,
      5751,
      3984,
      779,
      1465,
      14122,
      1499,
      8524,
      38883,
      536,
      1030,
      774,
      25723,
      896,
      1186,
      1499,
      2472,
      797,
      3160,
      803,
      779,
      15565,
      1090,
      11404,
      523,
      1019,
      2176,
      535,
      730,
      2999,
      875,
      521,
      730,
      534,
      2723,
      521,
      730,
      532,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      7818,
      523,
      509,
      534,
      523,
      41453,
      730,
      527,
      535,
      8900,
      22398,
      22819,
      708,
      2364,
      4346,
      968,
      779,
      15671,
      535,
      997,
      7464,
      779,
      8900,
      22819,
      27870,
      988,
      730,
      1374,
      811,
      730,
      1386,
      1094,
      8461,
      988,
      730,
      526,
      4471,
      730,
      1718,
      531,
      2927,
      941,
      3185,
      802,
      9233,
      59339,
      1845,
      896,
      1033,
      1423,
      1090,
      41938,
      4201,
      902,
      3761,
      27759,
      875,
      2166,
      523,
      17857,
      3967,
      1845,
      896,
      768,
      3585,
      22819,
      1499,
      3151,
      1612,
      1361,
      16671,
      521,
      4155,
      797,
      779,
      1361,
      1434,
      1872,
      884,
      13878,
      9690,
      523,
      1019,
      2176,
      535,
      730,
      1133,
      875,
      521,
      730,
      2117,
      2723,
      521,
      730,
      528,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      17602,
      521,
      810,
      779,
      22819,
      32660,
      963,
      730,
      1374,
      523,
      509,
      1155,
      523,
      41453,
      730,
      528,
      535,
      3681,
      9041,
      708,
      2364,
      4346,
      968,
      38992,
      1068,
      521,
      2341,
      791,
      968,
      43842,
      535,
      997,
      7464,
      779,
      10039,
      12881,
      768,
      3681,
      9041,
      884,
      779,
      52352,
      13646,
      5598,
      779,
      11917,
      2927,
      32826,
      7951,
      779,
      4588,
      797,
      11618,
      1203,
      5751,
      1316,
      1499,
      2144,
      1720,
      779,
      9041,
      810,
      3984,
      779,
      11917,
      15671,
      40917,
      779,
      13646,
      523,
      1311,
      953,
      11860,
      811,
      2021,
      1287,
      768,
      3822,
      2514,
      803,
      38992,
      1068,
      521,
      43842,
      810,
      15642,
      811,
      2312,
      811,
      779,
      3280,
      3177,
      8324,
      523,
      3253,
      11386,
      953,
      4281,
      523,
      509,
      1632,
      523,
      10972,
      1312,
      3436,
      708,
      13792,
      518,
      7223,
      4729,
      884,
      8995,
      27784,
      535,
      2993,
      3967,
      4468,
      11242,
      10978,
      1714,
      3537,
      779,
      5630,
      963,
      2254,
      3217,
      523,
      941,
      27246,
      1188,
      5773,
      811,
      779,
      17677,
      14065,
      875,
      779,
      3354,
      8153,
      811,
      2957,
      6586,
      1818,
      819,
      10388,
      518,
      54894,
      535,
      20470,
      938,
      4602,
      811,
      5514,
      884,
      779,
      9168,
      3392,
      803,
      779,
      7314,
      819,
      10120,
      518,
      941,
      29780,
      12548,
      50804,
      777,
      1845,
      896,
      779,
      17677,
      1090,
      8825,
      14018,
      875,
      3681,
      7092,
      21981,
      797,
      4963,
      810,
      896,
      779,
      9041,
      2724,
      33460,
      819,
      9285,
      518,
      1416,
      48053,
      535,
      779,
      17677,
      1188,
      5840,
      7980,
      23878,
      988,
      5105,
      536,
      779,
      6111,
      3378,
      1188,
      874,
      16022,
      797,
      779,
      3280,
      38666,
      819,
      11634,
      518,
      8528,
      3711,
      937,
      1888,
      535,
      9213,
      833,
      4468,
      896,
      779,
      1629,
      3711,
      937,
      1888,
      5329,
      779,
      15565,
      953,
      20425,
      797,
      6060,
      810,
      896,
      1352,
      46967,
      2716,
      1046,
      797,
      5277,
      536,
      779,
      15671,
      1188,
      2565,
      8297,
      1667,
      523,
      509,
      1374,
      523,
      13887,
      803,
      779,
      3280,
      8324,
      708,
      1098,
      3280,
      3177,
      8324,
      1188,
      874,
      4908,
      797,
      4526,
      536,
      779,
      5562,
      1188,
      874,
      9139,
      797,
      779,
      6844,
      38666,
      523,
      941,
      8324,
      8557,
      963,
      730,
      2197,
      535,
      1155,
      523,
      18,
      17408,
      953,
      11510,
      3185,
      802,
      9233,
      540,
      19,
      16,
      3352,
      3102,
      535,
      10593,
      3102,
      20,
      19,
      16,
      945,
      596,
      783,
      535,
      15642,
      20,
      21
     ],
     "markers": [
      1424,
      1432
     ],
     "logits_raw": [
      -0.2319762408733368,
      -2.207134246826172
     ],
     "probs_raw": [
      0.8781641125679016,
      0.12183595448732376
     ],
     "probs": [
      0.7560074925422668,
      0.24399243295192719
     ],
     "argmax_index": 0,
     "argmax": "odette",
     "top2_margin": 0.5120150595903397,
     "near_tie": false,
     "gold": "odette",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 3,
      "B": 4
     }
    },
    {
     "qid": "racks_support",
     "type": "score",
     "K": 4,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 1466,
     "positions": 1466,
     "bucket": 2048,
     "ids": [
      1,
      17,
      564,
      17769,
      3613,
      4425,
      12411,
      34632,
      2730,
      41891,
      516,
      31289,
      7272,
      550,
      10877,
      708,
      13635,
      3962,
      803,
      779,
      27845,
      5525,
      35590,
      4908,
      884,
      22045,
      730,
      1374,
      4690,
      730,
      1718,
      531,
      963,
      730,
      913,
      535,
      1664,
      797,
      33864,
      54898,
      2176,
      24579,
      7329,
      509,
      41678,
      535,
      730,
      3745,
      3967,
      843,
      1615,
      959,
      1143,
      13975,
      12930,
      518,
      810,
      730,
      528,
      24753,
      521,
      3136,
      1377,
      779,
      29780,
      12548,
      50804,
      777,
      521,
      2100,
      3852,
      1543,
      523,
      1030,
      779,
      12377,
      535,
      918,
      54069,
      523,
      3617,
      3962,
      535,
      9413,
      598,
      19718,
      843,
      52709,
      1126,
      1635,
      10412,
      6650,
      535,
      730,
      1632,
      3967,
      3567,
      1149,
      1514,
      6650,
      523,
      509,
      526,
      523,
      56975,
      810,
      967,
      24339,
      708,
      1098,
      12377,
      41975,
      3967,
      810,
      24753,
      810,
      14954,
      791,
      779,
      11917,
      15671,
      875,
      779,
      1517,
      803,
      779,
      5362,
      523,
      941,
      12778,
      6835,
      730,
      1664,
      3967,
      811,
      874,
      2251,
      875,
      768,
      3177,
      8324,
      811,
      874,
      967,
      777,
      931,
      523,
      3877,
      730,
      3745,
      3967,
      2251,
      521,
      779,
      8324,
      953,
      13096,
      967,
      777,
      931,
      523,
      509,
      527,
      523,
      3617,
      3962,
      803,
      779,
      2659,
      8900,
      3177,
      8324,
      708,
      1098,
      5112,
      803,
      779,
      8324,
      4908,
      884,
      730,
      1532,
      4690,
      730,
      1718,
      530,
      1166,
      12043,
      906,
      768,
      3347,
      3084,
      521,
      7885,
      968,
      9213,
      833,
      810,
      2341,
      791,
      968,
      28682,
      523,
      2479,
      1166,
      1295,
      14332,
      35517,
      896,
      1166,
      1014,
      4001,
      884,
      779,
      26168,
      523,
      509,
      528,
      523,
      21640,
      1090,
      2312,
      708,
      548,
      54069,
      17422,
      2851,
      779,
      1423,
      523,
      941,
      10039,
      9657,
      779,
      6844,
      7405,
      521,
      1144,
      9388,
      730,
      527,
      521,
      9300,
      875,
      779,
      11917,
      12562,
      3549,
      521,
      810,
      35130,
      1547,
      27297,
      45546,
      3256,
      779,
      8008,
      3509,
      521,
      1895,
      916,
      1209,
      1399,
      11583,
      20470,
      523,
      941,
      7982,
      875,
      768,
      55097,
      22212,
      884,
      8995,
      27784,
      6806,
      521,
      810,
      779,
      17677,
      1178,
      2166,
      11860,
      811,
      6715,
      1230,
      768,
      11013,
      8153,
      797,
      3354,
      523,
      49427,
      5313,
      14118,
      988,
      730,
      18833,
      811,
      730,
      22989,
      16671,
      523,
      1405,
      14954,
      791,
      779,
      15671,
      810,
      779,
      20470,
      1316,
      5514,
      779,
      38666,
      811,
      1973,
      3622,
      2759,
      3217,
      768,
      1423,
      523,
      509,
      529,
      523,
      9312,
      802,
      9233,
      1090,
      2312,
      708,
      1098,
      1230,
      9932,
      3185,
      802,
      9233,
      521,
      2345,
      586,
      521,
      6892,
      779,
      12324,
      875,
      779,
      1423,
      523,
      53813,
      953,
      730,
      531,
      521,
      21957,
      535,
      22398,
      23023,
      730,
      527,
      521,
      26547,
      521,
      779,
      6844,
      7405,
      730,
      527,
      521,
      9300,
      810,
      41634,
      730,
      526,
      521,
      30542,
      523,
      2315,
      2124,
      11761,
      953,
      730,
      529,
      521,
      25803,
      521,
      8787,
      38666,
      17236,
      843,
      526,
      521,
      21405,
      1109,
      12235,
      843,
      23617,
      1109,
      11917,
      33265,
      843,
      7537,
      518,
      810,
      7494,
      875,
      779,
      8008,
      3509,
      5158,
      6688,
      843,
      526,
      521,
      26668,
      1191,
      941,
      1423,
      8557,
      916,
      768,
      41938,
      803,
      730,
      526,
      521,
      32647,
      810,
      768,
      7493,
      8386,
      803,
      730,
      534,
      521,
      19222,
      523,
      941,
      12324,
      1320,
      1381,
      20425,
      968,
      902,
      7054,
      1924,
      5725,
      521,
      1316,
      1833,
      1295,
      4662,
      523,
      32826,
      12043,
      779,
      12324,
      968,
      768,
      1914,
      803,
      6108,
      521,
      916,
      1295,
      20402,
      2723,
      523,
      509,
      530,
      523,
      32964,
      988,
      779,
      3822,
      3892,
      708,
      26691,
      2514,
      843,
      549,
      37986,
      3271,
      779,
      2514,
      21440,
      730,
      5411,
      8267,
      35304,
      884,
      779,
      1375,
      977,
      5329,
      779,
      2254,
      810,
      50398,
      1547,
      803,
      779,
      2759,
      5854,
      20842,
      884,
      779,
      5253,
      523,
      1311,
      1499,
      1672,
      811,
      8153,
      779,
      8008,
      3509,
      875,
      21275,
      4476,
      797,
      779,
      6844,
      810,
      856,
      5400,
      875,
      1547,
      1209,
      20470,
      819,
      30193,
      2514,
      843,
      545,
      6310,
      793,
      3271,
      29504,
      779,
      6844,
      7405,
      521,
      779,
      2514,
      9657,
      768,
      30780,
      4915,
      797,
      4843,
      896,
      17322,
      1919,
      523,
      835,
      3316,
      4915,
      797,
      779,
      11917,
      856,
      13573,
      875,
      4471,
      1288,
      768,
      35563,
      1011,
      874,
      20560,
      963,
      768,
      16336,
      7264,
      819,
      35798,
      4977,
      810,
      5600,
      2514,
      843,
      549,
      831,
      833,
      3271,
      10325,
      988,
      779,
      22212,
      7982,
      521,
      779,
      2514,
      4468,
      2204,
      15073,
      11513,
      15489,
      810,
      768,
      26781,
      15761,
      884,
      13878,
      9690,
      811,
      779,
      17677,
      536,
      1547,
      803,
      779,
      15489,
      1052,
      1381,
      50398,
      523,
      509,
      531,
      523,
      7329,
      12562,
      3549,
      708,
      559,
      4182,
      4468,
      875,
      779,
      11917,
      15671,
      896,
      779,
      12562,
      12940,
      1178,
      1381,
      29878,
      963,
      730,
      1133,
      521,
      6019,
      523,
      3877,
      779,
      730,
      527,
      521,
      9300,
      988,
      779,
      6844,
      7405,
      521,
      779,
      3549,
      2166,
      12973,
      963,
      730,
      1632,
      521,
      14671,
      523,
      941,
      15671,
      1178,
      6944,
      811,
      1547,
      56302,
      8810,
      592,
      875,
      779,
      2280,
      810,
      45777,
      1149,
      13041,
      968,
      779,
      1764,
      803,
      3354,
      523,
      6162,
      4584,
      2155,
      2076,
      4526,
      1929,
      30541,
      1232,
      1000,
      797,
      779,
      12562,
      3494,
      2013,
      779,
      6844,
      521,
      810,
      902,
      8535,
      8923,
      1188,
      3999,
      779,
      12562,
      2076,
      1523,
      1336,
      10309,
      523,
      32826,
      5751,
      3984,
      779,
      10039,
      1499,
      3644,
      1523,
      3290,
      8372,
      536,
      28682,
      1845,
      896,
      779,
      15671,
      1499,
      2743,
      1892,
      811,
      768,
      3177,
      8324,
      2076,
      54526,
      1523,
      803,
      779,
      10039,
      1090,
      4636,
      523,
      3732,
      5484,
      9303,
      811,
      49904,
      768,
      32673,
      5043,
      875,
      779,
      3549,
      797,
      4963,
      521,
      810,
      2993,
      3967,
      12547,
      3501,
      811,
      1801,
      523,
      509,
      532,
      523,
      53776,
      803,
      12900,
      708,
      548,
      54069,
      11860,
      811,
      2472,
      2030,
      906,
      12377,
      810,
      953,
      840,
      5508,
      3800,
      946,
      920,
      4346,
      523,
      9413,
      598,
      19718,
      953,
      840,
      5508,
      3800,
      906,
      27246,
      946,
      920,
      4346,
      523,
      2345,
      586,
      13682,
      2208,
      906,
      3185,
      802,
      9233,
      1720,
      4250,
      1779,
      523,
      8726,
      3967,
      1166,
      33256,
      875,
      779,
      2375,
      535,
      10593,
      3102,
      810,
      15642,
      523,
      4162,
      3290,
      15450,
      988,
      2059,
      16397,
      521,
      768,
      7155,
      48506,
      953,
      4908,
      523,
      10593,
      3102,
      5068,
      730,
      2235,
      20402,
      810,
      15642,
      730,
      1443,
      521,
      916,
      730,
      527,
      20069,
      9244,
      12452,
      523,
      10593,
      3102,
      953,
      13096,
      11510,
      906,
      3185,
      802,
      9233,
      523,
      941,
      8324,
      14954,
      791,
      2345,
      586,
      6248,
      876,
      875,
      1210,
      1779,
      803,
      3714,
      523,
      509,
      533,
      523,
      41453,
      730,
      526,
      535,
      39391,
      1465,
      14122,
      884,
      779,
      7314,
      5253,
      708,
      2364,
      4346,
      968,
      1030,
      774,
      521,
      2341,
      791,
      968,
      15821,
      793,
      535,
      997,
      7464,
      779,
      10039,
      47882,
      1287,
      811,
      730,
      526,
      521,
      1086,
      988,
      1352,
      25511,
      884,
      2759,
      39391,
      1465,
      14122,
      963,
      779,
      9028,
      803,
      779,
      7314,
      5253,
      521,
      3910,
      811,
      17677,
      15001,
      2927,
      1030,
      774,
      10201,
      896,
      28285,
      5419,
      938,
      6654,
      979,
      3213,
      811,
      779,
      8746,
      1327,
      521,
      1144,
      30809,
      779,
      5799,
      810,
      12785,
      779,
      3789,
      8034,
      875,
      10415,
      824,
      5569,
      523,
      3732,
      5484,
      5751,
      3984,
      779,
      1465,
      14122,
      1499,
      8524,
      38883,
      536,
      1030,
      774,
      25723,
      896,
      1186,
      1499,
      2472,
      797,
      3160,
      803,
      779,
      15565,
      1090,
      11404,
      523,
      1019,
      2176,
      535,
      730,
      2999,
      875,
      521,
      730,
      534,
      2723,
      521,
      730,
      532,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      7818,
      523,
      509,
      534,
      523,
      41453,
      730,
      527,
      535,
      8900,
      22398,
      22819,
      708,
      2364,
      4346,
      968,
      779,
      15671,
      535,
      997,
      7464,
      779,
      8900,
      22819,
      27870,
      988,
      730,
      1374,
      811,
      730,
      1386,
      1094,
      8461,
      988,
      730,
      526,
      4471,
      730,
      1718,
      531,
      2927,
      941,
      3185,
      802,
      9233,
      59339,
      1845,
      896,
      1033,
      1423,
      1090,
      41938,
      4201,
      902,
      3761,
      27759,
      875,
      2166,
      523,
      17857,
      3967,
      1845,
      896,
      768,
      3585,
      22819,
      1499,
      3151,
      1612,
      1361,
      16671,
      521,
      4155,
      797,
      779,
      1361,
      1434,
      1872,
      884,
      13878,
      9690,
      523,
      1019,
      2176,
      535,
      730,
      1133,
      875,
      521,
      730,
      2117,
      2723,
      521,
      730,
      528,
      41754,
      806,
      1114,
      523,
      941,
      9557,
      953,
      17602,
      521,
      810,
      779,
      22819,
      32660,
      963,
      730,
      1374,
      523,
      509,
      1155,
      523,
      41453,
      730,
      528,
      535,
      3681,
      9041,
      708,
      2364,
      4346,
      968,
      38992,
      1068,
      521,
      2341,
      791,
      968,
      43842,
      535,
      997,
      7464,
      779,
      10039,
      12881,
      768,
      3681,
      9041,
      884,
      779,
      52352,
      13646,
      5598,
      779,
      11917,
      2927,
      32826,
      7951,
      779,
      4588,
      797,
      11618,
      1203,
      5751,
      1316,
      1499,
      2144,
      1720,
      779,
      9041,
      810,
      3984,
      779,
      11917,
      15671,
      40917,
      779,
      13646,
      523,
      1311,
      953,
      11860,
      811,
      2021,
      1287,
      768,
      3822,
      2514,
      803,
      38992,
      1068,
      521,
      43842,
      810,
      15642,
      811,
      2312,
      811,
      779,
      3280,
      3177,
      8324,
      523,
      3253,
      11386,
      953,
      4281,
      523,
      509,
      1632,
      523,
      10972,
      1312,
      3436,
      708,
      13792,
      518,
      7223,
      4729,
      884,
      8995,
      27784,
      535,
      2993,
      3967,
      4468,
      11242,
      10978,
      1714,
      3537,
      779,
      5630,
      963,
      2254,
      3217,
      523,
      941,
      27246,
      1188,
      5773,
      811,
      779,
      17677,
      14065,
      875,
      779,
      3354,
      8153,
      811,
      2957,
      6586,
      1818,
      819,
      10388,
      518,
      54894,
      535,
      20470,
      938,
      4602,
      811,
      5514,
      884,
      779,
      9168,
      3392,
      803,
      779,
      7314,
      819,
      10120,
      518,
      941,
      29780,
      12548,
      50804,
      777,
      1845,
      896,
      779,
      17677,
      1090,
      8825,
      14018,
      875,
      3681,
      7092,
      21981,
      797,
      4963,
      810,
      896,
      779,
      9041,
      2724,
      33460,
      819,
      9285,
      518,
      1416,
      48053,
      535,
      779,
      17677,
      1188,
      5840,
      7980,
      23878,
      988,
      5105,
      536,
      779,
      6111,
      3378,
      1188,
      874,
      16022,
      797,
      779,
      3280,
      38666,
      819,
      11634,
      518,
      8528,
      3711,
      937,
      1888,
      535,
      9213,
      833,
      4468,
      896,
      779,
      1629,
      3711,
      937,
      1888,
      5329,
      779,
      15565,
      953,
      20425,
      797,
      6060,
      810,
      896,
      1352,
      46967,
      2716,
      1046,
      797,
      5277,
      536,
      779,
      15671,
      1188,
      2565,
      8297,
      1667,
      523,
      509,
      1374,
      523,
      13887,
      803,
      779,
      3280,
      8324,
      708,
      1098,
      3280,
      3177,
      8324,
      1188,
      874,
      4908,
      797,
      4526,
      536,
      779,
      5562,
      1188,
      874,
      9139,
      797,
      779,
      6844,
      38666,
      523,
      941,
      8324,
      8557,
      963,
      730,
      2197,
      535,
      1155,
      523,
      18,
      4023,
      3636,
      953,
      779,
      2258,
      875,
      779,
      39391,
      1465,
      14122,
      9557,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      14873,
      968,
      1529,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      13305,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      7951,
      968,
      1529,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      42869,
      1069,
      20,
      21
     ],
     "markers": [
      1429,
      1439,
      1447,
      1457
     ],
     "logits_raw": [
      -4.230611324310303,
      0.1285986751317978,
      -1.4657516479492188,
      -2.841369867324829
     ],
     "probs_raw": [
      0.010092451237142086,
      0.7891826033592224,
      0.1602359414100647,
      0.04048895090818405
     ],
     "probs": [
      0.04854496940970421,
      0.6031103730201721,
      0.23998364806175232,
      0.10836095362901688
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.3631267249584198,
     "near_tie": false,
     "gold": "2",
     "correct": false,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 4
     }
    }
   ],
   "response": {
    "answers": {
     "fee_motion": {
      "type": "choice",
      "choice": "deferred",
      "confidence": 0.6915529370307922,
      "probabilities": {
       "approved": 0.15077616274356842,
       "rejected": 0.0647965595126152,
       "deferred": 0.6915529370307922,
       "withdrawn": 0.09287437796592712
      }
     },
     "quorate": {
      "type": "noul",
      "noul": 0.9495782256126404
     },
     "treasurer": {
      "type": "choice",
      "choice": "odette",
      "confidence": 0.7560074925422668,
      "probabilities": {
       "odette": 0.7560074925422668,
       "lewis": 0.24399243295192719
      }
     },
     "racks_support": {
      "type": "score",
      "score": 1.4081605300307274,
      "confidence": 0.6031103730201721,
      "probabilities": {
       "0": 0.04854496940970421,
       "1": 0.6031103730201721,
       "2": 0.23998364806175232,
       "3": 0.10836095362901688
      },
      "legend": {
       "0": "opposed by most",
       "1": "split",
       "2": "supported by most",
       "3": "unanimous"
      }
     }
    },
    "usage": {
     "input_tokens": 5815,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_a01",
   "mode": "text",
   "source": "own_json",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 618,
   "questions": [
    {
     "qid": "first",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 220,
     "positions": 220,
     "bucket": 256,
     "ids": [
      1,
      17,
      568,
      38685,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      5768,
      1878,
      997,
      7820,
      6881,
      997,
      5059,
      2874,
      884,
      8913,
      730,
      527,
      996,
      2534,
      884,
      1973,
      4426,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      3654,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      3410,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      9270,
      1878,
      997,
      7820,
      6881,
      997,
      29408,
      13040,
      963,
      779,
      2120,
      19025,
      22879,
      1113,
      5663,
      3273,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1374,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      2291,
      1912,
      4204,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      8884,
      1878,
      997,
      7820,
      6881,
      997,
      7367,
      875,
      768,
      2341,
      6562,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1664,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      1811,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      10981,
      1878,
      997,
      7820,
      6881,
      997,
      14100,
      965,
      803,
      15658,
      8997,
      3627,
      779,
      9397,
      5362,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      4536,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      51493,
      511,
      20352,
      18,
      28828,
      30437,
      1753,
      874,
      28010,
      1509,
      540,
      19,
      16,
      17905,
      522,
      5768,
      535,
      2343,
      2874,
      996,
      2534,
      20,
      19,
      16,
      17905,
      522,
      9270,
      535,
      5719,
      20391,
      5060,
      13040,
      20,
      19,
      16,
      17905,
      522,
      8884,
      535,
      8296,
      6562,
      20,
      19,
      16,
      17905,
      522,
      10981,
      535,
      23250,
      800,
      19944,
      20,
      21
     ],
     "markers": [
      179,
      190,
      201,
      210
     ],
     "logits_raw": [
      0.3843047022819519,
      -2.1585264205932617,
      -0.326029896736145,
      0.20786070823669434
     ],
     "probs_raw": [
      0.4152187407016754,
      0.03265422582626343,
      0.20407159626483917,
      0.34805533289909363
     ],
     "probs": [
      0.3778960704803467,
      0.061447251588106155,
      0.2275114357471466,
      0.33314526081085205
     ],
     "argmax_index": 0,
     "argmax": "HD-101",
     "top2_margin": 0.04475080966949463,
     "near_tie": false,
     "gold": "HD-104",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "hazard",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 203,
     "positions": 203,
     "bucket": 256,
     "ids": [
      1,
      17,
      568,
      38685,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      5768,
      1878,
      997,
      7820,
      6881,
      997,
      5059,
      2874,
      884,
      8913,
      730,
      527,
      996,
      2534,
      884,
      1973,
      4426,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      3654,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      3410,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      9270,
      1878,
      997,
      7820,
      6881,
      997,
      29408,
      13040,
      963,
      779,
      2120,
      19025,
      22879,
      1113,
      5663,
      3273,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1374,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      2291,
      1912,
      4204,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      8884,
      1878,
      997,
      7820,
      6881,
      997,
      7367,
      875,
      768,
      2341,
      6562,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1664,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      1811,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      10981,
      1878,
      997,
      7820,
      6881,
      997,
      14100,
      965,
      803,
      15658,
      8997,
      3627,
      779,
      9397,
      5362,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      4536,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      51493,
      511,
      20352,
      18,
      29589,
      1523,
      30437,
      8487,
      768,
      3023,
      6155,
      17745,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      181,
      193
     ],
     "logits_raw": [
      2.477010488510132,
      6.635200023651123
     ],
     "probs_raw": [
      0.015395121648907661,
      0.9846048951148987
     ],
     "probs": [
      0.9238207936286926,
      0.07617922872304916
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8476415649056435,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "load",
     "type": "score",
     "K": 2,
     "calibrate": true,
     "temperature_key": "score:2",
     "T": 1.0,
     "prefix": 0,
     "n_ids": 195,
     "positions": 195,
     "bucket": 256,
     "ids": [
      1,
      17,
      568,
      38685,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      5768,
      1878,
      997,
      7820,
      6881,
      997,
      5059,
      2874,
      884,
      8913,
      730,
      527,
      996,
      2534,
      884,
      1973,
      4426,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      3654,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      3410,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      9270,
      1878,
      997,
      7820,
      6881,
      997,
      29408,
      13040,
      963,
      779,
      2120,
      19025,
      22879,
      1113,
      5663,
      3273,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1374,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      2291,
      1912,
      4204,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      8884,
      1878,
      997,
      7820,
      6881,
      997,
      7367,
      875,
      768,
      2341,
      6562,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      1664,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      526,
      1811,
      511,
      3205,
      37080,
      593,
      32909,
      6881,
      997,
      13806,
      522,
      10981,
      1878,
      997,
      7820,
      6881,
      997,
      14100,
      965,
      803,
      15658,
      8997,
      3627,
      779,
      9397,
      5362,
      1878,
      997,
      782,
      29500,
      6881,
      997,
      4905,
      535,
      4536,
      1878,
      997,
      3165,
      1012,
      592,
      6881,
      997,
      51493,
      511,
      20352,
      18,
      4940,
      779,
      27838,
      1391,
      38596,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      4381,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      1391,
      38596,
      20,
      21
     ],
     "markers": [
      178,
      186
     ],
     "logits_raw": [
      -2.7285783290863037,
      2.443589687347412
     ],
     "probs_raw": [
      0.005640266463160515,
      0.9943597912788391
     ],
     "probs": [
      0.005640266463160515,
      0.9943597912788391
     ],
     "argmax_index": 1,
     "argmax": "1",
     "top2_margin": 0.9887195248156786,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "first": {
      "type": "choice",
      "choice": "HD-101",
      "confidence": 0.3778960704803467,
      "probabilities": {
       "HD-101": 0.3778960704803467,
       "HD-102": 0.061447251588106155,
       "HD-103": 0.2275114357471466,
       "HD-104": 0.33314526081085205
      }
     },
     "hazard": {
      "type": "noul",
      "noul": 0.9238207936286926
     },
     "load": {
      "type": "score",
      "score": 0.9943597912788391,
      "confidence": 0.9943597912788391,
      "probabilities": {
       "0": 0.005640266463160515,
       "1": 0.9943597912788391
      },
      "legend": {
       "0": "normal",
       "1": "overloaded"
      }
     }
    },
    "usage": {
     "input_tokens": 618,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_j01",
   "mode": "text",
   "source": "own_json",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 762,
   "questions": [
    {
     "qid": "overdue",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 247,
     "positions": 247,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      769,
      47958,
      6881,
      997,
      15807,
      564,
      522,
      1718,
      531,
      522,
      30907,
      527,
      1878,
      997,
      1271,
      2466,
      6881,
      997,
      547,
      1983,
      893,
      1174,
      18119,
      16643,
      1878,
      997,
      38765,
      770,
      6881,
      997,
      556,
      820,
      799,
      12890,
      12519,
      1157,
      4847,
      1878,
      997,
      1271,
      2686,
      6881,
      997,
      1718,
      531,
      522,
      6326,
      522,
      5843,
      1878,
      997,
      30717,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      5843,
      1878,
      997,
      19248,
      6881,
      997,
      885,
      38008,
      1878,
      997,
      2481,
      53339,
      4294,
      806,
      6881,
      730,
      527,
      521,
      997,
      10890,
      6881,
      1607,
      38685,
      5027,
      6881,
      997,
      2414,
      29248,
      47635,
      835,
      527,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      2108,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      526,
      523,
      3520,
      3205,
      37080,
      5027,
      6881,
      997,
      19938,
      1477,
      2643,
      10819,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      7896,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      525,
      523,
      5370,
      3205,
      37080,
      5027,
      6881,
      997,
      19589,
      36603,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      1374,
      523,
      525,
      602,
      5086,
      997,
      4093,
      35377,
      6881,
      730,
      22277,
      523,
      525,
      521,
      997,
      14623,
      572,
      10233,
      6881,
      730,
      525,
      523,
      527,
      521,
      997,
      35377,
      6881,
      730,
      25633,
      523,
      527,
      521,
      997,
      27317,
      957,
      572,
      55913,
      6881,
      997,
      18703,
      6337,
      2638,
      730,
      1664,
      3378,
      1878,
      997,
      593,
      53277,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      913,
      511,
      602,
      18,
      4940,
      779,
      60827,
      1139,
      1391,
      30717,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      225,
      237
     ],
     "logits_raw": [
      -0.45539525151252747,
      1.4913076162338257
     ],
     "probs_raw": [
      0.12491331994533539,
      0.8750866651535034
     ],
     "probs": [
      0.762830913066864,
      0.237169086933136
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.525661826133728,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "method",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 245,
     "positions": 245,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      769,
      47958,
      6881,
      997,
      15807,
      564,
      522,
      1718,
      531,
      522,
      30907,
      527,
      1878,
      997,
      1271,
      2466,
      6881,
      997,
      547,
      1983,
      893,
      1174,
      18119,
      16643,
      1878,
      997,
      38765,
      770,
      6881,
      997,
      556,
      820,
      799,
      12890,
      12519,
      1157,
      4847,
      1878,
      997,
      1271,
      2686,
      6881,
      997,
      1718,
      531,
      522,
      6326,
      522,
      5843,
      1878,
      997,
      30717,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      5843,
      1878,
      997,
      19248,
      6881,
      997,
      885,
      38008,
      1878,
      997,
      2481,
      53339,
      4294,
      806,
      6881,
      730,
      527,
      521,
      997,
      10890,
      6881,
      1607,
      38685,
      5027,
      6881,
      997,
      2414,
      29248,
      47635,
      835,
      527,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      2108,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      526,
      523,
      3520,
      3205,
      37080,
      5027,
      6881,
      997,
      19938,
      1477,
      2643,
      10819,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      7896,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      525,
      523,
      5370,
      3205,
      37080,
      5027,
      6881,
      997,
      19589,
      36603,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      1374,
      523,
      525,
      602,
      5086,
      997,
      4093,
      35377,
      6881,
      730,
      22277,
      523,
      525,
      521,
      997,
      14623,
      572,
      10233,
      6881,
      730,
      525,
      523,
      527,
      521,
      997,
      35377,
      6881,
      730,
      25633,
      523,
      527,
      521,
      997,
      27317,
      957,
      572,
      55913,
      6881,
      997,
      18703,
      6337,
      2638,
      730,
      1664,
      3378,
      1878,
      997,
      593,
      53277,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      913,
      511,
      602,
      18,
      4023,
      856,
      779,
      13608,
      5559,
      811,
      4025,
      540,
      19,
      16,
      5060,
      20,
      19,
      16,
      7493,
      41953,
      1100,
      20,
      19,
      16,
      14184,
      20,
      19,
      16,
      1636,
      1357,
      20,
      21
     ],
     "markers": [
      226,
      230,
      236,
      240
     ],
     "logits_raw": [
      -9.03740406036377,
      1.9014228582382202,
      -6.792163848876953,
      -6.893122673034668
     ],
     "probs_raw": [
      1.774932025000453e-05,
      0.999663233757019,
      0.00016760120342951268,
      0.0001515065087005496
     ],
     "probs": [
      0.00040230932063423097,
      0.9957361221313477,
      0.0020003572572022676,
      0.001861173310317099
     ],
     "argmax_index": 1,
     "argmax": "bank_transfer",
     "top2_margin": 0.9937357648741454,
     "near_tie": false,
     "gold": "bank_transfer",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "size",
     "type": "score",
     "K": 4,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 270,
     "positions": 270,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      769,
      47958,
      6881,
      997,
      15807,
      564,
      522,
      1718,
      531,
      522,
      30907,
      527,
      1878,
      997,
      1271,
      2466,
      6881,
      997,
      547,
      1983,
      893,
      1174,
      18119,
      16643,
      1878,
      997,
      38765,
      770,
      6881,
      997,
      556,
      820,
      799,
      12890,
      12519,
      1157,
      4847,
      1878,
      997,
      1271,
      2686,
      6881,
      997,
      1718,
      531,
      522,
      6326,
      522,
      5843,
      1878,
      997,
      30717,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      5843,
      1878,
      997,
      19248,
      6881,
      997,
      885,
      38008,
      1878,
      997,
      2481,
      53339,
      4294,
      806,
      6881,
      730,
      527,
      521,
      997,
      10890,
      6881,
      1607,
      38685,
      5027,
      6881,
      997,
      2414,
      29248,
      47635,
      835,
      527,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      2108,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      526,
      523,
      3520,
      3205,
      37080,
      5027,
      6881,
      997,
      19938,
      1477,
      2643,
      10819,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      7896,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      525,
      523,
      5370,
      3205,
      37080,
      5027,
      6881,
      997,
      19589,
      36603,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      10829,
      572,
      51363,
      6881,
      730,
      1374,
      523,
      525,
      602,
      5086,
      997,
      4093,
      35377,
      6881,
      730,
      22277,
      523,
      525,
      521,
      997,
      14623,
      572,
      10233,
      6881,
      730,
      525,
      523,
      527,
      521,
      997,
      35377,
      6881,
      730,
      25633,
      523,
      527,
      521,
      997,
      27317,
      957,
      572,
      55913,
      6881,
      997,
      18703,
      6337,
      2638,
      730,
      1664,
      3378,
      1878,
      997,
      593,
      53277,
      6881,
      997,
      1718,
      531,
      522,
      3654,
      522,
      913,
      511,
      602,
      18,
      4023,
      2611,
      856,
      779,
      60827,
      1139,
      3426,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      1490,
      730,
      2096,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      730,
      2096,
      811,
      730,
      7896,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      730,
      7896,
      811,
      730,
      4291,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      1391,
      730,
      4291,
      20,
      21
     ],
     "markers": [
      226,
      236,
      248,
      260
     ],
     "logits_raw": [
      -15.045595169067383,
      -6.663366317749023,
      -6.0419511795043945,
      -7.225985527038574
     ],
     "probs_raw": [
      6.670520815532655e-05,
      0.2914173901081085,
      0.5424911379814148,
      0.1660248339176178
     ],
     "probs": [
      0.002488102298229933,
      0.316214919090271,
      0.4528668224811554,
      0.22843007743358612
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.1366519033908844,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 2.3283064365386963e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "overdue": {
      "type": "noul",
      "noul": 0.762830913066864
     },
     "method": {
      "type": "choice",
      "choice": "bank_transfer",
      "confidence": 0.9957361221313477,
      "probabilities": {
       "card": 0.00040230932063423097,
       "bank_transfer": 0.9957361221313477,
       "cash": 0.0020003572572022676,
       "cheque": 0.001861173310317099
      }
     },
     "size": {
      "type": "score",
      "score": 1.9072387963533401,
      "confidence": 0.4528668224811554,
      "probabilities": {
       "0": 0.002488102298229933,
       "1": 0.316214919090271,
       "2": 0.4528668224811554,
       "3": 0.22843007743358612
      },
      "legend": {
       "0": "under 100",
       "1": "100 to 250",
       "2": "250 to 500",
       "3": "over 500"
      }
     }
    },
    "usage": {
     "input_tokens": 762,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_j02",
   "mode": "text",
   "source": "own_json",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 931,
   "questions": [
    {
     "qid": "next_action",
     "type": "choice",
     "K": 10,
     "calibrate": true,
     "temperature_key": "choice:6-10",
     "T": 1.1751071214675903,
     "prefix": 0,
     "n_ids": 378,
     "positions": 378,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      3667,
      6590,
      6881,
      997,
      42468,
      522,
      26260,
      1532,
      1878,
      997,
      15162,
      6881,
      997,
      557,
      1983,
      834,
      917,
      38281,
      1878,
      997,
      57267,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      6135,
      730,
      1133,
      535,
      2019,
      1878,
      997,
      26207,
      6881,
      1607,
      38685,
      38017,
      6881,
      997,
      556,
      2508,
      522,
      526,
      552,
      548,
      1878,
      997,
      3055,
      6881,
      997,
      46553,
      791,
      63437,
      730,
      526,
      14895,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      527,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      549,
      41555,
      522,
      14819,
      1878,
      997,
      3055,
      6881,
      997,
      41388,
      32849,
      17125,
      730,
      14819,
      836,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      19248,
      6881,
      997,
      1066,
      803,
      9051,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      12411,
      542,
      4894,
      17727,
      1878,
      997,
      3055,
      6881,
      997,
      26818,
      13192,
      521,
      730,
      1838,
      22173,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      528,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      602,
      5086,
      997,
      2354,
      11468,
      6881,
      37080,
      8618,
      6265,
      6881,
      997,
      2482,
      1930,
      2286,
      1145,
      1878,
      997,
      19132,
      800,
      36619,
      6881,
      997,
      14402,
      1991,
      791,
      1878,
      997,
      14402,
      1991,
      791,
      40040,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      3654,
      730,
      1632,
      535,
      2108,
      511,
      3205,
      997,
      27317,
      957,
      6881,
      37080,
      14164,
      6881,
      997,
      19563,
      1878,
      997,
      576,
      2922,
      3107,
      6881,
      730,
      2142,
      523,
      529,
      521,
      997,
      1891,
      1356,
      791,
      6881,
      730,
      525,
      523,
      525,
      3205,
      997,
      38765,
      770,
      5205,
      2176,
      6881,
      997,
      15090,
      856,
      1727,
      17125,
      540,
      859,
      953,
      15418,
      875,
      936,
      2927,
      602,
      18,
      3493,
      1753,
      779,
      15565,
      1112,
      3280,
      540,
      19,
      16,
      53467,
      5377,
      45556,
      43840,
      535,
      12620,
      1356,
      779,
      8688,
      896,
      953,
      1014,
      3567,
      20,
      19,
      16,
      1039,
      959,
      572,
      3667,
      535,
      26995,
      779,
      4394,
      2565,
      2030,
      20,
      19,
      16,
      5254,
      572,
      1608,
      5072,
      535,
      18388,
      779,
      2286,
      1145,
      1695,
      779,
      61026,
      856,
      20,
      19,
      16,
      23496,
      572,
      3667,
      535,
      2577,
      5072,
      779,
      2565,
      20,
      19,
      16,
      2785,
      53220,
      535,
      23361,
      779,
      11746,
      4332,
      20,
      19,
      16,
      25668,
      30024,
      535,
      27908,
      768,
      25668,
      5214,
      20,
      19,
      16,
      8297,
      4370,
      831,
      5278,
      535,
      1416,
      4128,
      15073,
      11391,
      20,
      19,
      16,
      10547,
      4370,
      808,
      36603,
      535,
      27589,
      779,
      13608,
      779,
      61026,
      953,
      14465,
      20,
      19,
      16,
      24744,
      15623,
      2265,
      535,
      32655,
      779,
      2565,
      875,
      24744,
      4202,
      20,
      19,
      16,
      1295,
      55692,
      535,
      28657,
      3988,
      811,
      874,
      4007,
      20,
      21
     ],
     "markers": [
      251,
      267,
      280,
      296,
      307,
      317,
      327,
      339,
      354,
      367
     ],
     "logits_raw": [
      -1.7629752159118652,
      -6.255130290985107,
      0.7563826441764832,
      -5.840479373931885,
      -10.807558059692383,
      -8.814050674438477,
      -13.214742660522461,
      -4.54431676864624,
      -5.7599897384643555,
      -2.928925037384033
     ],
     "probs_raw": [
      0.07224540412425995,
      0.0008088949834927917,
      0.8973326086997986,
      0.001224539359100163,
      8.527027603122406e-06,
      6.259893416427076e-05,
      7.6801677550975e-07,
      0.004475989378988743,
      0.0013271773932501674,
      0.022513583302497864
     ],
     "probs": [
      0.09913931041955948,
      0.0021679105702787638,
      0.8459538817405701,
      0.0030852300114929676,
      4.503615491557866e-05,
      0.0002456512884236872,
      5.806538865726907e-06,
      0.009296542964875698,
      0.0033039606641978025,
      0.03675662726163864
     ],
     "argmax_index": 2,
     "argmax": "track_parcel",
     "top2_margin": 0.7468145713210106,
     "near_tie": false,
     "gold": "refund_missing_item",
     "correct": false,
     "host_softmax_max_abs_dp": 9.313225746154785e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "partial",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 274,
     "positions": 274,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      3667,
      6590,
      6881,
      997,
      42468,
      522,
      26260,
      1532,
      1878,
      997,
      15162,
      6881,
      997,
      557,
      1983,
      834,
      917,
      38281,
      1878,
      997,
      57267,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      6135,
      730,
      1133,
      535,
      2019,
      1878,
      997,
      26207,
      6881,
      1607,
      38685,
      38017,
      6881,
      997,
      556,
      2508,
      522,
      526,
      552,
      548,
      1878,
      997,
      3055,
      6881,
      997,
      46553,
      791,
      63437,
      730,
      526,
      14895,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      527,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      549,
      41555,
      522,
      14819,
      1878,
      997,
      3055,
      6881,
      997,
      41388,
      32849,
      17125,
      730,
      14819,
      836,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      19248,
      6881,
      997,
      1066,
      803,
      9051,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      12411,
      542,
      4894,
      17727,
      1878,
      997,
      3055,
      6881,
      997,
      26818,
      13192,
      521,
      730,
      1838,
      22173,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      528,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      602,
      5086,
      997,
      2354,
      11468,
      6881,
      37080,
      8618,
      6265,
      6881,
      997,
      2482,
      1930,
      2286,
      1145,
      1878,
      997,
      19132,
      800,
      36619,
      6881,
      997,
      14402,
      1991,
      791,
      1878,
      997,
      14402,
      1991,
      791,
      40040,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      3654,
      730,
      1632,
      535,
      2108,
      511,
      3205,
      997,
      27317,
      957,
      6881,
      37080,
      14164,
      6881,
      997,
      19563,
      1878,
      997,
      576,
      2922,
      3107,
      6881,
      730,
      2142,
      523,
      529,
      521,
      997,
      1891,
      1356,
      791,
      6881,
      730,
      525,
      523,
      525,
      3205,
      997,
      38765,
      770,
      5205,
      2176,
      6881,
      997,
      15090,
      856,
      1727,
      17125,
      540,
      859,
      953,
      15418,
      875,
      936,
      2927,
      602,
      18,
      28519,
      1550,
      1192,
      803,
      779,
      2565,
      39993,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      252,
      264
     ],
     "logits_raw": [
      1.2194080352783203,
      5.291305065155029
     ],
     "probs_raw": [
      0.0167593602091074,
      0.9832406640052795
     ],
     "probs": [
      0.9200953841209412,
      0.07990466803312302
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8401907160878181,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "charged_claim",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 279,
     "positions": 279,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      3667,
      6590,
      6881,
      997,
      42468,
      522,
      26260,
      1532,
      1878,
      997,
      15162,
      6881,
      997,
      557,
      1983,
      834,
      917,
      38281,
      1878,
      997,
      57267,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      6135,
      730,
      1133,
      535,
      2019,
      1878,
      997,
      26207,
      6881,
      1607,
      38685,
      38017,
      6881,
      997,
      556,
      2508,
      522,
      526,
      552,
      548,
      1878,
      997,
      3055,
      6881,
      997,
      46553,
      791,
      63437,
      730,
      526,
      14895,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      527,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      549,
      41555,
      522,
      14819,
      1878,
      997,
      3055,
      6881,
      997,
      41388,
      32849,
      17125,
      730,
      14819,
      836,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      526,
      521,
      997,
      19248,
      6881,
      997,
      1066,
      803,
      9051,
      511,
      3205,
      37080,
      38017,
      6881,
      997,
      12411,
      542,
      4894,
      17727,
      1878,
      997,
      3055,
      6881,
      997,
      26818,
      13192,
      521,
      730,
      1838,
      22173,
      1878,
      997,
      590,
      1937,
      6881,
      730,
      528,
      521,
      997,
      19248,
      6881,
      997,
      2354,
      9487,
      511,
      602,
      5086,
      997,
      2354,
      11468,
      6881,
      37080,
      8618,
      6265,
      6881,
      997,
      2482,
      1930,
      2286,
      1145,
      1878,
      997,
      19132,
      800,
      36619,
      6881,
      997,
      14402,
      1991,
      791,
      1878,
      997,
      14402,
      1991,
      791,
      40040,
      6881,
      997,
      1718,
      531,
      522,
      6468,
      522,
      3654,
      730,
      1632,
      535,
      2108,
      511,
      3205,
      997,
      27317,
      957,
      6881,
      37080,
      14164,
      6881,
      997,
      19563,
      1878,
      997,
      576,
      2922,
      3107,
      6881,
      730,
      2142,
      523,
      529,
      521,
      997,
      1891,
      1356,
      791,
      6881,
      730,
      525,
      523,
      525,
      3205,
      997,
      38765,
      770,
      5205,
      2176,
      6881,
      997,
      15090,
      856,
      1727,
      17125,
      540,
      859,
      953,
      15418,
      875,
      936,
      2927,
      602,
      18,
      29589,
      779,
      13608,
      2992,
      1186,
      10274,
      875,
      3503,
      1186,
      2172,
      1014,
      6984,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      257,
      269
     ],
     "logits_raw": [
      2.150092363357544,
      -0.286288321018219
     ],
     "probs_raw": [
      0.9195597767829895,
      0.0804402306675911
     ],
     "probs": [
      0.18814170360565186,
      0.8118582963943481
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.6237165927886963,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "next_action": {
      "type": "choice",
      "choice": "track_parcel",
      "confidence": 0.8459538817405701,
      "probabilities": {
       "refund_missing_item": 0.09913931041955948,
       "resend_order": 0.0021679105702787638,
       "track_parcel": 0.8459538817405701,
       "cancel_order": 0.0030852300114929676,
       "change_address": 4.503615491557866e-05,
       "discount_code": 0.0002456512884236872,
       "replace_damaged": 5.806538865726907e-06,
       "confirm_delivery": 0.009296542964875698,
       "fraud_review": 0.0033039606641978025,
       "no_action": 0.03675662726163864
      }
     },
     "partial": {
      "type": "noul",
      "noul": 0.9200953841209412
     },
     "charged_claim": {
      "type": "noul",
      "noul": 0.18814170360565186
     }
    },
    "usage": {
     "input_tokens": 931,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_m01",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 1408,
   "questions": [
    {
     "qid": "bedrooms",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 164,
     "positions": 164,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      4023,
      1697,
      6100,
      14488,
      2177,
      779,
      9645,
      1052,
      540,
      19,
      16,
      1235,
      20,
      19,
      16,
      1547,
      20,
      19,
      16,
      2204,
      20,
      21
     ],
     "markers": [
      152,
      156,
      160
     ],
     "logits_raw": [
      -2.658261299133301,
      -0.02049952745437622,
      -5.027771472930908
     ],
     "probs_raw": [
      0.06633322685956955,
      0.9274629354476929,
      0.006203918252140284
     ],
     "probs": [
      0.1287752240896225,
      0.8475256562232971,
      0.02369912900030613
     ],
     "argmax_index": 1,
     "argmax": "two",
     "top2_margin": 0.7187504321336746,
     "near_tie": false,
     "gold": "two",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 7,
      "B": 8
     }
    },
    {
     "qid": "lift",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 170,
     "positions": 170,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      4940,
      1413,
      768,
      12372,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      148,
      160
     ],
     "logits_raw": [
      0.049930721521377563,
      9.483137130737305
     ],
     "probs_raw": [
      8.001578680705279e-05,
      0.9999200105667114
     ],
     "probs": [
      0.9965332746505737,
      0.003466650377959013
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9930666242726147,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 6,
      "B": 8
     }
    },
    {
     "qid": "furnished",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 177,
     "positions": 177,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      4023,
      60603,
      856,
      779,
      9645,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      9104,
      1250,
      2075,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      20618,
      60603,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      7408,
      60603,
      20,
      21
     ],
     "markers": [
      149,
      159,
      168
     ],
     "logits_raw": [
      -0.9351052641868591,
      -1.015489101409912,
      -0.2348705232143402
     ],
     "probs_raw": [
      0.25400131940841675,
      0.23438279330730438,
      0.5116159319877625
     ],
     "probs": [
      0.2895609736442566,
      0.27641525864601135,
      0.43402379751205444
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.14446282386779785,
     "near_tie": false,
     "gold": "0",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 8
     }
    },
    {
     "qid": "rabbit",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 179,
     "positions": 179,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      45313,
      768,
      44416,
      3089,
      768,
      36802,
      540,
      19,
      16,
      18672,
      535,
      2595,
      19226,
      20,
      19,
      16,
      968,
      572,
      890,
      22748,
      535,
      11005,
      916,
      779,
      2496,
      46130,
      1090,
      10449,
      20,
      19,
      16,
      1295,
      535,
      4524,
      6767,
      20,
      21
     ],
     "markers": [
      150,
      157,
      172
     ],
     "logits_raw": [
      -1.355165719985962,
      1.5525476932525635,
      0.4870193600654602
     ],
     "probs_raw": [
      0.03902411088347435,
      0.7147216200828552,
      0.24625429511070251
     ],
     "probs": [
      0.07868150621652603,
      0.627973198890686,
      0.29334530234336853
     ],
     "argmax_index": 1,
     "argmax": "by_agreement",
     "top2_margin": 0.3346278965473175,
     "near_tie": false,
     "gold": "no",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 8
     }
    },
    {
     "qid": "parking",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 174,
     "positions": 174,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      29589,
      779,
      9645,
      2743,
      916,
      768,
      23171,
      3494,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      152,
      164
     ],
     "logits_raw": [
      1.846391201019287,
      -2.831726312637329
     ],
     "probs_raw": [
      0.9907891154289246,
      0.009210866875946522
     ],
     "probs": [
      0.056922901421785355,
      0.9430771470069885
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.8861542455852032,
     "near_tie": false,
     "gold": "false",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 5,
      "B": 8
     }
    },
    {
     "qid": "bill",
     "type": "choice",
     "K": 5,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 176,
     "positions": 176,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      28828,
      5363,
      856,
      4557,
      797,
      779,
      16429,
      540,
      19,
      16,
      10366,
      20,
      19,
      16,
      4916,
      20,
      19,
      16,
      1883,
      20,
      19,
      16,
      9399,
      20,
      19,
      16,
      10735,
      535,
      3253,
      5363,
      856,
      4557,
      20,
      21
     ],
     "markers": [
      151,
      155,
      159,
      163,
      167
     ],
     "logits_raw": [
      -6.876915454864502,
      -14.46700668334961,
      -6.447972774505615,
      -9.973302841186523,
      1.3173890113830566
     ],
     "probs_raw": [
      0.0002760255301836878,
      1.3951301980341668e-07,
      0.00042387392022646964,
      1.2479739780246746e-05,
      0.9992875456809998
     ],
     "probs": [
      0.0028496168088167906,
      1.2591870472533628e-05,
      0.0038713153917342424,
      0.0003120219334959984,
      0.9929545521736145
     ],
     "argmax_index": 4,
     "argmax": "none",
     "top2_margin": 0.9890832367818803,
     "near_tie": false,
     "gold": "water",
     "correct": false,
     "host_softmax_max_abs_dp": 1.1920928955078125e-07,
     "batch": {
      "call": 0,
      "index": 3,
      "B": 8
     }
    },
    {
     "qid": "weekday_viewing",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 175,
     "positions": 175,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      9036,
      779,
      9645,
      874,
      16521,
      884,
      768,
      3025,
      2367,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      153,
      165
     ],
     "logits_raw": [
      1.393991470336914,
      2.843672752380371
     ],
     "probs_raw": [
      0.1900506168603897,
      0.8099493384361267
     ],
     "probs": [
      0.7047432661056519,
      0.29525667428970337
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.4094865918159485,
     "near_tie": false,
     "gold": "false",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 4,
      "B": 8
     }
    },
    {
     "qid": "term",
     "type": "score",
     "K": 4,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 193,
     "positions": 193,
     "bucket": 256,
     "ids": [
      1,
      17,
      12832,
      776,
      811,
      2944,
      1141,
      2642,
      1050,
      799,
      40315,
      9690,
      521,
      15398,
      799,
      12890,
      708,
      15780,
      1985,
      791,
      5652,
      9645,
      884,
      779,
      4976,
      8913,
      803,
      768,
      4204,
      916,
      768,
      12372,
      523,
      55657,
      730,
      526,
      521,
      5411,
      1094,
      5805,
      521,
      28553,
      7449,
      811,
      4009,
      6272,
      516,
      16429,
      521,
      9447,
      3676,
      5699,
      730,
      1374,
      4493,
      523,
      20448,
      988,
      730,
      526,
      4526,
      523,
      879,
      3113,
      938,
      1014,
      4557,
      6794,
      1883,
      523,
      873,
      1808,
      4192,
      968,
      10449,
      843,
      15344,
      9889,
      810,
      15622,
      1550,
      521,
      1295,
      1312,
      5165,
      1191,
      941,
      9645,
      856,
      9104,
      1250,
      2075,
      10325,
      988,
      768,
      56926,
      810,
      768,
      24910,
      7073,
      523,
      879,
      1445,
      8784,
      797,
      779,
      39932,
      536,
      1295,
      23171,
      3494,
      523,
      13483,
      1327,
      884,
      826,
      4306,
      24463,
      1685,
      730,
      1155,
      535,
      1056,
      810,
      730,
      1532,
      535,
      1056,
      523,
      22194,
      779,
      11380,
      521,
      873,
      3085,
      521,
      811,
      2643,
      768,
      25647,
      523,
      18,
      4023,
      1787,
      856,
      779,
      9447,
      3676,
      5699,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      1490,
      730,
      531,
      4493,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      730,
      531,
      4493,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      730,
      1374,
      4493,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      1391,
      730,
      1374,
      4493,
      20,
      21
     ],
     "markers": [
      151,
      162,
      172,
      182
     ],
     "logits_raw": [
      -3.809436559677124,
      -12.830514907836914,
      5.591951847076416,
      0.9847365617752075
     ],
     "probs_raw": [
      8.178635471267626e-05,
      9.882718643439148e-09,
      0.9900380373001099,
      0.00988015253096819
     ],
     "probs": [
      0.004064416512846947,
      2.2106480173533782e-05,
      0.9309849143028259,
      0.06492854654788971
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.8660563677549362,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 1.1920928955078125e-07,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 8
     }
    }
   ],
   "response": {
    "answers": {
     "bedrooms": {
      "type": "choice",
      "choice": "two",
      "confidence": 0.8475256562232971,
      "probabilities": {
       "one": 0.1287752240896225,
       "two": 0.8475256562232971,
       "three": 0.02369912900030613
      }
     },
     "lift": {
      "type": "noul",
      "noul": 0.9965332746505737
     },
     "furnished": {
      "type": "score",
      "score": 1.1444628536701202,
      "confidence": 0.43402379751205444,
      "probabilities": {
       "0": 0.2895609736442566,
       "1": 0.27641525864601135,
       "2": 0.43402379751205444
      },
      "legend": {
       "0": "unfurnished",
       "1": "partly furnished",
       "2": "fully furnished"
      }
     },
     "rabbit": {
      "type": "choice",
      "choice": "by_agreement",
      "confidence": 0.627973198890686,
      "probabilities": {
       "yes": 0.07868150621652603,
       "by_agreement": 0.627973198890686,
       "no": 0.29334530234336853
      }
     },
     "parking": {
      "type": "noul",
      "noul": 0.056922901421785355
     },
     "bill": {
      "type": "choice",
      "choice": "none",
      "confidence": 0.9929545521736145,
      "probabilities": {
       "electricity": 0.0028496168088167906,
       "gas": 1.2591870472533628e-05,
       "water": 0.0038713153917342424,
       "internet": 0.0003120219334959984,
       "none": 0.9929545521736145
      }
     },
     "weekday_viewing": {
      "type": "noul",
      "noul": 0.7047432661056519
     },
     "term": {
      "type": "score",
      "score": 2.0567775747294945,
      "confidence": 0.9309849143028259,
      "probabilities": {
       "0": 0.004064416512846947,
       "1": 2.2106480173533782e-05,
       "2": 0.9309849143028259,
       "3": 0.06492854654788971
      },
      "legend": {
       "0": "under 6 months",
       "1": "6 months",
       "2": "12 months",
       "3": "over 12 months"
      }
     }
    },
    "usage": {
     "input_tokens": 1408,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_t01",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 660,
   "questions": [
    {
     "qid": "team",
     "type": "choice",
     "K": 5,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 187,
     "positions": 187,
     "bucket": 256,
     "ids": [
      1,
      17,
      561,
      32909,
      2679,
      31464,
      526,
      1141,
      8414,
      730,
      1718,
      531,
      522,
      6135,
      522,
      1536,
      730,
      4905,
      535,
      1374,
      708,
      6314,
      535,
      50642,
      843,
      38765,
      770,
      2908,
      730,
      1718,
      528,
      1015,
      41518,
      535,
      61958,
      791,
      11126,
      875,
      4690,
      509,
      42120,
      521,
      859,
      6217,
      884,
      779,
      9855,
      834,
      917,
      3016,
      786,
      984,
      1512,
      19771,
      2984,
      523,
      4411,
      5060,
      8294,
      4427,
      1547,
      16620,
      803,
      730,
      2752,
      523,
      2475,
      884,
      4690,
      730,
      526,
      521,
      1203,
      859,
      1550,
      5068,
      1235,
      6377,
      523,
      859,
      4001,
      45942,
      1727,
      7493,
      810,
      1186,
      5689,
      1123,
      811,
      3452,
      1010,
      1509,
      523,
      13831,
      53467,
      779,
      49006,
      7786,
      523,
      859,
      6351,
      1236,
      1672,
      811,
      36298,
      5514,
      969,
      875,
      1547,
      6272,
      2158,
      859,
      6217,
      35754,
      521,
      7929,
      4690,
      730,
      898,
      523,
      20018,
      521,
      50642,
      18,
      28828,
      3410,
      1753,
      9936,
      1033,
      30437,
      1509,
      540,
      19,
      16,
      5363,
      800,
      535,
      61958,
      774,
      521,
      60827,
      1864,
      810,
      53467,
      592,
      20,
      19,
      16,
      5514,
      969,
      535,
      16208,
      774,
      521,
      2286,
      3670,
      810,
      11746,
      14721,
      20,
      19,
      16,
      3863,
      535,
      11309,
      4086,
      521,
      2984,
      3807,
      810,
      784,
      20231,
      20,
      19,
      16,
      10017,
      535,
      5135,
      933,
      5647,
      50022,
      20,
      19,
      16,
      1312,
      20,
      21
     ],
     "markers": [
      133,
      147,
      161,
      174,
      183
     ],
     "logits_raw": [
      5.459228515625,
      -0.8426796197891235,
      1.1409907341003418,
      -3.7750244140625,
      -2.193328619003296
     ],
     "probs_raw": [
      0.9845149517059326,
      0.0018044234020635486,
      0.013117029331624508,
      9.612519352231175e-05,
      0.00046747553278692067
     ],
     "probs": [
      0.9412418603897095,
      0.010438592173159122,
      0.04305680841207504,
      0.0012850860366597772,
      0.0039776465855538845
     ],
     "argmax_index": 0,
     "argmax": "billing",
     "top2_margin": 0.8981850519776344,
     "near_tie": false,
     "gold": "billing",
     "correct": true,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 4
     }
    },
    {
     "qid": "several_actions",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 157,
     "positions": 157,
     "bucket": 256,
     "ids": [
      1,
      17,
      561,
      32909,
      2679,
      31464,
      526,
      1141,
      8414,
      730,
      1718,
      531,
      522,
      6135,
      522,
      1536,
      730,
      4905,
      535,
      1374,
      708,
      6314,
      535,
      50642,
      843,
      38765,
      770,
      2908,
      730,
      1718,
      528,
      1015,
      41518,
      535,
      61958,
      791,
      11126,
      875,
      4690,
      509,
      42120,
      521,
      859,
      6217,
      884,
      779,
      9855,
      834,
      917,
      3016,
      786,
      984,
      1512,
      19771,
      2984,
      523,
      4411,
      5060,
      8294,
      4427,
      1547,
      16620,
      803,
      730,
      2752,
      523,
      2475,
      884,
      4690,
      730,
      526,
      521,
      1203,
      859,
      1550,
      5068,
      1235,
      6377,
      523,
      859,
      4001,
      45942,
      1727,
      7493,
      810,
      1186,
      5689,
      1123,
      811,
      3452,
      1010,
      1509,
      523,
      13831,
      53467,
      779,
      49006,
      7786,
      523,
      859,
      6351,
      1236,
      1672,
      811,
      36298,
      5514,
      969,
      875,
      1547,
      6272,
      2158,
      859,
      6217,
      35754,
      521,
      7929,
      4690,
      730,
      898,
      523,
      20018,
      521,
      50642,
      18,
      29589,
      779,
      30437,
      3452,
      875,
      1209,
      1399,
      1235,
      4272,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      135,
      147
     ],
     "logits_raw": [
      -1.5088059902191162,
      2.8384313583374023
     ],
     "probs_raw": [
      0.012777145951986313,
      0.9872227907180786
     ],
     "probs": [
      0.931430995464325,
      0.06856900453567505
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8628619909286499,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 4
     }
    },
    {
     "qid": "upset",
     "type": "score",
     "K": 4,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 166,
     "positions": 166,
     "bucket": 256,
     "ids": [
      1,
      17,
      561,
      32909,
      2679,
      31464,
      526,
      1141,
      8414,
      730,
      1718,
      531,
      522,
      6135,
      522,
      1536,
      730,
      4905,
      535,
      1374,
      708,
      6314,
      535,
      50642,
      843,
      38765,
      770,
      2908,
      730,
      1718,
      528,
      1015,
      41518,
      535,
      61958,
      791,
      11126,
      875,
      4690,
      509,
      42120,
      521,
      859,
      6217,
      884,
      779,
      9855,
      834,
      917,
      3016,
      786,
      984,
      1512,
      19771,
      2984,
      523,
      4411,
      5060,
      8294,
      4427,
      1547,
      16620,
      803,
      730,
      2752,
      523,
      2475,
      884,
      4690,
      730,
      526,
      521,
      1203,
      859,
      1550,
      5068,
      1235,
      6377,
      523,
      859,
      4001,
      45942,
      1727,
      7493,
      810,
      1186,
      5689,
      1123,
      811,
      3452,
      1010,
      1509,
      523,
      13831,
      53467,
      779,
      49006,
      7786,
      523,
      859,
      6351,
      1236,
      1672,
      811,
      36298,
      5514,
      969,
      875,
      1547,
      6272,
      2158,
      859,
      6217,
      35754,
      521,
      7929,
      4690,
      730,
      898,
      523,
      20018,
      521,
      50642,
      18,
      4023,
      30133,
      856,
      779,
      13608,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      19243,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      13545,
      876,
      35740,
      791,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      44658,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      23732,
      20,
      21
     ],
     "markers": [
      131,
      139,
      150,
      158
     ],
     "logits_raw": [
      -0.5534851551055908,
      -2.147355318069458,
      -0.1437389999628067,
      -2.5713460445404053
     ],
     "probs_raw": [
      0.35180142521858215,
      0.07146420329809189,
      0.5299661159515381,
      0.046768318861722946
     ],
     "probs": [
      0.3359356224536896,
      0.13370925188064575,
      0.4257071018218994,
      0.10464800149202347
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.08977147936820984,
     "near_tie": false,
     "gold": "1",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 4
     }
    },
    {
     "qid": "refund",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 150,
     "positions": 150,
     "bucket": 256,
     "ids": [
      1,
      17,
      561,
      32909,
      2679,
      31464,
      526,
      1141,
      8414,
      730,
      1718,
      531,
      522,
      6135,
      522,
      1536,
      730,
      4905,
      535,
      1374,
      708,
      6314,
      535,
      50642,
      843,
      38765,
      770,
      2908,
      730,
      1718,
      528,
      1015,
      41518,
      535,
      61958,
      791,
      11126,
      875,
      4690,
      509,
      42120,
      521,
      859,
      6217,
      884,
      779,
      9855,
      834,
      917,
      3016,
      786,
      984,
      1512,
      19771,
      2984,
      523,
      4411,
      5060,
      8294,
      4427,
      1547,
      16620,
      803,
      730,
      2752,
      523,
      2475,
      884,
      4690,
      730,
      526,
      521,
      1203,
      859,
      1550,
      5068,
      1235,
      6377,
      523,
      859,
      4001,
      45942,
      1727,
      7493,
      810,
      1186,
      5689,
      1123,
      811,
      3452,
      1010,
      1509,
      523,
      13831,
      53467,
      779,
      49006,
      7786,
      523,
      859,
      6351,
      1236,
      1672,
      811,
      36298,
      5514,
      969,
      875,
      1547,
      6272,
      2158,
      859,
      6217,
      35754,
      521,
      7929,
      4690,
      730,
      898,
      523,
      20018,
      521,
      50642,
      18,
      9906,
      1356,
      5940,
      540,
      19,
      16,
      4831,
      535,
      3253,
      53467,
      856,
      21968,
      20,
      19,
      16,
      3347,
      535,
      941,
      13608,
      22922,
      21090,
      875,
      4636,
      1892,
      20,
      21
     ],
     "markers": [
      129,
      138
     ],
     "logits_raw": [
      -5.6721014976501465,
      12.44875717163086
     ],
     "probs_raw": [
      1.3496180173433459e-08,
      1.0
     ],
     "probs": [
      0.9999810457229614,
      1.8929660654976033e-05
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.9999621160623064,
     "near_tie": false,
     "gold": "true",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 3,
      "B": 4
     }
    }
   ],
   "response": {
    "answers": {
     "team": {
      "type": "choice",
      "choice": "billing",
      "confidence": 0.9412418603897095,
      "probabilities": {
       "billing": 0.9412418603897095,
       "deliveries": 0.010438592173159122,
       "account": 0.04305680841207504,
       "technical": 0.0012850860366597772,
       "other": 0.0039776465855538845
      }
     },
     "several_actions": {
      "type": "noul",
      "noul": 0.931430995464325
     },
     "upset": {
      "type": "score",
      "score": 1.299067460000515,
      "confidence": 0.4257071018218994,
      "probabilities": {
       "0": 0.3359356224536896,
       "1": 0.13370925188064575,
       "2": 0.4257071018218994,
       "3": 0.10464800149202347
      },
      "legend": {
       "0": "calm",
       "1": "mildly annoyed",
       "2": "frustrated",
       "3": "angry"
      }
     },
     "refund": {
      "type": "noul",
      "noul": 0.9999810457229614
     }
    },
    "usage": {
     "input_tokens": 660,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_t04",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 753,
   "questions": [
    {
     "qid": "crash_owner",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 191,
     "positions": 191,
     "bucket": 256,
     "ids": [
      1,
      17,
      7491,
      1983,
      1395,
      57394,
      1141,
      9573,
      3410,
      2472,
      6688,
      521,
      22153,
      708,
      41678,
      535,
      835,
      14007,
      521,
      6623,
      521,
      1157,
      1837,
      860,
      521,
      911,
      14851,
      708,
      542,
      14007,
      535,
      10414,
      779,
      49550,
      39198,
      8660,
      536,
      13668,
      884,
      2289,
      4202,
      2076,
      61590,
      819,
      30819,
      535,
      779,
      21064,
      884,
      33990,
      875,
      18204,
      730,
      1374,
      7039,
      856,
      2510,
      2714,
      536,
      950,
      1011,
      30961,
      936,
      1550,
      884,
      1235,
      1927,
      9161,
      523,
      18578,
      791,
      3032,
      779,
      6625,
      5756,
      12116,
      779,
      2341,
      9161,
      884,
      16012,
      819,
      2435,
      1837,
      860,
      535,
      6872,
      8788,
      875,
      730,
      529,
      523,
      527,
      43360,
      536,
      3988,
      5743,
      988,
      911,
      14851,
      819,
      547,
      14851,
      535,
      41669,
      13793,
      9167,
      536,
      1188,
      8685,
      1157,
      1837,
      860,
      779,
      19771,
      5592,
      5196,
      5743,
      4008,
      819,
      15888,
      6015,
      535,
      6872,
      730,
      529,
      523,
      527,
      14967,
      988,
      22045,
      811,
      17980,
      1929,
      803,
      779,
      33990,
      21064,
      523,
      6623,
      40917,
      779,
      21064,
      536,
      835,
      14007,
      1188,
      9256,
      916,
      1889,
      27968,
      523,
      18,
      17408,
      40917,
      779,
      33990,
      21064,
      540,
      19,
      16,
      768,
      14007,
      535,
      835,
      14007,
      20,
      19,
      16,
      7572,
      535,
      6623,
      20,
      19,
      16,
      979,
      47222,
      535,
      1157,
      1837,
      860,
      20,
      19,
      16,
      3783,
      582,
      535,
      911,
      14851,
      20,
      21
     ],
     "markers": [
      160,
      168,
      174,
      183
     ],
     "logits_raw": [
      -1.1562695503234863,
      -0.11443820595741272,
      -2.6990933418273926,
      -4.24807596206665
     ],
     "probs_raw": [
      0.2442837953567505,
      0.692398726940155,
      0.05222216248512268,
      0.011095328256487846
     ],
     "probs": [
      0.2819421887397766,
      0.593431293964386,
      0.09365379065275192,
      0.030972765758633614
     ],
     "argmax_index": 1,
     "argmax": "ben",
     "top2_margin": 0.3114891052246094,
     "near_tie": false,
     "gold": "ben",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 4
     }
    },
    {
     "qid": "blocked",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 182,
     "positions": 182,
     "bucket": 256,
     "ids": [
      1,
      17,
      7491,
      1983,
      1395,
      57394,
      1141,
      9573,
      3410,
      2472,
      6688,
      521,
      22153,
      708,
      41678,
      535,
      835,
      14007,
      521,
      6623,
      521,
      1157,
      1837,
      860,
      521,
      911,
      14851,
      708,
      542,
      14007,
      535,
      10414,
      779,
      49550,
      39198,
      8660,
      536,
      13668,
      884,
      2289,
      4202,
      2076,
      61590,
      819,
      30819,
      535,
      779,
      21064,
      884,
      33990,
      875,
      18204,
      730,
      1374,
      7039,
      856,
      2510,
      2714,
      536,
      950,
      1011,
      30961,
      936,
      1550,
      884,
      1235,
      1927,
      9161,
      523,
      18578,
      791,
      3032,
      779,
      6625,
      5756,
      12116,
      779,
      2341,
      9161,
      884,
      16012,
      819,
      2435,
      1837,
      860,
      535,
      6872,
      8788,
      875,
      730,
      529,
      523,
      527,
      43360,
      536,
      3988,
      5743,
      988,
      911,
      14851,
      819,
      547,
      14851,
      535,
      41669,
      13793,
      9167,
      536,
      1188,
      8685,
      1157,
      1837,
      860,
      779,
      19771,
      5592,
      5196,
      5743,
      4008,
      819,
      15888,
      6015,
      535,
      6872,
      730,
      529,
      523,
      527,
      14967,
      988,
      22045,
      811,
      17980,
      1929,
      803,
      779,
      33990,
      21064,
      523,
      6623,
      40917,
      779,
      21064,
      536,
      835,
      14007,
      1188,
      9256,
      916,
      1889,
      27968,
      523,
      18,
      4940,
      1523,
      3410,
      5484,
      26781,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      160,
      172
     ],
     "logits_raw": [
      2.0561749935150146,
      -0.3539355993270874
     ],
     "probs_raw": [
      0.9175950288772583,
      0.0824049636721611
     ],
     "probs": [
      0.19056159257888794,
      0.8094383478164673
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.6188767552375793,
     "near_tie": false,
     "gold": "true",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 3,
      "B": 4
     }
    },
    {
     "qid": "release_day",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 192,
     "positions": 192,
     "bucket": 256,
     "ids": [
      1,
      17,
      7491,
      1983,
      1395,
      57394,
      1141,
      9573,
      3410,
      2472,
      6688,
      521,
      22153,
      708,
      41678,
      535,
      835,
      14007,
      521,
      6623,
      521,
      1157,
      1837,
      860,
      521,
      911,
      14851,
      708,
      542,
      14007,
      535,
      10414,
      779,
      49550,
      39198,
      8660,
      536,
      13668,
      884,
      2289,
      4202,
      2076,
      61590,
      819,
      30819,
      535,
      779,
      21064,
      884,
      33990,
      875,
      18204,
      730,
      1374,
      7039,
      856,
      2510,
      2714,
      536,
      950,
      1011,
      30961,
      936,
      1550,
      884,
      1235,
      1927,
      9161,
      523,
      18578,
      791,
      3032,
      779,
      6625,
      5756,
      12116,
      779,
      2341,
      9161,
      884,
      16012,
      819,
      2435,
      1837,
      860,
      535,
      6872,
      8788,
      875,
      730,
      529,
      523,
      527,
      43360,
      536,
      3988,
      5743,
      988,
      911,
      14851,
      819,
      547,
      14851,
      535,
      41669,
      13793,
      9167,
      536,
      1188,
      8685,
      1157,
      1837,
      860,
      779,
      19771,
      5592,
      5196,
      5743,
      4008,
      819,
      15888,
      6015,
      535,
      6872,
      730,
      529,
      523,
      527,
      14967,
      988,
      22045,
      811,
      17980,
      1929,
      803,
      779,
      33990,
      21064,
      523,
      6623,
      40917,
      779,
      21064,
      536,
      835,
      14007,
      1188,
      9256,
      916,
      1889,
      27968,
      523,
      18,
      4453,
      856,
      6872,
      730,
      529,
      523,
      527,
      2166,
      13573,
      540,
      19,
      16,
      823,
      18908,
      535,
      22045,
      20,
      19,
      16,
      2261,
      9008,
      535,
      16012,
      20,
      19,
      16,
      10954,
      892,
      535,
      17980,
      20,
      19,
      16,
      10654,
      535,
      4524,
      8091,
      20,
      21
     ],
     "markers": [
      164,
      171,
      178,
      185
     ],
     "logits_raw": [
      -0.7694559693336487,
      -1.497985601425171,
      4.049992561340332,
      -2.367877960205078
     ],
     "probs_raw": [
      0.007962951436638832,
      0.0038430634886026382,
      0.9865837693214417,
      0.0016102311201393604
     ],
     "probs": [
      0.030133897438645363,
      0.01790776662528515,
      0.9423384070396423,
      0.009619934484362602
     ],
     "argmax_index": 2,
     "argmax": "monday",
     "top2_margin": 0.912204509600997,
     "near_tie": false,
     "gold": "monday",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 4
     }
    },
    {
     "qid": "blockers",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 188,
     "positions": 188,
     "bucket": 256,
     "ids": [
      1,
      17,
      7491,
      1983,
      1395,
      57394,
      1141,
      9573,
      3410,
      2472,
      6688,
      521,
      22153,
      708,
      41678,
      535,
      835,
      14007,
      521,
      6623,
      521,
      1157,
      1837,
      860,
      521,
      911,
      14851,
      708,
      542,
      14007,
      535,
      10414,
      779,
      49550,
      39198,
      8660,
      536,
      13668,
      884,
      2289,
      4202,
      2076,
      61590,
      819,
      30819,
      535,
      779,
      21064,
      884,
      33990,
      875,
      18204,
      730,
      1374,
      7039,
      856,
      2510,
      2714,
      536,
      950,
      1011,
      30961,
      936,
      1550,
      884,
      1235,
      1927,
      9161,
      523,
      18578,
      791,
      3032,
      779,
      6625,
      5756,
      12116,
      779,
      2341,
      9161,
      884,
      16012,
      819,
      2435,
      1837,
      860,
      535,
      6872,
      8788,
      875,
      730,
      529,
      523,
      527,
      43360,
      536,
      3988,
      5743,
      988,
      911,
      14851,
      819,
      547,
      14851,
      535,
      41669,
      13793,
      9167,
      536,
      1188,
      8685,
      1157,
      1837,
      860,
      779,
      19771,
      5592,
      5196,
      5743,
      4008,
      819,
      15888,
      6015,
      535,
      6872,
      730,
      529,
      523,
      527,
      14967,
      988,
      22045,
      811,
      17980,
      1929,
      803,
      779,
      33990,
      21064,
      523,
      6623,
      40917,
      779,
      21064,
      536,
      835,
      14007,
      1188,
      9256,
      916,
      1889,
      27968,
      523,
      18,
      4023,
      1697,
      1519,
      2312,
      768,
      4409,
      770,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      10735,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      1235,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      1547,
      933,
      1209,
      20,
      21
     ],
     "markers": [
      162,
      170,
      178
     ],
     "logits_raw": [
      -2.957319974899292,
      -4.160699367523193,
      -2.986135482788086
     ],
     "probs_raw": [
      0.44018465280532837,
      0.13213378190994263,
      0.42768150568008423
     ],
     "probs": [
      0.40285491943359375,
      0.20094417035579681,
      0.39620086550712585
     ],
     "argmax_index": 0,
     "argmax": "0",
     "top2_margin": 0.0066540539264678955,
     "near_tie": true,
     "gold": "1",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 4
     }
    }
   ],
   "response": {
    "answers": {
     "crash_owner": {
      "type": "choice",
      "choice": "ben",
      "confidence": 0.593431293964386,
      "probabilities": {
       "aiko": 0.2819421887397766,
       "ben": 0.593431293964386,
       "chloe": 0.09365379065275192,
       "femi": 0.030972765758633614
      }
     },
     "blocked": {
      "type": "noul",
      "noul": 0.19056159257888794
     },
     "release_day": {
      "type": "choice",
      "choice": "monday",
      "confidence": 0.9423384070396423,
      "probabilities": {
       "thursday": 0.030133897438645363,
       "friday": 0.01790776662528515,
       "monday": 0.9423384070396423,
       "unknown": 0.009619934484362602
      }
     },
     "blockers": {
      "type": "score",
      "score": 0.9933459013700485,
      "confidence": 0.40285491943359375,
      "probabilities": {
       "0": 0.40285491943359375,
       "1": 0.20094417035579681,
       "2": 0.39620086550712585
      },
      "legend": {
       "0": "none",
       "1": "one",
       "2": "two or more"
      }
     }
    },
    "usage": {
     "input_tokens": 753,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_t06",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 727,
   "questions": [
    {
     "qid": "first_request",
     "type": "choice",
     "K": 10,
     "calibrate": true,
     "temperature_key": "choice:6-10",
     "T": 1.1751071214675903,
     "prefix": 0,
     "n_ids": 294,
     "positions": 294,
     "bucket": 512,
     "ids": [
      1,
      17,
      568,
      913,
      535,
      5843,
      570,
      54053,
      843,
      5652,
      730,
      25454,
      3271,
      15012,
      521,
      779,
      16542,
      797,
      1727,
      5362,
      7404,
      1901,
      3822,
      810,
      936,
      1090,
      34517,
      708,
      568,
      913,
      535,
      6135,
      570,
      13660,
      26862,
      843,
      564,
      959,
      54898,
      2176,
      13901,
      3271,
      859,
      6217,
      29640,
      811,
      8498,
      896,
      510,
      859,
      6016,
      8685,
      5791,
      988,
      12003,
      2084,
      3599,
      819,
      568,
      913,
      535,
      6326,
      570,
      54053,
      535,
      14165,
      523,
      1236,
      1011,
      859,
      3089,
      779,
      5362,
      3032,
      730,
      527,
      5806,
      27968,
      540,
      1727,
      8842,
      856,
      963,
      730,
      528,
      708,
      568,
      913,
      535,
      6468,
      570,
      13660,
      26862,
      535,
      25957,
      64210,
      3032,
      730,
      1536,
      535,
      1056,
      856,
      3023,
      875,
      730,
      898,
      14076,
      521,
      6283,
      859,
      1614,
      936,
      1784,
      568,
      913,
      535,
      6468,
      570,
      54053,
      535,
      18672,
      8694,
      708,
      568,
      913,
      535,
      1698,
      570,
      54053,
      535,
      2510,
      31731,
      4010,
      810,
      936,
      1090,
      5629,
      52125,
      708,
      568,
      913,
      535,
      2117,
      570,
      13660,
      26862,
      535,
      4998,
      6650,
      521,
      1494,
      59063,
      856,
      884,
      2772,
      1868,
      523,
      1405,
      1188,
      874,
      916,
      1010,
      797,
      730,
      1155,
      5112,
      521,
      810,
      1023,
      1011,
      4876,
      1010,
      811,
      5362,
      730,
      26740,
      1288,
      1010,
      6863,
      523,
      18,
      3493,
      2172,
      779,
      21522,
      3452,
      875,
      1509,
      540,
      19,
      16,
      5362,
      52799,
      535,
      9267,
      933,
      20056,
      811,
      779,
      5362,
      20,
      19,
      16,
      5019,
      45625,
      1066,
      535,
      41072,
      779,
      5362,
      2919,
      1399,
      13376,
      20,
      19,
      16,
      25583,
      1706,
      535,
      38332,
      25583,
      1706,
      933,
      56043,
      20,
      19,
      16,
      12003,
      535,
      39781,
      797,
      779,
      5362,
      856,
      11694,
      20,
      19,
      16,
      5363,
      800,
      535,
      835,
      3952,
      1337,
      779,
      5363,
      20,
      19,
      16,
      2643,
      800,
      572,
      6386,
      535,
      15015,
      779,
      14721,
      933,
      5362,
      2432,
      20,
      19,
      16,
      5807,
      43840,
      535,
      39781,
      953,
      5807,
      20,
      19,
      16,
      10925,
      535,
      835,
      10925,
      38203,
      20,
      19,
      16,
      5600,
      535,
      18525,
      916,
      768,
      47502,
      933,
      8842,
      20,
      19,
      16,
      1312,
      20,
      21
     ],
     "markers": [
      190,
      202,
      215,
      226,
      237,
      248,
      262,
      271,
      279,
      290
     ],
     "logits_raw": [
      -3.301970958709717,
      -0.8134485483169556,
      -5.719726085662842,
      1.0345563888549805,
      -6.834534168243408,
      -2.957244634628296,
      -8.278870582580566,
      -11.795267105102539,
      -2.055696725845337,
      -1.509829044342041
     ],
     "probs_raw": [
      0.009950069710612297,
      0.11983334273099899,
      0.000886764726601541,
      0.7605994939804077,
      0.00029083952540531754,
      0.014045554213225842,
      6.860979192424566e-05,
      2.0381423837534385e-06,
      0.03460000455379486,
      0.05972325801849365
     ],
     "probs": [
      0.01712716370820999,
      0.14236073195934296,
      0.0021884406451135874,
      0.6860777735710144,
      0.000847472867462784,
      0.022966191172599792,
      0.0002479304384905845,
      1.2437837540346663e-05,
      0.04946316033601761,
      0.0787087231874466
     ],
     "argmax_index": 3,
     "argmax": "maintenance",
     "top2_margin": 0.5437170416116714,
     "near_tie": false,
     "gold": "maintenance",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    },
    {
     "qid": "fixed",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 217,
     "positions": 217,
     "bucket": 256,
     "ids": [
      1,
      17,
      568,
      913,
      535,
      5843,
      570,
      54053,
      843,
      5652,
      730,
      25454,
      3271,
      15012,
      521,
      779,
      16542,
      797,
      1727,
      5362,
      7404,
      1901,
      3822,
      810,
      936,
      1090,
      34517,
      708,
      568,
      913,
      535,
      6135,
      570,
      13660,
      26862,
      843,
      564,
      959,
      54898,
      2176,
      13901,
      3271,
      859,
      6217,
      29640,
      811,
      8498,
      896,
      510,
      859,
      6016,
      8685,
      5791,
      988,
      12003,
      2084,
      3599,
      819,
      568,
      913,
      535,
      6326,
      570,
      54053,
      535,
      14165,
      523,
      1236,
      1011,
      859,
      3089,
      779,
      5362,
      3032,
      730,
      527,
      5806,
      27968,
      540,
      1727,
      8842,
      856,
      963,
      730,
      528,
      708,
      568,
      913,
      535,
      6468,
      570,
      13660,
      26862,
      535,
      25957,
      64210,
      3032,
      730,
      1536,
      535,
      1056,
      856,
      3023,
      875,
      730,
      898,
      14076,
      521,
      6283,
      859,
      1614,
      936,
      1784,
      568,
      913,
      535,
      6468,
      570,
      54053,
      535,
      18672,
      8694,
      708,
      568,
      913,
      535,
      1698,
      570,
      54053,
      535,
      2510,
      31731,
      4010,
      810,
      936,
      1090,
      5629,
      52125,
      708,
      568,
      913,
      535,
      2117,
      570,
      13660,
      26862,
      535,
      4998,
      6650,
      521,
      1494,
      59063,
      856,
      884,
      2772,
      1868,
      523,
      1405,
      1188,
      874,
      916,
      1010,
      797,
      730,
      1155,
      5112,
      521,
      810,
      1023,
      1011,
      4876,
      1010,
      811,
      5362,
      730,
      26740,
      1288,
      1010,
      6863,
      523,
      18,
      24860,
      779,
      16542,
      3220,
      1381,
      9167,
      968,
      779,
      1764,
      803,
      779,
      22537,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      195,
      207
     ],
     "logits_raw": [
      2.7867722511291504,
      -0.9277684688568115
     ],
     "probs_raw": [
      0.9762129783630371,
      0.02378701977431774
     ],
     "probs": [
      0.09716012328863144,
      0.9028398990631104
     ],
     "argmax_index": 1,
     "argmax": "false",
     "top2_margin": 0.8056797757744789,
     "near_tie": false,
     "gold": "false",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "patience",
     "type": "score",
     "K": 3,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 216,
     "positions": 216,
     "bucket": 256,
     "ids": [
      1,
      17,
      568,
      913,
      535,
      5843,
      570,
      54053,
      843,
      5652,
      730,
      25454,
      3271,
      15012,
      521,
      779,
      16542,
      797,
      1727,
      5362,
      7404,
      1901,
      3822,
      810,
      936,
      1090,
      34517,
      708,
      568,
      913,
      535,
      6135,
      570,
      13660,
      26862,
      843,
      564,
      959,
      54898,
      2176,
      13901,
      3271,
      859,
      6217,
      29640,
      811,
      8498,
      896,
      510,
      859,
      6016,
      8685,
      5791,
      988,
      12003,
      2084,
      3599,
      819,
      568,
      913,
      535,
      6326,
      570,
      54053,
      535,
      14165,
      523,
      1236,
      1011,
      859,
      3089,
      779,
      5362,
      3032,
      730,
      527,
      5806,
      27968,
      540,
      1727,
      8842,
      856,
      963,
      730,
      528,
      708,
      568,
      913,
      535,
      6468,
      570,
      13660,
      26862,
      535,
      25957,
      64210,
      3032,
      730,
      1536,
      535,
      1056,
      856,
      3023,
      875,
      730,
      898,
      14076,
      521,
      6283,
      859,
      1614,
      936,
      1784,
      568,
      913,
      535,
      6468,
      570,
      54053,
      535,
      18672,
      8694,
      708,
      568,
      913,
      535,
      1698,
      570,
      54053,
      535,
      2510,
      31731,
      4010,
      810,
      936,
      1090,
      5629,
      52125,
      708,
      568,
      913,
      535,
      2117,
      570,
      13660,
      26862,
      535,
      4998,
      6650,
      521,
      1494,
      59063,
      856,
      884,
      2772,
      1868,
      523,
      1405,
      1188,
      874,
      916,
      1010,
      797,
      730,
      1155,
      5112,
      521,
      810,
      1023,
      1011,
      4876,
      1010,
      811,
      5362,
      730,
      26740,
      1288,
      1010,
      6863,
      523,
      18,
      4023,
      5087,
      856,
      779,
      21522,
      968,
      779,
      1764,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      5087,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      2269,
      51908,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      23732,
      20,
      21
     ],
     "markers": [
      191,
      199,
      208
     ],
     "logits_raw": [
      -0.20327231287956238,
      -2.439265489578247,
      -3.508890390396118
     ],
     "probs_raw": [
      0.8744603395462036,
      0.09346750378608704,
      0.03207217901945114
     ],
     "probs": [
      0.7029393911361694,
      0.19303540885448456,
      0.10402525961399078
     ],
     "argmax_index": 0,
     "argmax": "0",
     "top2_margin": 0.5099039822816849,
     "near_tie": false,
     "gold": "1",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "first_request": {
      "type": "choice",
      "choice": "maintenance",
      "confidence": 0.6860777735710144,
      "probabilities": {
       "room_service": 0.01712716370820999,
       "late_checkout": 0.14236073195934296,
       "towels": 0.0021884406451135874,
       "maintenance": 0.6860777735710144,
       "billing": 0.000847472867462784,
       "booking_change": 0.022966191172599792,
       "lost_item": 0.0002479304384905845,
       "noise": 1.2437837540346663e-05,
       "transport": 0.04946316033601761,
       "other": 0.0787087231874466
      }
     },
     "fixed": {
      "type": "noul",
      "noul": 0.09716012328863144
     },
     "patience": {
      "type": "score",
      "score": 0.4010859280824661,
      "confidence": 0.7029393911361694,
      "probabilities": {
       "0": 0.7029393911361694,
       "1": 0.19303540885448456,
       "2": 0.10402525961399078
      },
      "legend": {
       "0": "patient",
       "1": "impatient",
       "2": "angry"
      }
     }
    },
    "usage": {
     "input_tokens": 727,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_t07",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 199,
   "questions": [
    {
     "qid": "slot",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 199,
     "positions": 199,
     "bucket": 256,
     "ids": [
      1,
      17,
      556,
      820,
      799,
      12890,
      33496,
      1141,
      2692,
      793,
      18284,
      19094,
      521,
      16012,
      730,
      1443,
      535,
      3167,
      708,
      13694,
      5052,
      521,
      1033,
      856,
      54017,
      523,
      859,
      1052,
      768,
      16084,
      20328,
      884,
      22092,
      963,
      730,
      534,
      535,
      1664,
      916,
      3583,
      2412,
      786,
      523,
      39781,
      4010,
      1287,
      963,
      1336,
      521,
      1101,
      859,
      1595,
      811,
      4876,
      936,
      523,
      859,
      1011,
      1112,
      22153,
      1720,
      730,
      1386,
      535,
      1056,
      933,
      1523,
      1394,
      884,
      22045,
      8532,
      523,
      13831,
      1868,
      1123,
      1892,
      811,
      10547,
      523,
      859,
      1011,
      1901,
      3999,
      12645,
      2013,
      779,
      2285,
      521,
      1101,
      1720,
      730,
      1133,
      535,
      1056,
      856,
      2238,
      523,
      28296,
      1010,
      12588,
      22935,
      42474,
      3280,
      3025,
      535,
      837,
      1037,
      730,
      1632,
      535,
      1056,
      521,
      16666,
      730,
      1386,
      535,
      1664,
      521,
      16666,
      730,
      1417,
      535,
      1056,
      521,
      1097,
      594,
      730,
      4528,
      535,
      1664,
      521,
      1097,
      594,
      730,
      1374,
      535,
      1664,
      523,
      18,
      28828,
      803,
      1518,
      3262,
      42474,
      25113,
      779,
      59850,
      540,
      19,
      16,
      766,
      1037,
      572,
      7984,
      525,
      535,
      22092,
      730,
      1632,
      535,
      1056,
      20,
      19,
      16,
      28782,
      572,
      11610,
      525,
      535,
      22153,
      730,
      1386,
      535,
      1664,
      20,
      19,
      16,
      823,
      594,
      572,
      10293,
      525,
      535,
      22045,
      730,
      1374,
      535,
      1664,
      20,
      19,
      16,
      10735,
      535,
      11357,
      803,
      1377,
      25113,
      20,
      21
     ],
     "markers": [
      149,
      163,
      176,
      190
     ],
     "logits_raw": [
      0.19124549627304077,
      -1.5787943601608276,
      1.2088134288787842,
      -0.8737671971321106
     ],
     "probs_raw": [
      0.23356255888938904,
      0.039781827479600906,
      0.6461410522460938,
      0.08051449805498123
     ],
     "probs": [
      0.2618938982486725,
      0.07396066188812256,
      0.5417618751525879,
      0.12238364666700363
     ],
     "argmax_index": 2,
     "argmax": "thu_1230",
     "top2_margin": 0.2798679769039154,
     "near_tie": false,
     "gold": "wed_1530",
     "correct": false,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "slot": {
      "type": "choice",
      "choice": "thu_1230",
      "confidence": 0.5417618751525879,
      "probabilities": {
       "tue_1100": 0.2618938982486725,
       "wed_1530": 0.07396066188812256,
       "thu_1230": 0.5417618751525879,
       "none": 0.12238364666700363
      }
     }
    },
    "usage": {
     "input_tokens": 199,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "own_t10",
   "mode": "text",
   "source": "own_text",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 490,
   "questions": [
    {
     "qid": "topic",
     "type": "choice",
     "K": 5,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 163,
     "positions": 163,
     "bucket": 256,
     "ids": [
      1,
      17,
      3097,
      779,
      17612,
      11380,
      521,
      9855,
      834,
      917,
      7920,
      509,
      550,
      4238,
      797,
      9645,
      730,
      534,
      810,
      859,
      1283,
      4862,
      521,
      875,
      779,
      4976,
      1394,
      1033,
      5805,
      521,
      1337,
      779,
      10925,
      988,
      779,
      21780,
      7494,
      797,
      779,
      39932,
      523,
      941,
      12719,
      938,
      1593,
      3032,
      1720,
      44396,
      521,
      810,
      779,
      823,
      51143,
      4834,
      1287,
      1626,
      779,
      8913,
      803,
      1727,
      32874,
      523,
      859,
      1052,
      5751,
      875,
      779,
      10170,
      4934,
      811,
      874,
      6235,
      811,
      730,
      4972,
      535,
      1056,
      522,
      2019,
      535,
      1056,
      521,
      906,
      779,
      3622,
      7360,
      2992,
      523,
      2182,
      6300,
      3807,
      968,
      779,
      1764,
      803,
      3280,
      3025,
      521,
      859,
      1188,
      9920,
      936,
      916,
      779,
      10698,
      516,
      15671,
      523,
      509,
      566,
      2670,
      521,
      2161,
      887,
      3309,
      843,
      40723,
      730,
      534,
      518,
      18,
      3493,
      856,
      779,
      7627,
      1337,
      540,
      19,
      16,
      10925,
      535,
      63144,
      20,
      19,
      16,
      38183,
      535,
      4274,
      5569,
      20,
      19,
      16,
      16429,
      535,
      55657,
      933,
      16620,
      20,
      19,
      16,
      23171,
      535,
      4514,
      800,
      20,
      19,
      16,
      22906,
      535,
      873,
      1808,
      20,
      21
     ],
     "markers": [
      128,
      134,
      141,
      149,
      156
     ],
     "logits_raw": [
      6.030937671661377,
      -7.381817817687988,
      -4.890927314758301,
      -10.31625747680664,
      -10.759885787963867
     ],
     "probs_raw": [
      0.9999804496765137,
      1.495910510129761e-06,
      1.805866304493975e-05,
      7.952342429007331e-08,
      5.103049005583671e-08
     ],
     "probs": [
      0.9995075464248657,
      6.897762068547308e-05,
      0.00040875584818422794,
      8.47906540002441e-06,
      6.176179795147618e-06
     ],
     "argmax_index": 0,
     "argmax": "noise",
     "top2_margin": 0.9990987905766815,
     "near_tie": false,
     "gold": "noise",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 1,
      "B": 3
     }
    },
    {
     "qid": "first_time",
     "type": "noul",
     "K": 2,
     "calibrate": true,
     "temperature_key": "noul:2",
     "T": 1.6663223505020142,
     "prefix": 0,
     "n_ids": 154,
     "positions": 154,
     "bucket": 256,
     "ids": [
      1,
      17,
      3097,
      779,
      17612,
      11380,
      521,
      9855,
      834,
      917,
      7920,
      509,
      550,
      4238,
      797,
      9645,
      730,
      534,
      810,
      859,
      1283,
      4862,
      521,
      875,
      779,
      4976,
      1394,
      1033,
      5805,
      521,
      1337,
      779,
      10925,
      988,
      779,
      21780,
      7494,
      797,
      779,
      39932,
      523,
      941,
      12719,
      938,
      1593,
      3032,
      1720,
      44396,
      521,
      810,
      779,
      823,
      51143,
      4834,
      1287,
      1626,
      779,
      8913,
      803,
      1727,
      32874,
      523,
      859,
      1052,
      5751,
      875,
      779,
      10170,
      4934,
      811,
      874,
      6235,
      811,
      730,
      4972,
      535,
      1056,
      522,
      2019,
      535,
      1056,
      521,
      906,
      779,
      3622,
      7360,
      2992,
      523,
      2182,
      6300,
      3807,
      968,
      779,
      1764,
      803,
      3280,
      3025,
      521,
      859,
      1188,
      9920,
      936,
      916,
      779,
      10698,
      516,
      15671,
      523,
      509,
      566,
      2670,
      521,
      2161,
      887,
      3309,
      843,
      40723,
      730,
      534,
      518,
      18,
      4940,
      1033,
      779,
      1509,
      1394,
      779,
      9978,
      6537,
      2190,
      540,
      19,
      16,
      4831,
      535,
      1295,
      521,
      779,
      8294,
      2177,
      1014,
      3965,
      20,
      19,
      16,
      3347,
      535,
      18672,
      521,
      779,
      8294,
      10995,
      20,
      21
     ],
     "markers": [
      132,
      144
     ],
     "logits_raw": [
      -0.21387678384780884,
      4.600820541381836
     ],
     "probs_raw": [
      0.008044439367949963,
      0.991955578327179
     ],
     "probs": [
      0.9473206996917725,
      0.05267929285764694
     ],
     "argmax_index": 0,
     "argmax": "true",
     "top2_margin": 0.8946414068341255,
     "near_tie": false,
     "gold": "false",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 2,
      "B": 3
     }
    },
    {
     "qid": "escalation",
     "type": "score",
     "K": 4,
     "calibrate": true,
     "temperature_key": "score:3-5",
     "T": 1.7301132678985596,
     "prefix": 0,
     "n_ids": 173,
     "positions": 173,
     "bucket": 256,
     "ids": [
      1,
      17,
      3097,
      779,
      17612,
      11380,
      521,
      9855,
      834,
      917,
      7920,
      509,
      550,
      4238,
      797,
      9645,
      730,
      534,
      810,
      859,
      1283,
      4862,
      521,
      875,
      779,
      4976,
      1394,
      1033,
      5805,
      521,
      1337,
      779,
      10925,
      988,
      779,
      21780,
      7494,
      797,
      779,
      39932,
      523,
      941,
      12719,
      938,
      1593,
      3032,
      1720,
      44396,
      521,
      810,
      779,
      823,
      51143,
      4834,
      1287,
      1626,
      779,
      8913,
      803,
      1727,
      32874,
      523,
      859,
      1052,
      5751,
      875,
      779,
      10170,
      4934,
      811,
      874,
      6235,
      811,
      730,
      4972,
      535,
      1056,
      522,
      2019,
      535,
      1056,
      521,
      906,
      779,
      3622,
      7360,
      2992,
      523,
      2182,
      6300,
      3807,
      968,
      779,
      1764,
      803,
      3280,
      3025,
      521,
      859,
      1188,
      9920,
      936,
      916,
      779,
      10698,
      516,
      15671,
      523,
      509,
      566,
      2670,
      521,
      2161,
      887,
      3309,
      843,
      40723,
      730,
      534,
      518,
      18,
      4023,
      3414,
      856,
      779,
      9978,
      9217,
      811,
      2337,
      1033,
      540,
      19,
      16,
      2119,
      730,
      525,
      535,
      1295,
      3759,
      3801,
      20,
      19,
      16,
      2119,
      730,
      526,
      535,
      768,
      36685,
      20,
      19,
      16,
      2119,
      730,
      527,
      535,
      779,
      10698,
      516,
      15671,
      20,
      19,
      16,
      2119,
      730,
      528,
      535,
      768,
      23749,
      933,
      7326,
      20,
      21
     ],
     "markers": [
      132,
      142,
      151,
      162
     ],
     "logits_raw": [
      -2.038007974624634,
      -1.6411914825439453,
      4.013549327850342,
      -8.809741973876953
     ],
     "probs_raw": [
      0.0023404823150485754,
      0.003480492625385523,
      0.9941763877868652,
      2.681505065993406e-06
     ],
     "probs": [
      0.02831273339688778,
      0.035611558705568314,
      0.9355105757713318,
      0.000565113325137645
     ],
     "argmax_index": 2,
     "argmax": "2",
     "top2_margin": 0.8998990170657635,
     "near_tie": false,
     "gold": "2",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 3
     }
    }
   ],
   "response": {
    "answers": {
     "topic": {
      "type": "choice",
      "choice": "noise",
      "confidence": 0.9995075464248657,
      "probabilities": {
       "noise": 0.9995075464248657,
       "repairs": 6.897762068547308e-05,
       "rent": 0.00040875584818422794,
       "parking": 8.47906540002441e-06,
       "pets": 6.176179795147618e-06
      }
     },
     "first_time": {
      "type": "noul",
      "noul": 0.9473206996917725
     },
     "escalation": {
      "type": "score",
      "score": 1.9083280502236448,
      "confidence": 0.9355105757713318,
      "probabilities": {
       "0": 0.02831273339688778,
       "1": 0.035611558705568314,
       "2": 0.9355105757713318,
       "3": 0.000565113325137645
      },
      "legend": {
       "0": "no further step",
       "1": "a reminder",
       "2": "the residents' committee",
       "3": "a lawyer or court"
      }
     }
    },
    "usage": {
     "input_tokens": 490,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_005dcf6d2c0f21d64796",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 98,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 98,
     "positions": 98,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2681,
      28117,
      1509,
      6767,
      768,
      8153,
      777,
      811,
      7157,
      779,
      1530,
      793,
      523,
      13629,
      779,
      5089,
      1689,
      22922,
      46159,
      896,
      15001,
      523,
      3253,
      2919,
      6177,
      9647,
      523,
      18,
      14875,
      779,
      2681,
      28117,
      1090,
      10500,
      15001,
      6177,
      536,
      768,
      28687,
      32491,
      1706,
      779,
      7045,
      8825,
      523,
      3354,
      779,
      8153,
      777,
      7157,
      779,
      1530,
      793,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      58,
      69,
      81
     ],
     "logits_raw": [
      2.0582985877990723,
      1.225986361503601,
      2.4099020957946777
     ],
     "probs_raw": [
      0.3500926196575165,
      0.15230506658554077,
      0.49760234355926514
     ],
     "probs": [
      0.35244399309158325,
      0.19448202848434448,
      0.4530739486217499
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.10062995553016663,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4530739486217499,
      "probabilities": {
       "permitted": 0.35244399309158325,
       "prohibited": 0.19448202848434448,
       "insufficient": 0.4530739486217499
      }
     }
    },
    "usage": {
     "input_tokens": 98,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_00bb4cc043f7d2b288d4",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      12153,
      4943,
      876,
      22558,
      9365,
      779,
      11256,
      8355,
      523,
      18,
      18693,
      902,
      3863,
      2853,
      1550,
      1403,
      779,
      12153,
      3575,
      6229,
      9365,
      896,
      2853,
      523,
      3354,
      779,
      11256,
      8355,
      874,
      7093,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      35,
      52,
      64
     ],
     "logits_raw": [
      2.306075096130371,
      0.6486483812332153,
      4.865217208862305
     ],
     "probs_raw": [
      0.07084484398365021,
      0.013505072332918644,
      0.9156500697135925
     ],
     "probs": [
      0.1328362673521042,
      0.04065634310245514,
      0.8265073895454407
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.6936711221933365,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.8265073895454407,
      "probabilities": {
       "insufficient": 0.1328362673521042,
       "prohibited": 0.04065634310245514,
       "permitted": 0.8265073895454407
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_02f2b95a5c5a4777a45c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 82,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 82,
     "positions": 82,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      856,
      779,
      16495,
      57490,
      810,
      856,
      11649,
      3222,
      811,
      9936,
      1033,
      3869,
      523,
      46050,
      931,
      879,
      856,
      779,
      3981,
      3321,
      57490,
      810,
      856,
      1236,
      3222,
      811,
      9936,
      936,
      523,
      18,
      52739,
      835,
      19148,
      835,
      856,
      3222,
      536,
      7056,
      879,
      1550,
      1403,
      835,
      856,
      12359,
      810,
      879,
      856,
      3222,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      55,
      63,
      71
     ],
     "logits_raw": [
      2.056809425354004,
      1.9211969375610352,
      0.8836579918861389
     ],
     "probs_raw": [
      0.4581752121448517,
      0.40006986260414124,
      0.14175499975681305
     ],
     "probs": [
      0.4273075461387634,
      0.38785475492477417,
      0.18483762443065643
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.03945279121398926,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.4273075461387634,
      "probabilities": {
       "B": 0.4273075461387634,
       "A": 0.38785475492477417,
       "insufficient": 0.18483762443065643
      }
     }
    },
    "usage": {
     "input_tokens": 82,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_03d919fd0e9b3a7cd1e6",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      40348,
      35283,
      768,
      5562,
      23819,
      1203,
      6602,
      1295,
      35101,
      5562,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      35101,
      5562,
      856,
      797,
      5105,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      28,
      39,
      49
     ],
     "logits_raw": [
      2.0700466632843018,
      -3.3887336254119873,
      -0.8867999315261841
     ],
     "probs_raw": [
      0.9467533826828003,
      0.004031983204185963,
      0.0492146797478199
     ],
     "probs": [
      0.8762499690055847,
      0.01774740032851696,
      0.10600268840789795
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.7702472805976868,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.862645149230957e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8762499690055847,
      "probabilities": {
       "insufficient": 0.8762499690055847,
       "supported": 0.01774740032851696,
       "contradicted": 0.10600268840789795
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_07ee70cfe4b01330ce89",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3059,
      14350,
      768,
      61026,
      32634,
      1203,
      1295,
      9100,
      933,
      12552,
      2203,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      61026,
      1178,
      1381,
      45943,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      28,
      38,
      49
     ],
     "logits_raw": [
      -10.376697540283203,
      -5.948400974273682,
      1.1245343685150146
     ],
     "probs_raw": [
      1.0108947208209429e-05,
      0.0008470146567560732,
      0.9991428852081299
     ],
     "probs": [
      0.0002685684012249112,
      0.006351233460009098,
      0.9933801889419556
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.9870289554819465,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.9933801889419556,
      "probabilities": {
       "supported": 0.0002685684012249112,
       "contradicted": 0.006351233460009098,
       "insufficient": 0.9933801889419556
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_0aa1447680e4f25862bb",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 97,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 97,
     "positions": 97,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      53076,
      7944,
      779,
      7494,
      28553,
      906,
      10274,
      523,
      941,
      10423,
      4462,
      1856,
      7944,
      936,
      906,
      946,
      38008,
      523,
      9758,
      7944,
      938,
      2835,
      523,
      18,
      35516,
      7494,
      8544,
      1403,
      779,
      28553,
      856,
      8112,
      10274,
      536,
      797,
      19980,
      521,
      779,
      10423,
      4462,
      1856,
      1391,
      58340,
      779,
      26862,
      53076,
      523,
      3354,
      936,
      874,
      5789,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      57,
      74,
      86
     ],
     "logits_raw": [
      2.831890106201172,
      1.8205256462097168,
      2.5837559700012207
     ],
     "probs_raw": [
      0.46642276644706726,
      0.16964839398860931,
      0.36392882466316223
     ],
     "probs": [
      0.4304542541503906,
      0.20901064574718475,
      0.3605351746082306
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.06991907954216003,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4304542541503906,
      "probabilities": {
       "insufficient": 0.4304542541503906,
       "prohibited": 0.20901064574718475,
       "permitted": 0.3605351746082306
      }
     }
    },
    "usage": {
     "input_tokens": 97,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_0aa24a6701ac9c56c4a0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 84,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 84,
     "positions": 84,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      21009,
      30403,
      32340,
      521,
      1757,
      22922,
      46159,
      779,
      52847,
      2076,
      779,
      3308,
      523,
      46050,
      931,
      879,
      535,
      941,
      21009,
      30403,
      32340,
      810,
      779,
      5156,
      3084,
      44987,
      936,
      6728,
      797,
      5323,
      523,
      18,
      15534,
      779,
      13719,
      916,
      6654,
      6612,
      32340,
      15001,
      536,
      768,
      28687,
      32491,
      1706,
      902,
      7045,
      8825,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      57,
      68,
      76
     ],
     "logits_raw": [
      2.172837495803833,
      2.860612392425537,
      1.9425041675567627
     ],
     "probs_raw": [
      0.2643018066883087,
      0.52577143907547,
      0.20992669463157654
     ],
     "probs": [
      0.28713083267211914,
      0.46929967403411865,
      0.24356946349143982
     ],
     "argmax_index": 1,
     "argmax": "B",
     "top2_margin": 0.1821688413619995,
     "near_tie": false,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.46929967403411865,
      "probabilities": {
       "insufficient": 0.28713083267211914,
       "B": 0.46929967403411865,
       "A": 0.24356946349143982
      }
     }
    },
    "usage": {
     "input_tokens": 84,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_0b43ea8e24e74d621f1b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 62,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 62,
     "positions": 62,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      23120,
      44987,
      936,
      25539,
      779,
      16400,
      31286,
      811,
      779,
      13608,
      1090,
      28623,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      16400,
      31286,
      1052,
      1381,
      25539,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      30,
      41,
      51
     ],
     "logits_raw": [
      -1.1856169700622559,
      6.04213809967041,
      -3.6982507705688477
     ],
     "probs_raw": [
      0.0007255796226672828,
      0.9992156028747559,
      5.881147808395326e-05
     ],
     "probs": [
      0.005686212796717882,
      0.9933690428733826,
      0.0009447598131373525
     ],
     "argmax_index": 1,
     "argmax": "supported",
     "top2_margin": 0.9876828300766647,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9933690428733826,
      "probabilities": {
       "insufficient": 0.005686212796717882,
       "supported": 0.9933690428733826,
       "contradicted": 0.0009447598131373525
      }
     }
    },
    "usage": {
     "input_tokens": 62,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_0ebd299739302d8b057b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      3446,
      3498,
      779,
      2391,
      931,
      1178,
      3191,
      8784,
      536,
      779,
      51746,
      30174,
      3498,
      936,
      856,
      797,
      8784,
      523,
      9758,
      7944,
      938,
      2835,
      521,
      810,
      779,
      30174,
      12199,
      19980,
      1490,
      779,
      13061,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1490,
      779,
      21706,
      13061,
      521,
      779,
      2391,
      931,
      856,
      6654,
      797,
      8784,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      56,
      67,
      78
     ],
     "logits_raw": [
      2.0599265098571777,
      0.8353641033172607,
      0.7449114322662354
     ],
     "probs_raw": [
      0.6400587558746338,
      0.18810448050498962,
      0.1718367636203766
     ],
     "probs": [
      0.5531457662582397,
      0.23064285516738892,
      0.2162114381790161
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.32250291109085083,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.5531457662582397,
      "probabilities": {
       "contradicted": 0.5531457662582397,
       "insufficient": 0.23064285516738892,
       "supported": 0.2162114381790161
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_1105577da4c8dad4609d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 92,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 92,
     "positions": 92,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      3446,
      3498,
      779,
      2391,
      931,
      856,
      797,
      8784,
      523,
      941,
      51746,
      30174,
      3498,
      936,
      1178,
      3191,
      8784,
      523,
      9758,
      7944,
      938,
      2835,
      523,
      941,
      21706,
      13061,
      3498,
      779,
      51746,
      30174,
      12199,
      6249,
      9448,
      19148,
      1518,
      1547,
      5617,
      9171,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      26862,
      3446,
      7944,
      779,
      2391,
      931,
      797,
      8784,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      60,
      71,
      82
     ],
     "logits_raw": [
      1.914824366569519,
      1.321368932723999,
      -0.17958185076713562
     ],
     "probs_raw": [
      0.5968159437179565,
      0.329690158367157,
      0.07349391281604767
     ],
     "probs": [
      0.5323480367660522,
      0.34840649366378784,
      0.11924546957015991
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.1839415431022644,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.5323480367660522,
      "probabilities": {
       "contradicted": 0.5323480367660522,
       "insufficient": 0.34840649366378784,
       "supported": 0.11924546957015991
      }
     }
    },
    "usage": {
     "input_tokens": 92,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_19d3d8d0bf773e88be7b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 64,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 64,
     "positions": 64,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      8825,
      953,
      14708,
      523,
      46050,
      931,
      879,
      535,
      941,
      5771,
      953,
      5068,
      536,
      779,
      9170,
      6728,
      1044,
      2017,
      3019,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      14708,
      8825,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      37,
      45,
      53
     ],
     "logits_raw": [
      0.8596923351287842,
      -1.9263551235198975,
      -0.09066939353942871
     ],
     "probs_raw": [
      0.6904811263084412,
      0.04257814958691597,
      0.26694080233573914
     ],
     "probs": [
      0.608325719833374,
      0.08314049988985062,
      0.3085337281227112
     ],
     "argmax_index": 0,
     "argmax": "A",
     "top2_margin": 0.29979199171066284,
     "near_tie": false,
     "gold": "A",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.608325719833374,
      "probabilities": {
       "A": 0.608325719833374,
       "B": 0.08314049988985062,
       "insufficient": 0.3085337281227112
      }
     }
    },
    "usage": {
     "input_tokens": 64,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_19f37d5a97034bd7d04f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      33648,
      1090,
      37181,
      6175,
      13546,
      779,
      30122,
      9041,
      810,
      4943,
      876,
      13748,
      774,
      779,
      1232,
      33388,
      920,
      2543,
      10074,
      523,
      18,
      542,
      37181,
      6175,
      22558,
      9544,
      8544,
      811,
      779,
      3508,
      936,
      13546,
      523,
      3354,
      779,
      33648,
      7157,
      779,
      1232,
      33388,
      920,
      2543,
      10074,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      48,
      65,
      77
     ],
     "logits_raw": [
      2.548405170440674,
      3.762322187423706,
      -0.5323378443717957
     ],
     "probs_raw": [
      0.2266252338886261,
      0.7629669308662415,
      0.010407790541648865
     ],
     "probs": [
      0.28646430373191833,
      0.6818165183067322,
      0.031719230115413666
     ],
     "argmax_index": 1,
     "argmax": "prohibited",
     "top2_margin": 0.39535221457481384,
     "near_tie": false,
     "gold": "prohibited",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.6818165183067322,
      "probabilities": {
       "insufficient": 0.28646430373191833,
       "prohibited": 0.6818165183067322,
       "permitted": 0.031719230115413666
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_1a81ead69e93e5d0c86a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      12153,
      21968,
      902,
      3863,
      10711,
      521,
      1203,
      779,
      7387,
      811,
      874,
      7093,
      938,
      49033,
      523,
      18,
      18693,
      902,
      3863,
      2853,
      1550,
      1403,
      779,
      12153,
      3575,
      6229,
      9365,
      896,
      2853,
      523,
      3354,
      779,
      11256,
      8355,
      874,
      7093,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      41,
      52,
      64
     ],
     "logits_raw": [
      -5.296720027923584,
      -2.989729642868042,
      2.016939878463745
     ],
     "probs_raw": [
      0.0006615055026486516,
      0.006644258741289377,
      0.9926942586898804
     ],
     "probs": [
      0.005209682509303093,
      0.027071693912148476,
      0.9677186608314514
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.9406469669193029,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.9677186608314514,
      "probabilities": {
       "permitted": 0.005209682509303093,
       "prohibited": 0.027071693912148476,
       "insufficient": 0.9677186608314514
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_1df851a202994a07f123",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      11860,
      2090,
      1321,
      6835,
      2059,
      779,
      3956,
      8649,
      810,
      779,
      5196,
      8170,
      523,
      941,
      11746,
      3084,
      44987,
      779,
      3956,
      8649,
      11423,
      810,
      22922,
      3498,
      779,
      8170,
      953,
      1014,
      14465,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      11860,
      2090,
      1321,
      856,
      5156,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      47,
      58,
      69
     ],
     "logits_raw": [
      0.49528419971466064,
      1.0892655849456787,
      -9.397531509399414
     ],
     "probs_raw": [
      0.35571545362472534,
      0.6442665457725525,
      1.7976592062041163e-05
     ],
     "probs": [
      0.39535412192344666,
      0.6043087244033813,
      0.0003372168866917491
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.2089546024799347,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6043087244033813,
      "probabilities": {
       "contradicted": 0.39535412192344666,
       "insufficient": 0.6043087244033813,
       "supported": 0.0003372168866917491
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_2175379f4c69e6207626",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 65,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 65,
     "positions": 65,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      6567,
      938,
      33037,
      29067,
      7441,
      536,
      1295,
      1927,
      953,
      2685,
      523,
      46050,
      931,
      879,
      535,
      941,
      7850,
      33037,
      1927,
      7410,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      6023,
      1927,
      1725,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      38,
      49,
      57
     ],
     "logits_raw": [
      0.35558879375457764,
      -0.2648470997810364,
      0.7199549674987793
     ],
     "probs_raw": [
      0.3358735144138336,
      0.1806025356054306,
      0.4835239052772522
     ],
     "probs": [
      0.34022027254104614,
      0.2184140384197235,
      0.44136565923690796
     ],
     "argmax_index": 2,
     "argmax": "A",
     "top2_margin": 0.10114538669586182,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.44136565923690796,
      "probabilities": {
       "insufficient": 0.34022027254104614,
       "B": 0.2184140384197235,
       "A": 0.44136565923690796
      }
     }
    },
    "usage": {
     "input_tokens": 65,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_21ca7a8b970e36b8e20f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 99,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 99,
     "positions": 99,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      16396,
      3575,
      6229,
      779,
      5937,
      11430,
      811,
      51132,
      41955,
      523,
      941,
      5937,
      11430,
      17498,
      779,
      4889,
      811,
      902,
      16701,
      521,
      1316,
      1178,
      1295,
      1312,
      17784,
      10562,
      523,
      18,
      1098,
      5937,
      11430,
      1400,
      51132,
      41955,
      1203,
      1400,
      1014,
      40500,
      896,
      10562,
      523,
      11005,
      30403,
      2396,
      2053,
      1400,
      51132,
      523,
      3354,
      779,
      16701,
      51132,
      768,
      3346,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      59,
      71,
      88
     ],
     "logits_raw": [
      0.8783601522445679,
      1.3168089389801025,
      3.038339614868164
     ],
     "probs_raw": [
      0.08911655843257904,
      0.1381574273109436,
      0.7727259397506714
     ],
     "probs": [
      0.14192068576812744,
      0.19411849975585938,
      0.6639607548713684
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.46984225511550903,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.6639607548713684,
      "probabilities": {
       "prohibited": 0.14192068576812744,
       "insufficient": 0.19411849975585938,
       "permitted": 0.6639607548713684
      }
     }
    },
    "usage": {
     "input_tokens": 99,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_22ec57822db50ec121cf",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 61,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 61,
     "positions": 61,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      859,
      64078,
      26677,
      768,
      4025,
      17686,
      523,
      46050,
      931,
      879,
      535,
      859,
      22879,
      26677,
      768,
      4025,
      17686,
      523,
      18,
      52739,
      779,
      13719,
      12469,
      768,
      4025,
      17686,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      34,
      45,
      53
     ],
     "logits_raw": [
      0.028011880815029144,
      0.252865195274353,
      1.7705678939819336
     ],
     "probs_raw": [
      0.12556403875350952,
      0.157223641872406,
      0.7172123193740845
     ],
     "probs": [
      0.17710436880588531,
      0.20796306431293488,
      0.6149325966835022
     ],
     "argmax_index": 2,
     "argmax": "B",
     "top2_margin": 0.4069695323705673,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.6149325966835022,
      "probabilities": {
       "insufficient": 0.17710436880588531,
       "A": 0.20796306431293488,
       "B": 0.6149325966835022
      }
     }
    },
    "usage": {
     "input_tokens": 61,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_25172cd6ddc49ab0ca70",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 70,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 70,
     "positions": 70,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      22773,
      22922,
      2396,
      2240,
      3507,
      3893,
      803,
      779,
      59534,
      523,
      18,
      31678,
      3893,
      856,
      6767,
      1550,
      1720,
      3507,
      25252,
      980,
      17784,
      523,
      3354,
      936,
      6305,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      30,
      42,
      59
     ],
     "logits_raw": [
      2.136638641357422,
      3.2041687965393066,
      0.9671251773834229
     ],
     "probs_raw": [
      0.2370395064353943,
      0.6893553733825684,
      0.0736050233244896
     ],
     "probs": [
      0.27952614426612854,
      0.5992463827133179,
      0.1212274357676506
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.31972023844718933,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5992463827133179,
      "probabilities": {
       "prohibited": 0.27952614426612854,
       "insufficient": 0.5992463827133179,
       "permitted": 0.1212274357676506
      }
     }
    },
    "usage": {
     "input_tokens": 70,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_2608370d01783735c3ea",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      40348,
      10234,
      26956,
      779,
      7627,
      811,
      4963,
      523,
      8874,
      3575,
      12800,
      768,
      23819,
      27809,
      5105,
      953,
      5405,
      810,
      4963,
      953,
      36072,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      35101,
      5562,
      856,
      797,
      4963,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      39,
      49,
      60
     ],
     "logits_raw": [
      -6.859125137329102,
      0.526771068572998,
      -1.9133896827697754
     ],
     "probs_raw": [
      0.0005699152243323624,
      0.9193147420883179,
      0.08011537790298462
     ],
     "probs": [
      0.004332481883466244,
      0.8473929166793823,
      0.14827457070350647
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.6991183459758759,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8473929166793823,
      "probabilities": {
       "supported": 0.004332481883466244,
       "insufficient": 0.8473929166793823,
       "contradicted": 0.14827457070350647
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_2708193212d8a4a523c7",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 95,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 95,
     "positions": 95,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      4887,
      11917,
      810,
      2254,
      2059,
      10547,
      1186,
      938,
      3161,
      810,
      11500,
      875,
      779,
      8324,
      523,
      18,
      21509,
      779,
      2254,
      1550,
      906,
      768,
      3981,
      3321,
      1403,
      779,
      4887,
      11917,
      856,
      58754,
      933,
      4994,
      3972,
      1108,
      536,
      9141,
      2643,
      779,
      4887,
      11917,
      523,
      3354,
      779,
      2254,
      874,
      2643,
      791,
      1490,
      1033,
      5792,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      55,
      67,
      78
     ],
     "logits_raw": [
      0.818483293056488,
      3.5373549461364746,
      2.113175868988037
     ],
     "probs_raw": [
      0.0504717193543911,
      0.765312910079956,
      0.18421539664268494
     ],
     "probs": [
      0.09527885168790817,
      0.6644770503044128,
      0.24024412035942078
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.42423292994499207,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.6644770503044128,
      "probabilities": {
       "prohibited": 0.09527885168790817,
       "permitted": 0.6644770503044128,
       "insufficient": 0.24024412035942078
      }
     }
    },
    "usage": {
     "input_tokens": 95,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_2bf540d515a51755b25c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      13831,
      41071,
      1124,
      779,
      11019,
      523,
      46050,
      931,
      879,
      535,
      13085,
      1123,
      1531,
      936,
      1499,
      2144,
      521,
      1203,
      1112,
      1014,
      41071,
      1124,
      936,
      4763,
      523,
      18,
      52739,
      779,
      13719,
      3575,
      4651,
      41071,
      3665,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      41,
      52,
      60
     ],
     "logits_raw": [
      0.8004128932952881,
      0.6302740573883057,
      0.15096712112426758
     ],
     "probs_raw": [
      0.4226751923561096,
      0.35654670000076294,
      0.22077812254428864
     ],
     "probs": [
      0.39771389961242676,
      0.352198988199234,
      0.25008711218833923
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.04551491141319275,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.39771389961242676,
      "probabilities": {
       "insufficient": 0.39771389961242676,
       "B": 0.352198988199234,
       "A": 0.25008711218833923
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_31eb4533d2c89959ec46",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 90,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 90,
     "positions": 90,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2681,
      28117,
      6516,
      902,
      2960,
      15001,
      6177,
      521,
      1203,
      13834,
      1352,
      4140,
      4689,
      1523,
      2919,
      9448,
      938,
      3161,
      523,
      18,
      14875,
      779,
      2681,
      28117,
      1090,
      10500,
      15001,
      6177,
      536,
      768,
      28687,
      32491,
      1706,
      779,
      7045,
      8825,
      523,
      3354,
      779,
      8153,
      777,
      7157,
      779,
      1530,
      793,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      50,
      67,
      79
     ],
     "logits_raw": [
      2.566591262817383,
      0.23663032054901123,
      0.29303932189941406
     ],
     "probs_raw": [
      0.8331629037857056,
      0.08106638491153717,
      0.08577068150043488
     ],
     "probs": [
      0.7212923765182495,
      0.1365465521812439,
      0.1421610563993454
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.5791313201189041,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7212923765182495,
      "probabilities": {
       "insufficient": 0.7212923765182495,
       "prohibited": 0.1365465521812439,
       "permitted": 0.1421610563993454
      }
     }
    },
    "usage": {
     "input_tokens": 90,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_3226202ba1ada2570e9f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3434,
      919,
      5609,
      797,
      779,
      20610,
      1203,
      4117,
      5329,
      936,
      523,
      18,
      15246,
      779,
      8530,
      7330,
      521,
      768,
      2230,
      6835,
      3822,
      797,
      779,
      20610,
      536,
      22115,
      6955,
      856,
      28441,
      523,
      3354,
      936,
      874,
      12800,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      39,
      51,
      62
     ],
     "logits_raw": [
      1.931678295135498,
      1.6135848760604858,
      2.03139328956604
     ],
     "probs_raw": [
      0.3530586063861847,
      0.25686243176460266,
      0.39007890224456787
     ],
     "probs": [
      0.3483629524707794,
      0.2775549590587616,
      0.3740820288658142
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.02571907639503479,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.3740820288658142,
      "probabilities": {
       "prohibited": 0.3483629524707794,
       "permitted": 0.2775549590587616,
       "insufficient": 0.3740820288658142
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_32f36cd0fbbfb3604586",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      11860,
      2090,
      1321,
      6835,
      779,
      3956,
      8649,
      810,
      5196,
      8170,
      523,
      941,
      11746,
      3084,
      44987,
      896,
      2059,
      11423,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      11860,
      2090,
      1321,
      856,
      5156,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      36,
      46,
      57
     ],
     "logits_raw": [
      -1.0280404090881348,
      0.17128825187683105,
      -1.9770114421844482
     ],
     "probs_raw": [
      0.21253859996795654,
      0.7051793932914734,
      0.08228201419115067
     ],
     "probs": [
      0.258857786655426,
      0.6097228527069092,
      0.1314193308353424
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.35086506605148315,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6097228527069092,
      "probabilities": {
       "supported": 0.258857786655426,
       "insufficient": 0.6097228527069092,
       "contradicted": 0.1314193308353424
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_33d6e5da58f0fee2490d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8170,
      7850,
      11820,
      37414,
      9007,
      4611,
      1203,
      1014,
      1509,
      9764,
      833,
      4611,
      523,
      18,
      1098,
      1509,
      9764,
      833,
      59534,
      6835,
      1550,
      1509,
      9764,
      833,
      4611,
      523,
      3354,
      1033,
      8170,
      6984,
      936,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      36,
      48,
      65
     ],
     "logits_raw": [
      -2.6971306800842285,
      2.78682541847229,
      -2.3960108757019043
     ],
     "probs_raw": [
      0.004112706519663334,
      0.9903295040130615,
      0.0055577936582267284
     ],
     "probs": [
      0.019044214859604836,
      0.9573412537574768,
      0.023614587262272835
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.933726666495204,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.9573412537574768,
      "probabilities": {
       "prohibited": 0.019044214859604836,
       "insufficient": 0.9573412537574768,
       "permitted": 0.023614587262272835
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_36273dd4777876a26338",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3059,
      5813,
      779,
      45853,
      810,
      8112,
      779,
      61026,
      1090,
      8744,
      4623,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      61026,
      1178,
      1381,
      45943,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      28,
      39,
      49
     ],
     "logits_raw": [
      -2.4476537704467773,
      8.073257446289062,
      -0.6171743869781494
     ],
     "probs_raw": [
      2.696134652069304e-05,
      0.9998049139976501,
      0.00016815461276564747
     ],
     "probs": [
      0.000543197151273489,
      0.9974485039710999,
      0.002008317969739437
     ],
     "argmax_index": 1,
     "argmax": "supported",
     "top2_margin": 0.9954401860013604,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 2.3283064365386963e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9974485039710999,
      "probabilities": {
       "contradicted": 0.000543197151273489,
       "supported": 0.9974485039710999,
       "insufficient": 0.002008317969739437
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_36c71bb649b7d5eccb1f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      59063,
      44987,
      19459,
      60468,
      856,
      4175,
      810,
      19459,
      42074,
      788,
      856,
      49550,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      19459,
      60468,
      856,
      4175,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      28,
      38,
      49
     ],
     "logits_raw": [
      7.094854354858398,
      0.14647667109966278,
      -2.7314815521240234
     ],
     "probs_raw": [
      0.9989867806434631,
      0.0009592188289389014,
      5.395556945586577e-05
     ],
     "probs": [
      0.9921786785125732,
      0.0069338493049144745,
      0.0008874361519701779
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.9852448292076588,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9921786785125732,
      "probabilities": {
       "supported": 0.9921786785125732,
       "insufficient": 0.0069338493049144745,
       "contradicted": 0.0008874361519701779
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_37cebc3792e98049439f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      45609,
      7330,
      9938,
      592,
      38183,
      811,
      779,
      797,
      41722,
      23120,
      1403,
      936,
      1011,
      2463,
      779,
      1336,
      536,
      1517,
      779,
      5329,
      23120,
      1550,
      1403,
      779,
      797,
      41722,
      23120,
      4584,
      523,
      9758,
      29142,
      10547,
      1186,
      1011,
      2463,
      1033,
      12940,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      5329,
      23120,
      1011,
      2463,
      1033,
      12940,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      56,
      67,
      77
     ],
     "logits_raw": [
      1.0944440364837646,
      2.0053186416625977,
      -0.7762060165405273
     ],
     "probs_raw": [
      0.2746860384941101,
      0.6830058097839355,
      0.042308107018470764
     ],
     "probs": [
      0.3144998848438263,
      0.6028423309326172,
      0.08265769481658936
     ],
     "argmax_index": 1,
     "argmax": "supported",
     "top2_margin": 0.2883424460887909,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.6028423309326172,
      "probabilities": {
       "insufficient": 0.3144998848438263,
       "supported": 0.6028423309326172,
       "contradicted": 0.08265769481658936
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_386e10b1c1ce906473a4",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 98,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 98,
     "positions": 98,
     "bucket": 256,
     "ids": [
      1,
      17,
      49290,
      5019,
      11528,
      6261,
      875,
      20155,
      1230,
      1798,
      521,
      6794,
      896,
      10144,
      39208,
      3469,
      1134,
      15419,
      2563,
      523,
      46050,
      931,
      835,
      535,
      4411,
      38120,
      953,
      5019,
      1929,
      803,
      768,
      20155,
      1230,
      1047,
      810,
      5260,
      779,
      4225,
      19214,
      523,
      46050,
      931,
      879,
      535,
      29207,
      953,
      5019,
      1929,
      803,
      768,
      20155,
      1230,
      1047,
      1203,
      39273,
      779,
      19214,
      523,
      18,
      15534,
      779,
      38120,
      46102,
      1490,
      779,
      9519,
      11287,
      810,
      32294,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      71,
      79,
      87
     ],
     "logits_raw": [
      1.2154874801635742,
      3.1739749908447266,
      1.4671909809112549
     ],
     "probs_raw": [
      0.10666880756616592,
      0.7561322450637817,
      0.13719898462295532
     ],
     "probs": [
      0.16004574298858643,
      0.6483826637268066,
      0.19157147407531738
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.45681118965148926,
     "near_tie": false,
     "gold": "A",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.6483826637268066,
      "probabilities": {
       "B": 0.16004574298858643,
       "A": 0.6483826637268066,
       "insufficient": 0.19157147407531738
      }
     }
    },
    "usage": {
     "input_tokens": 98,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_3b0958cabc339005cf66",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 63,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 63,
     "positions": 63,
     "bucket": 256,
     "ids": [
      1,
      17,
      542,
      6634,
      35283,
      902,
      7267,
      2630,
      19736,
      768,
      12037,
      933,
      3984,
      779,
      3863,
      856,
      37640,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      7267,
      12037,
      953,
      37640,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      31,
      42,
      52
     ],
     "logits_raw": [
      0.8897602558135986,
      -6.432289123535156,
      -0.8964612483978271
     ],
     "probs_raw": [
      0.8559790253639221,
      0.000565636670216918,
      0.14345543086528778
     ],
     "probs": [
      0.7785053253173828,
      0.004166017286479473,
      0.21732863783836365
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.5611766874790192,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7785053253173828,
      "probabilities": {
       "insufficient": 0.7785053253173828,
       "supported": 0.004166017286479473,
       "contradicted": 0.21732863783836365
      }
     }
    },
    "usage": {
     "input_tokens": 63,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_3dd9c3ac98f8ddf7bdd3",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 64,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 64,
     "positions": 64,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3059,
      12930,
      12874,
      875,
      45853,
      779,
      61026,
      810,
      22922,
      3498,
      1295,
      45853,
      1178,
      1381,
      5813,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      4479,
      6034,
      45853,
      12874,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      32,
      43,
      53
     ],
     "logits_raw": [
      -0.29899075627326965,
      -2.115078926086426,
      -0.10533981025218964
     ],
     "probs_raw": [
      0.4208163917064667,
      0.06845032423734665,
      0.5107333064079285
     ],
     "probs": [
      0.4129457473754883,
      0.11284510046243668,
      0.47420912981033325
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.06126338243484497,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.47420912981033325,
      "probabilities": {
       "contradicted": 0.4129457473754883,
       "supported": 0.11284510046243668,
       "insufficient": 0.47420912981033325
      }
     }
    },
    "usage": {
     "input_tokens": 64,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_3e75f2b84e8e08e3d3f2",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      10046,
      6289,
      779,
      13732,
      523,
      46050,
      931,
      879,
      535,
      941,
      5647,
      1178,
      768,
      7043,
      20453,
      523,
      18,
      52739,
      779,
      13719,
      12469,
      768,
      4025,
      17686,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      33,
      41,
      49
     ],
     "logits_raw": [
      0.2606961727142334,
      -3.224853515625,
      0.07640176266431808
     ],
     "probs_raw": [
      0.5369623899459839,
      0.016450868919491768,
      0.4465867280960083
     ],
     "probs": [
      0.5103158354759216,
      0.04231620952486992,
      0.44736793637275696
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.06294789910316467,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.5103158354759216,
      "probabilities": {
       "B": 0.5103158354759216,
       "A": 0.04231620952486992,
       "insufficient": 0.44736793637275696
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_3f783eec5c90812479bc",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3434,
      919,
      5609,
      5329,
      779,
      20610,
      810,
      4117,
      5662,
      936,
      523,
      18,
      542,
      12518,
      1090,
      2230,
      6835,
      4624,
      797,
      779,
      20610,
      810,
      1178,
      1295,
      21701,
      27434,
      523,
      3354,
      936,
      874,
      12800,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      36,
      47,
      59
     ],
     "logits_raw": [
      2.9592502117156982,
      1.7731610536575317,
      2.0034730434417725
     ],
     "probs_raw": [
      0.591741681098938,
      0.1807258129119873,
      0.2275324910879135
     ],
     "probs": [
      0.5171136260032654,
      0.22162669897079468,
      0.26125967502593994
     ],
     "argmax_index": 0,
     "argmax": "permitted",
     "top2_margin": 0.25585395097732544,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.5171136260032654,
      "probabilities": {
       "permitted": 0.5171136260032654,
       "prohibited": 0.22162669897079468,
       "insufficient": 0.26125967502593994
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4089c199b8b796ec3ed8",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 94,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 94,
     "positions": 94,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1547,
      40111,
      938,
      5668,
      521,
      1203,
      13834,
      16425,
      4689,
      14503,
      2583,
      856,
      4617,
      523,
      18,
      21509,
      779,
      2254,
      1550,
      906,
      768,
      3981,
      3321,
      1403,
      779,
      4887,
      11917,
      856,
      58754,
      933,
      4994,
      3972,
      1108,
      536,
      9141,
      2643,
      779,
      4887,
      11917,
      523,
      3354,
      779,
      2254,
      874,
      2643,
      791,
      1490,
      1033,
      5792,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      54,
      71,
      83
     ],
     "logits_raw": [
      2.3879141807556152,
      0.4206157922744751,
      2.479020833969116
     ],
     "probs_raw": [
      0.44738325476646423,
      0.06255944818258286,
      0.4900572597980499
     ],
     "probs": [
      0.432425856590271,
      0.10606959462165833,
      0.4615044593811035
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.02907860279083252,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.4615044593811035,
      "probabilities": {
       "insufficient": 0.432425856590271,
       "prohibited": 0.10606959462165833,
       "permitted": 0.4615044593811035
      }
     }
    },
    "usage": {
     "input_tokens": 94,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_42384af9ae56beb37ff5",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 91,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 91,
     "positions": 91,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7330,
      22558,
      2314,
      53859,
      968,
      779,
      7576,
      1203,
      23369,
      2796,
      1523,
      3759,
      53859,
      523,
      46050,
      931,
      835,
      535,
      859,
      5068,
      3346,
      17784,
      10562,
      5828,
      988,
      779,
      7576,
      523,
      46050,
      931,
      879,
      535,
      859,
      5068,
      936,
      1550,
      988,
      835,
      521,
      810,
      1052,
      1295,
      1312,
      8825,
      523,
      18,
      15534,
      779,
      13719,
      5977,
      13289,
      10562,
      4834,
      1550,
      1626,
      768,
      33776,
      2341,
      53859,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      64,
      75,
      83
     ],
     "logits_raw": [
      1.6472855806350708,
      2.0896692276000977,
      2.033372163772583
     ],
     "probs_raw": [
      0.24828527867794037,
      0.3864343762397766,
      0.36528030037879944
     ],
     "probs": [
      0.27105966210365295,
      0.3717978298664093,
      0.3571425676345825
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.014655262231826782,
     "near_tie": true,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.3717978298664093,
      "probabilities": {
       "insufficient": 0.27105966210365295,
       "A": 0.3717978298664093,
       "B": 0.3571425676345825
      }
     }
    },
    "usage": {
     "input_tokens": 91,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_42c54fa2ad45743e24d8",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      49290,
      5019,
      11528,
      6261,
      875,
      20155,
      1230,
      1798,
      521,
      6794,
      896,
      10144,
      39208,
      3469,
      1134,
      15419,
      2563,
      523,
      46050,
      931,
      835,
      810,
      46050,
      931,
      879,
      2312,
      5019,
      11528,
      6261,
      1203,
      3176,
      13834,
      1230,
      1047,
      20306,
      4689,
      19214,
      2203,
      523,
      18,
      15534,
      779,
      38120,
      46102,
      1490,
      779,
      9519,
      11287,
      810,
      32294,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      54,
      62,
      73
     ],
     "logits_raw": [
      0.7830364108085632,
      2.160759449005127,
      1.3004415035247803
     ],
     "probs_raw": [
      0.1505223959684372,
      0.596950888633728,
      0.2525267004966736
     ],
     "probs": [
      0.19520875811576843,
      0.5222931504249573,
      0.2824980318546295
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.23979511857032776,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5222931504249573,
      "probabilities": {
       "A": 0.19520875811576843,
       "insufficient": 0.5222931504249573,
       "B": 0.2824980318546295
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4444dad25e438f128673",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3434,
      919,
      5609,
      797,
      779,
      20610,
      1203,
      4117,
      5329,
      936,
      523,
      18,
      542,
      12518,
      1090,
      2230,
      6835,
      4624,
      797,
      779,
      20610,
      810,
      1178,
      1295,
      21701,
      27434,
      523,
      3354,
      936,
      874,
      12800,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      36,
      48,
      59
     ],
     "logits_raw": [
      2.421286106109619,
      4.4906415939331055,
      2.1723551750183105
     ],
     "probs_raw": [
      0.10309968143701553,
      0.8165203332901001,
      0.08038000017404556
     ],
     "probs": [
      0.16071435809135437,
      0.7047528624534607,
      0.13453277945518494
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.5440385043621063,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.7047528624534607,
      "probabilities": {
       "prohibited": 0.16071435809135437,
       "permitted": 0.7047528624534607,
       "insufficient": 0.13453277945518494
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_44574c44e243ee1b2f52",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      40348,
      10234,
      26956,
      779,
      7627,
      811,
      4963,
      523,
      8874,
      3575,
      12800,
      768,
      23819,
      27809,
      5105,
      953,
      5405,
      810,
      4963,
      953,
      36072,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      35101,
      5562,
      856,
      797,
      5105,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      39,
      50,
      61
     ],
     "logits_raw": [
      -0.28942108154296875,
      -3.0245723724365234,
      2.4573111534118652
     ],
     "probs_raw": [
      0.06003668159246445,
      0.003895431524142623,
      0.9360678791999817
     ],
     "probs": [
      0.12112487852573395,
      0.017167197540402412,
      0.8617079257965088
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.7405830472707748,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.8617079257965088,
      "probabilities": {
       "insufficient": 0.12112487852573395,
       "contradicted": 0.017167197540402412,
       "supported": 0.8617079257965088
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_44bae0c8a1615a096c43",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 74,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 74,
     "positions": 74,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      6634,
      8269,
      896,
      779,
      7850,
      33037,
      1927,
      7410,
      523,
      46050,
      931,
      879,
      535,
      941,
      6634,
      15994,
      33037,
      29067,
      7441,
      1203,
      3498,
      1295,
      1927,
      953,
      2685,
      523,
      18,
      52739,
      779,
      13719,
      8960,
      1550,
      902,
      946,
      1513,
      58009,
      1927,
      9100,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      47,
      55,
      63
     ],
     "logits_raw": [
      -1.737524390220642,
      0.3417292535305023,
      0.8451249003410339
     ],
     "probs_raw": [
      0.04498295858502388,
      0.35979601740837097,
      0.5952211022377014
     ],
     "probs": [
      0.08515271544456482,
      0.3760550320148468,
      0.5387921929359436
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.1627371609210968,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5387921929359436,
      "probabilities": {
       "B": 0.08515271544456482,
       "A": 0.3760550320148468,
       "insufficient": 0.5387921929359436
      }
     }
    },
    "usage": {
     "input_tokens": 74,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4533dd91267b4d815690",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 93,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 93,
     "positions": 93,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8414,
      5799,
      22654,
      1178,
      768,
      12803,
      13789,
      810,
      1236,
      6034,
      5231,
      41095,
      523,
      9758,
      11445,
      938,
      11649,
      523,
      18,
      41914,
      1587,
      22879,
      8414,
      5799,
      523,
      45681,
      936,
      875,
      768,
      12803,
      13789,
      521,
      6794,
      896,
      5231,
      41095,
      3469,
      4201,
      936,
      797,
      808,
      16979,
      523,
      3354,
      1033,
      1789,
      874,
      11086,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      53,
      64,
      81
     ],
     "logits_raw": [
      3.4331963062286377,
      2.286992311477661,
      1.4278408288955688
     ],
     "probs_raw": [
      0.6884902119636536,
      0.218830406665802,
      0.09267935901880264
     ],
     "probs": [
      0.5953508019447327,
      0.2625322937965393,
      0.14211688935756683
     ],
     "argmax_index": 0,
     "argmax": "permitted",
     "top2_margin": 0.33281850814819336,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.5953508019447327,
      "probabilities": {
       "permitted": 0.5953508019447327,
       "insufficient": 0.2625322937965393,
       "prohibited": 0.14211688935756683
      }
     }
    },
    "usage": {
     "input_tokens": 93,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_46b7029b9a704138b77a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      3312,
      17980,
      779,
      12153,
      30403,
      33300,
      1450,
      52518,
      523,
      2357,
      22092,
      1689,
      22922,
      46159,
      896,
      52847,
      523,
      941,
      3084,
      3498,
      17688,
      1486,
      23496,
      7045,
      15001,
      521,
      810,
      1295,
      2919,
      15001,
      953,
      2997,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      12153,
      6516,
      15001,
      884,
      17980,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      49,
      60,
      71
     ],
     "logits_raw": [
      0.612383246421814,
      0.810704231262207,
      2.3589706420898438
     ],
     "probs_raw": [
      0.12571735680103302,
      0.1532939225435257,
      0.7209887504577637
     ],
     "probs": [
      0.17748259007930756,
      0.2044944167137146,
      0.6180230379104614
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.4135286211967468,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.6180230379104614,
      "probabilities": {
       "insufficient": 0.17748259007930756,
       "contradicted": 0.2044944167137146,
       "supported": 0.6180230379104614
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_46d6a0731bd288c370f0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 67,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 67,
     "positions": 67,
     "bucket": 256,
     "ids": [
      1,
      17,
      30729,
      856,
      5668,
      797,
      31804,
      1337,
      18577,
      521,
      1203,
      1295,
      10562,
      25274,
      933,
      53859,
      4450,
      938,
      21706,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      8376,
      1178,
      10562,
      811,
      51132,
      16993,
      18577,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      35,
      46,
      57
     ],
     "logits_raw": [
      2.5726563930511475,
      -0.306399405002594,
      -4.652129173278809
     ],
     "probs_raw": [
      0.9461489915847778,
      0.053162023425102234,
      0.0006890883669257164
     ],
     "probs": [
      0.882128119468689,
      0.1128116250038147,
      0.005060174502432346
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.7693164944648743,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.882128119468689,
      "probabilities": {
       "insufficient": 0.882128119468689,
       "contradicted": 0.1128116250038147,
       "supported": 0.005060174502432346
      }
     }
    },
    "usage": {
     "input_tokens": 67,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4aa83af95bb69e90bc2d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 97,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 97,
     "positions": 97,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      16396,
      3575,
      6229,
      779,
      5937,
      11430,
      811,
      51132,
      41955,
      523,
      941,
      5937,
      11430,
      17498,
      779,
      4889,
      811,
      902,
      16701,
      521,
      1316,
      1178,
      1295,
      1312,
      17784,
      10562,
      523,
      18,
      1098,
      5937,
      11430,
      1400,
      40500,
      3346,
      17784,
      811,
      902,
      16701,
      521,
      810,
      5296,
      1101,
      25274,
      779,
      16701,
      17784,
      10562,
      523,
      3354,
      1033,
      16701,
      51132,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      57,
      68,
      80
     ],
     "logits_raw": [
      2.168987989425659,
      0.28088173270225525,
      1.7221136093139648
     ],
     "probs_raw": [
      0.5583527684211731,
      0.08451125770807266,
      0.35713595151901245
     ],
     "probs": [
      0.5034531950950623,
      0.1306791752576828,
      0.36586761474609375
     ],
     "argmax_index": 0,
     "argmax": "permitted",
     "top2_margin": 0.1375855803489685,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.5034531950950623,
      "probabilities": {
       "permitted": 0.5034531950950623,
       "prohibited": 0.1306791752576828,
       "insufficient": 0.36586761474609375
      }
     }
    },
    "usage": {
     "input_tokens": 97,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4aee37a6a155747766aa",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 95,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 95,
     "positions": 95,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      53076,
      7944,
      779,
      28553,
      946,
      38008,
      521,
      2158,
      779,
      10423,
      4462,
      1856,
      7944,
      936,
      10274,
      523,
      9758,
      7944,
      938,
      2835,
      523,
      18,
      35516,
      7494,
      8544,
      1403,
      779,
      28553,
      856,
      8112,
      10274,
      536,
      797,
      19980,
      521,
      779,
      10423,
      4462,
      1856,
      1391,
      58340,
      779,
      26862,
      53076,
      523,
      3354,
      936,
      874,
      5789,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      55,
      66,
      78
     ],
     "logits_raw": [
      1.37359619140625,
      1.4480979442596436,
      2.7218360900878906
     ],
     "probs_raw": [
      0.16869127750396729,
      0.18173906207084656,
      0.6495696902275085
     ],
     "probs": [
      0.21392832696437836,
      0.22562186419963837,
      0.5604497790336609
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.3348279148340225,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5604497790336609,
      "probabilities": {
       "permitted": 0.21392832696437836,
       "prohibited": 0.22562186419963837,
       "insufficient": 0.5604497790336609
      }
     }
    },
    "usage": {
     "input_tokens": 95,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4b3d794c004566610181",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      6765,
      4943,
      876,
      22558,
      18109,
      16711,
      803,
      779,
      2984,
      523,
      18,
      560,
      965,
      16711,
      1550,
      916,
      779,
      6765,
      1090,
      11599,
      15001,
      523,
      3354,
      16711,
      874,
      4997,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      31,
      43,
      60
     ],
     "logits_raw": [
      -0.2797635793685913,
      1.3045634031295776,
      4.427176475524902
     ],
     "probs_raw": [
      0.008577145636081696,
      0.04182225093245506,
      0.9496005773544312
     ],
     "probs": [
      0.030341804027557373,
      0.09409179538488388,
      0.8755663633346558
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.7814745679497719,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.8755663633346558,
      "probabilities": {
       "prohibited": 0.030341804027557373,
       "insufficient": 0.09409179538488388,
       "permitted": 0.8755663633346558
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4d7a003342db5b0a17e3",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      42271,
      44987,
      43152,
      803,
      768,
      8825,
      5771,
      1203,
      22922,
      3498,
      1295,
      9170,
      6177,
      1178,
      1381,
      2047,
      523,
      46050,
      931,
      879,
      535,
      941,
      42271,
      44987,
      896,
      779,
      8825,
      953,
      14708,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      43152,
      916,
      779,
      9170,
      2510,
      1044,
      2017,
      3019,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      52,
      63,
      71
     ],
     "logits_raw": [
      -1.0214812755584717,
      -1.9063987731933594,
      -0.2571205794811249
     ],
     "probs_raw": [
      0.2808697521686554,
      0.11592849344015121,
      0.6032017469406128
     ],
     "probs": [
      0.30695444345474243,
      0.16313357651233673,
      0.529911994934082
     ],
     "argmax_index": 2,
     "argmax": "B",
     "top2_margin": 0.2229575514793396,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.529911994934082,
      "probabilities": {
       "insufficient": 0.30695444345474243,
       "A": 0.16313357651233673,
       "B": 0.529911994934082
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_4da6aedc1139224eeff2",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 104,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 104,
     "positions": 104,
     "bucket": 256,
     "ids": [
      1,
      17,
      49290,
      5019,
      11528,
      6261,
      875,
      20155,
      1230,
      1798,
      521,
      6794,
      896,
      10144,
      39208,
      3469,
      1134,
      15419,
      2563,
      523,
      46050,
      931,
      835,
      535,
      4411,
      38120,
      953,
      5019,
      1929,
      803,
      768,
      20155,
      1230,
      1047,
      810,
      5260,
      779,
      4225,
      19214,
      523,
      46050,
      931,
      879,
      535,
      29207,
      953,
      5019,
      1929,
      803,
      768,
      20155,
      1230,
      1047,
      1203,
      39273,
      779,
      19214,
      523,
      18,
      15534,
      779,
      38120,
      1134,
      15419,
      2441,
      10285,
      968,
      779,
      19214,
      32294,
      9618,
      779,
      1230,
      1047,
      11287,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      77,
      88,
      96
     ],
     "logits_raw": [
      1.825056552886963,
      0.6948303580284119,
      2.1762826442718506
     ],
     "probs_raw": [
      0.3644622266292572,
      0.117706798017025,
      0.517831027507782
     ],
     "probs": [
      0.3661385178565979,
      0.1633097380399704,
      0.4705516993999481
     ],
     "argmax_index": 2,
     "argmax": "A",
     "top2_margin": 0.10441318154335022,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.4705516993999481,
      "probabilities": {
       "insufficient": 0.3661385178565979,
       "B": 0.1633097380399704,
       "A": 0.4705516993999481
      }
     }
    },
    "usage": {
     "input_tokens": 104,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_51ceab3ce622c7c759f1",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      10720,
      768,
      25473,
      813,
      1914,
      521,
      768,
      3338,
      7061,
      1165,
      779,
      12037,
      803,
      768,
      33390,
      7267,
      523,
      941,
      3893,
      6634,
      4478,
      1033,
      856,
      37640,
      810,
      1295,
      2151,
      7267,
      953,
      7935,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      7267,
      12037,
      953,
      37640,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      46,
      57,
      68
     ],
     "logits_raw": [
      -0.9429095387458801,
      0.016189217567443848,
      6.5538153648376465
     ],
     "probs_raw": [
      0.0005537894903682172,
      0.0014450271846726537,
      0.9980011582374573
     ],
     "probs": [
      0.004657903220504522,
      0.009241331368684769,
      0.9861007332801819
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.9768594019114971,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9861007332801819,
      "probabilities": {
       "contradicted": 0.004657903220504522,
       "insufficient": 0.009241331368684769,
       "supported": 0.9861007332801819
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_533d4423d311de82b27d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 82,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 82,
     "positions": 82,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      13608,
      21090,
      1531,
      902,
      41071,
      3665,
      1499,
      2144,
      1203,
      22922,
      45818,
      3785,
      936,
      41071,
      1975,
      4763,
      523,
      46050,
      931,
      879,
      535,
      941,
      13608,
      13765,
      592,
      779,
      15565,
      811,
      41071,
      1124,
      779,
      11019,
      523,
      18,
      52739,
      779,
      13719,
      14065,
      1550,
      875,
      2203,
      2158,
      916,
      4289,
      800,
      52847,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      55,
      66,
      74
     ],
     "logits_raw": [
      1.1153349876403809,
      0.07922947406768799,
      -2.398470163345337
     ],
     "probs_raw": [
      0.7222212553024292,
      0.25626859068870544,
      0.021510181948542595
     ],
     "probs": [
      0.6417179703712463,
      0.3061330020427704,
      0.052149027585983276
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.33558496832847595,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6417179703712463,
      "probabilities": {
       "insufficient": 0.6417179703712463,
       "A": 0.3061330020427704,
       "B": 0.052149027585983276
      }
     }
    },
    "usage": {
     "input_tokens": 82,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_56c419b12cd37cbe790c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 74,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 74,
     "positions": 74,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      21522,
      16461,
      521,
      997,
      1098,
      21008,
      3498,
      25583,
      1706,
      938,
      21706,
      521,
      1203,
      10735,
      1166,
      4617,
      523,
      859,
      25202,
      896,
      8294,
      2927,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      21522,
      30261,
      774,
      779,
      21008,
      1090,
      50624,
      8294,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      42,
      53,
      64
     ],
     "logits_raw": [
      0.183104008436203,
      -0.15960577130317688,
      -3.6311588287353516
     ],
     "probs_raw": [
      0.5774011611938477,
      0.4098648726940155,
      0.012733984738588333
     ],
     "probs": [
      0.5410023927688599,
      0.42352527379989624,
      0.03547229990363121
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.11747711896896362,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.5410023927688599,
      "probabilities": {
       "contradicted": 0.5410023927688599,
       "insufficient": 0.42352527379989624,
       "supported": 0.03547229990363121
      }
     }
    },
    "usage": {
     "input_tokens": 74,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_579beed184ffc1104aa9",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 72,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 72,
     "positions": 72,
     "bucket": 256,
     "ids": [
      1,
      17,
      542,
      5156,
      13443,
      6835,
      2059,
      10621,
      523,
      46050,
      931,
      835,
      535,
      941,
      39037,
      856,
      12930,
      1203,
      779,
      33407,
      2312,
      856,
      26117,
      523,
      46050,
      931,
      879,
      535,
      9758,
      779,
      39037,
      810,
      33407,
      2312,
      938,
      12930,
      523,
      18,
      15534,
      779,
      5156,
      13509,
      13443,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      45,
      53,
      61
     ],
     "logits_raw": [
      3.4579966068267822,
      1.1308937072753906,
      2.0269265174865723
     ],
     "probs_raw": [
      0.7481496334075928,
      0.07300297170877457,
      0.17884740233421326
     ],
     "probs": [
      0.6453805565834045,
      0.12242553383111954,
      0.2321939319372177
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.41318662464618683,
     "near_tie": false,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.6453805565834045,
      "probabilities": {
       "B": 0.6453805565834045,
       "A": 0.12242553383111954,
       "insufficient": 0.2321939319372177
      }
     }
    },
    "usage": {
     "input_tokens": 72,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_587ced261c6092cd1325",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 77,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 77,
     "positions": 77,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      13608,
      21090,
      1531,
      902,
      41071,
      3665,
      1499,
      2144,
      1203,
      22922,
      45818,
      3785,
      936,
      41071,
      1975,
      4763,
      523,
      46050,
      931,
      879,
      535,
      941,
      13608,
      13765,
      592,
      779,
      15565,
      811,
      41071,
      1124,
      779,
      11019,
      523,
      18,
      52739,
      779,
      13719,
      3575,
      4651,
      41071,
      3665,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      50,
      61,
      69
     ],
     "logits_raw": [
      1.8004045486450195,
      -1.860771656036377,
      -0.186965674161911
     ],
     "probs_raw": [
      0.8600244522094727,
      0.022104576230049133,
      0.11787096410989761
     ],
     "probs": [
      0.7604895234107971,
      0.05562574788928032,
      0.18388469517230988
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.5766048282384872,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7604895234107971,
      "probabilities": {
       "insufficient": 0.7604895234107971,
       "B": 0.05562574788928032,
       "A": 0.18388469517230988
      }
     }
    },
    "usage": {
     "input_tokens": 77,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_599b3092b6fd18312d23",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      29117,
      800,
      953,
      30403,
      810,
      779,
      5156,
      3084,
      3498,
      15001,
      6728,
      797,
      5323,
      523,
      46050,
      931,
      879,
      535,
      29117,
      800,
      953,
      30403,
      521,
      1757,
      22922,
      47927,
      523,
      18,
      15534,
      779,
      13719,
      916,
      6654,
      6612,
      32340,
      15001,
      536,
      768,
      28687,
      32491,
      1706,
      902,
      7045,
      8825,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      52,
      60,
      71
     ],
     "logits_raw": [
      3.349609851837158,
      2.186221122741699,
      2.6548283100128174
     ],
     "probs_raw": [
      0.5519954562187195,
      0.17245754599571228,
      0.27554699778556824
     ],
     "probs": [
      0.4891495704650879,
      0.21306899189949036,
      0.29778143763542175
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.19136813282966614,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.4891495704650879,
      "probabilities": {
       "B": 0.4891495704650879,
       "insufficient": 0.21306899189949036,
       "A": 0.29778143763542175
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_5a37d1dfcad20810a76a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 83,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 83,
     "positions": 83,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      4887,
      11917,
      810,
      2254,
      2059,
      10547,
      1186,
      938,
      3161,
      810,
      11500,
      875,
      779,
      8324,
      523,
      18,
      15246,
      779,
      8530,
      5792,
      521,
      4021,
      11500,
      3161,
      35908,
      1400,
      874,
      2643,
      791,
      2630,
      17150,
      523,
      3354,
      779,
      2254,
      874,
      2643,
      791,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      43,
      55,
      66
     ],
     "logits_raw": [
      -0.7450401186943054,
      2.489084005355835,
      2.3256301879882812
     ],
     "probs_raw": [
      0.02085919864475727,
      0.5294926166534424,
      0.4496481716632843
     ],
     "probs": [
      0.049891456961631775,
      0.5027567744255066,
      0.4473517835140228
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.055404990911483765,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.5027567744255066,
      "probabilities": {
       "prohibited": 0.049891456961631775,
       "permitted": 0.5027567744255066,
       "insufficient": 0.4473517835140228
      }
     }
    },
    "usage": {
     "input_tokens": 83,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_5d31b4bf2d0fc193f53b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 86,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 86,
     "positions": 86,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      49104,
      7944,
      3822,
      884,
      779,
      2090,
      1321,
      1203,
      14350,
      1295,
      11746,
      3084,
      875,
      4021,
      8688,
      523,
      18,
      8475,
      32144,
      2524,
      2090,
      1321,
      8544,
      1403,
      2059,
      779,
      19089,
      16687,
      810,
      37610,
      1126,
      1052,
      1381,
      21706,
      523,
      3354,
      2524,
      2090,
      1321,
      874,
      12043,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      46,
      57,
      74
     ],
     "logits_raw": [
      1.4837925434112549,
      2.482841968536377,
      -2.960299491882324
     ],
     "probs_raw": [
      0.26828014850616455,
      0.7285681962966919,
      0.0031516929157078266
     ],
     "probs": [
      0.32433220744132996,
      0.6621069312095642,
      0.013560839928686619
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.33777472376823425,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6621069312095642,
      "probabilities": {
       "permitted": 0.32433220744132996,
       "insufficient": 0.6621069312095642,
       "prohibited": 0.013560839928686619
      }
     }
    },
    "usage": {
     "input_tokens": 86,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_6149a17bc154f9c5b4a0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 86,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 86,
     "positions": 86,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1789,
      5792,
      44117,
      60706,
      43675,
      9618,
      8414,
      25885,
      521,
      1203,
      22922,
      13748,
      774,
      1883,
      5896,
      1919,
      1403,
      779,
      29114,
      856,
      60706,
      523,
      1470,
      8414,
      29114,
      856,
      60706,
      1929,
      1883,
      11851,
      936,
      523,
      3253,
      1312,
      3360,
      856,
      2251,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      29114,
      1178,
      768,
      15868,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      54,
      64,
      75
     ],
     "logits_raw": [
      -0.638434648513794,
      1.2378493547439575,
      1.305659532546997
     ],
     "probs_raw": [
      0.0688871443271637,
      0.4497777223587036,
      0.4813351035118103
     ],
     "probs": [
      0.1132504865527153,
      0.4326384365558624,
      0.4541110396385193
     ],
     "argmax_index": 2,
     "argmax": "contradicted",
     "top2_margin": 0.02147260308265686,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.4541110396385193,
      "probabilities": {
       "supported": 0.1132504865527153,
       "insufficient": 0.4326384365558624,
       "contradicted": 0.4541110396385193
      }
     }
    },
    "usage": {
     "input_tokens": 86,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_61ac20b07a5189231e15",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 85,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 85,
     "positions": 85,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      6765,
      22558,
      10004,
      2782,
      768,
      2984,
      797,
      902,
      7477,
      4611,
      62248,
      810,
      4943,
      876,
      23369,
      2796,
      18109,
      16711,
      523,
      18,
      32135,
      4611,
      19494,
      856,
      6767,
      1403,
      779,
      6765,
      22558,
      896,
      1517,
      523,
      3354,
      779,
      2984,
      4650,
      797,
      779,
      4611,
      62248,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      45,
      57,
      68
     ],
     "logits_raw": [
      1.741729736328125,
      2.7703092098236084,
      1.8944990634918213
     ],
     "probs_raw": [
      0.2015257179737091,
      0.5636855363845825,
      0.23478880524635315
     ],
     "probs": [
      0.23807977139949799,
      0.49638843536376953,
      0.2655317783355713
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.23085665702819824,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.49638843536376953,
      "probabilities": {
       "prohibited": 0.23807977139949799,
       "permitted": 0.49638843536376953,
       "insufficient": 0.2655317783355713
      }
     }
    },
    "usage": {
     "input_tokens": 85,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_64d63f86e328aff1481c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 97,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 97,
     "positions": 97,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      45609,
      7330,
      9938,
      592,
      38183,
      811,
      779,
      797,
      41722,
      23120,
      1403,
      936,
      1011,
      2463,
      779,
      1336,
      536,
      1517,
      779,
      5329,
      23120,
      1550,
      1403,
      779,
      797,
      41722,
      23120,
      4584,
      523,
      941,
      797,
      41722,
      23120,
      4584,
      2463,
      1033,
      12940,
      536,
      779,
      5329,
      23120,
      1011,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      45609,
      7330,
      9938,
      592,
      1033,
      12940,
      811,
      779,
      5329,
      23120,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      65,
      76,
      87
     ],
     "logits_raw": [
      1.448075771331787,
      0.11779351532459259,
      1.8871092796325684
     ],
     "probs_raw": [
      0.3551628291606903,
      0.0939059853553772,
      0.5509311556816101
     ],
     "probs": [
      0.36297598481178284,
      0.1403396874666214,
      0.49668434262275696
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.13370835781097412,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.49668434262275696,
      "probabilities": {
       "contradicted": 0.36297598481178284,
       "insufficient": 0.1403396874666214,
       "supported": 0.49668434262275696
      }
     }
    },
    "usage": {
     "input_tokens": 97,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_674ca4e9346216255466",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 66,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 66,
     "positions": 66,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      10423,
      57119,
      12199,
      19980,
      521,
      1203,
      46050,
      931,
      835,
      810,
      46050,
      931,
      879,
      3176,
      1550,
      2826,
      6289,
      2630,
      57119,
      933,
      22537,
      5742,
      523,
      18,
      15534,
      779,
      13719,
      8177,
      4262,
      811,
      779,
      19670,
      10423,
      57119,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      39,
      50,
      58
     ],
     "logits_raw": [
      3.3179497718811035,
      1.9383889436721802,
      1.3750910758972168
     ],
     "probs_raw": [
      0.7168546915054321,
      0.1804244965314865,
      0.10272078961133957
     ],
     "probs": [
      0.6161913871765137,
      0.23000140488147736,
      0.15380728244781494
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.3861899822950363,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6161913871765137,
      "probabilities": {
       "insufficient": 0.6161913871765137,
       "B": 0.23000140488147736,
       "A": 0.15380728244781494
      }
     }
    },
    "usage": {
     "input_tokens": 66,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_695dd61195ca025c4f4c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 69,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 69,
     "positions": 69,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      28356,
      893,
      44987,
      896,
      779,
      52414,
      1079,
      953,
      14465,
      523,
      46050,
      931,
      879,
      535,
      941,
      28356,
      893,
      7637,
      811,
      5514,
      768,
      52414,
      1079,
      27968,
      523,
      18,
      52739,
      779,
      13719,
      13591,
      768,
      3866,
      11746,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      42,
      50,
      61
     ],
     "logits_raw": [
      -0.545011043548584,
      -1.2503870725631714,
      -0.6067150831222534
     ],
     "probs_raw": [
      0.41083216667175293,
      0.2029193788766861,
      0.3862484395503998
     ],
     "probs": [
      0.3904627561569214,
      0.2359113097190857,
      0.3736259341239929
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.016836822032928467,
     "near_tie": true,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.3904627561569214,
      "probabilities": {
       "B": 0.3904627561569214,
       "insufficient": 0.2359113097190857,
       "A": 0.3736259341239929
      }
     }
    },
    "usage": {
     "input_tokens": 69,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_6d64a8459f64eac7f4cc",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 94,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 94,
     "positions": 94,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      45609,
      7330,
      9938,
      592,
      38183,
      811,
      779,
      797,
      41722,
      23120,
      1403,
      936,
      1011,
      2463,
      779,
      1336,
      536,
      1517,
      779,
      5329,
      23120,
      1550,
      1403,
      779,
      797,
      41722,
      23120,
      4584,
      523,
      42749,
      23120,
      1090,
      4978,
      811,
      2463,
      1033,
      12940,
      856,
      21706,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      45609,
      7330,
      9938,
      592,
      1033,
      12940,
      811,
      779,
      5329,
      23120,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      62,
      73,
      84
     ],
     "logits_raw": [
      0.3540732264518738,
      1.012641191482544,
      -0.8345171213150024
     ],
     "probs_raw": [
      0.30895915627479553,
      0.5969163775444031,
      0.09412453323602676
     ],
     "probs": [
      0.3301945626735687,
      0.5285418629646301,
      0.14126355946063995
     ],
     "argmax_index": 1,
     "argmax": "contradicted",
     "top2_margin": 0.1983473002910614,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.5285418629646301,
      "probabilities": {
       "insufficient": 0.3301945626735687,
       "contradicted": 0.5285418629646301,
       "supported": 0.14126355946063995
      }
     }
    },
    "usage": {
     "input_tokens": 94,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_6db17cc22bd656d78558",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3084,
      35283,
      779,
      26862,
      53076,
      810,
      4462,
      1856,
      1203,
      14350,
      13834,
      28553,
      5742,
      523,
      18,
      35516,
      7494,
      8544,
      1403,
      779,
      28553,
      856,
      8112,
      10274,
      536,
      797,
      19980,
      521,
      779,
      10423,
      4462,
      1856,
      1391,
      58340,
      779,
      26862,
      53076,
      523,
      3354,
      936,
      874,
      5789,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      47,
      59,
      70
     ],
     "logits_raw": [
      1.8385608196258545,
      2.026137590408325,
      2.54236102104187
     ],
     "probs_raw": [
      0.23653285205364227,
      0.285334974527359,
      0.4781322181224823
     ],
     "probs": [
      0.2633902430534363,
      0.3011564314365387,
      0.4354533553123474
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.13429692387580872,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4354533553123474,
      "probabilities": {
       "prohibited": 0.2633902430534363,
       "permitted": 0.3011564314365387,
       "insufficient": 0.4354533553123474
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_712f6ffb763eef7c9a12",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      10046,
      31494,
      768,
      18958,
      875,
      768,
      4025,
      17686,
      1550,
      811,
      17575,
      2723,
      936,
      523,
      46050,
      931,
      879,
      535,
      941,
      10046,
      30261,
      774,
      26677,
      768,
      4025,
      17686,
      523,
      18,
      52739,
      779,
      13719,
      12469,
      768,
      4025,
      17686,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      44,
      52,
      60
     ],
     "logits_raw": [
      -0.45648908615112305,
      0.1176786869764328,
      0.13668951392173767
     ],
     "probs_raw": [
      0.21808426082134247,
      0.38724181056022644,
      0.39467400312423706
     ],
     "probs": [
      0.24785038828849792,
      0.37352126836776733,
      0.37862834334373474
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.005107074975967407,
     "near_tie": true,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.37862834334373474,
      "probabilities": {
       "B": 0.24785038828849792,
       "A": 0.37352126836776733,
       "insufficient": 0.37862834334373474
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_75b470586834df8c568d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 67,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 67,
     "positions": 67,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      21522,
      52075,
      779,
      21008,
      1203,
      4852,
      1295,
      7082,
      1337,
      25583,
      1706,
      933,
      779,
      5776,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      21522,
      30261,
      774,
      779,
      21008,
      1090,
      50624,
      8294,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      35,
      45,
      56
     ],
     "logits_raw": [
      -4.853909492492676,
      0.4020563066005707,
      0.866544246673584
     ],
     "probs_raw": [
      0.0020090413745492697,
      0.3851463794708252,
      0.6128445863723755
     ],
     "probs": [
      0.009686571545898914,
      0.4137553572654724,
      0.5765581130981445
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.16280275583267212,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5765581130981445,
      "probabilities": {
       "supported": 0.009686571545898914,
       "contradicted": 0.4137553572654724,
       "insufficient": 0.5765581130981445
      }
     }
    },
    "usage": {
     "input_tokens": 67,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_76c7a79b1ff2363a60c5",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 62,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 62,
     "positions": 62,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      59063,
      3498,
      19459,
      42074,
      788,
      856,
      4175,
      523,
      50432,
      60468,
      856,
      22922,
      49550,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      19459,
      42074,
      788,
      856,
      4175,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      30,
      40,
      51
     ],
     "logits_raw": [
      3.787590980529785,
      1.6854655742645264,
      -0.33135542273521423
     ],
     "probs_raw": [
      0.878381073474884,
      0.10733503848314285,
      0.014283912256360054
     ],
     "probs": [
      0.7839997410774231,
      0.17464955151081085,
      0.04135073721408844
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.6093501895666122,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.7839997410774231,
      "probabilities": {
       "supported": 0.7839997410774231,
       "insufficient": 0.17464955151081085,
       "contradicted": 0.04135073721408844
      }
     }
    },
    "usage": {
     "input_tokens": 62,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_774f7f9245f83c796d44",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 70,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 70,
     "positions": 70,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7576,
      1090,
      53859,
      7330,
      856,
      21706,
      521,
      1203,
      46050,
      931,
      835,
      810,
      46050,
      931,
      879,
      3176,
      1550,
      1149,
      6289,
      810,
      1295,
      10562,
      37078,
      523,
      18,
      15534,
      779,
      13719,
      12467,
      6612,
      3346,
      17784,
      10562,
      1490,
      779,
      9519,
      53859,
      7330,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      43,
      51,
      59
     ],
     "logits_raw": [
      2.467068910598755,
      1.1629825830459595,
      1.8013712167739868
     ],
     "probs_raw": [
      0.5601188540458679,
      0.1520276963710785,
      0.2878533899784088
     ],
     "probs": [
      0.4961564838886261,
      0.1954556256532669,
      0.3083878755569458
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.1877686083316803,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.4961564838886261,
      "probabilities": {
       "B": 0.4961564838886261,
       "A": 0.1954556256532669,
       "insufficient": 0.3083878755569458
      }
     }
    },
    "usage": {
     "input_tokens": 70,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_796d5c0da6eaadae6a3f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      21009,
      30403,
      32340,
      521,
      1757,
      22922,
      46159,
      779,
      52847,
      2076,
      779,
      3308,
      523,
      46050,
      931,
      879,
      535,
      941,
      21009,
      30403,
      32340,
      810,
      779,
      5156,
      3084,
      44987,
      936,
      6728,
      797,
      5323,
      523,
      18,
      15534,
      779,
      13719,
      62439,
      902,
      52847,
      896,
      953,
      2919,
      47927,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      51,
      59,
      67
     ],
     "logits_raw": [
      3.8280439376831055,
      1.9527827501296997,
      1.7718702554702759
     ],
     "probs_raw": [
      0.7804832458496094,
      0.11965972185134888,
      0.09985702484846115
     ],
     "probs": [
      0.6701697111129761,
      0.17555658519268036,
      0.15427377820014954
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.4946131259202957,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.6701697111129761,
      "probabilities": {
       "B": 0.6701697111129761,
       "A": 0.17555658519268036,
       "insufficient": 0.15427377820014954
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_7ada97b6f99f4788e5c7",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 74,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 74,
     "positions": 74,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      42271,
      44987,
      43152,
      803,
      768,
      8825,
      5771,
      1203,
      22922,
      3498,
      1295,
      9170,
      6177,
      1178,
      1381,
      2047,
      523,
      46050,
      931,
      879,
      535,
      941,
      42271,
      44987,
      896,
      779,
      8825,
      953,
      14708,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      14708,
      8825,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      47,
      55,
      63
     ],
     "logits_raw": [
      0.2991751730442047,
      -2.6162571907043457,
      -0.412447452545166
     ],
     "probs_raw": [
      0.6472375988960266,
      0.03506771847605705,
      0.31769469380378723
     ],
     "probs": [
      0.579340934753418,
      0.07218901813030243,
      0.3484700620174408
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.23087087273597717,
     "near_tie": false,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.579340934753418,
      "probabilities": {
       "B": 0.579340934753418,
       "A": 0.07218901813030243,
       "insufficient": 0.3484700620174408
      }
     }
    },
    "usage": {
     "input_tokens": 74,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_7ed75d854d99b3a81916",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      856,
      16495,
      810,
      46050,
      931,
      879,
      856,
      779,
      3981,
      3321,
      521,
      1203,
      13834,
      13719,
      1090,
      4978,
      811,
      9936,
      779,
      3869,
      856,
      21706,
      523,
      18,
      52739,
      835,
      19148,
      835,
      856,
      3222,
      536,
      7056,
      879,
      1550,
      1403,
      835,
      856,
      12359,
      810,
      879,
      856,
      3222,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      49,
      60,
      68
     ],
     "logits_raw": [
      2.1733789443969727,
      1.3448199033737183,
      1.0819072723388672
     ],
     "probs_raw": [
      0.5642066597938538,
      0.2463766634464264,
      0.18941661715507507
     ],
     "probs": [
      0.49705639481544495,
      0.2750169038772583,
      0.22792671620845795
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.22203949093818665,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.49705639481544495,
      "probabilities": {
       "insufficient": 0.49705639481544495,
       "B": 0.2750169038772583,
       "A": 0.22792671620845795
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_823bf736b92e9d8fa66d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8170,
      15791,
      875,
      4611,
      521,
      1203,
      18724,
      2203,
      856,
      1014,
      21706,
      523,
      18,
      1098,
      1509,
      9764,
      833,
      59534,
      6835,
      1550,
      1509,
      9764,
      833,
      4611,
      523,
      3354,
      1033,
      8170,
      6984,
      936,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      35,
      47,
      64
     ],
     "logits_raw": [
      -1.5380306243896484,
      2.647414207458496,
      -0.08526135236024857
     ],
     "probs_raw": [
      0.014084970578551292,
      0.9257027506828308,
      0.06021236255764961
     ],
     "probs": [
      0.042185135185718536,
      0.8387300372123718,
      0.11908479779958725
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.7196452394127846,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8387300372123718,
      "probabilities": {
       "prohibited": 0.042185135185718536,
       "insufficient": 0.8387300372123718,
       "permitted": 0.11908479779958725
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_82c2cd97c160ad27951b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 62,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 62,
     "positions": 62,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      8449,
      16057,
      768,
      60621,
      523,
      46050,
      931,
      879,
      535,
      941,
      12153,
      6602,
      768,
      2375,
      3955,
      523,
      18,
      52739,
      779,
      13719,
      14065,
      811,
      12940,
      779,
      7605,
      8449,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      35,
      46,
      54
     ],
     "logits_raw": [
      0.8707722425460815,
      -2.1442296504974365,
      0.39768466353416443
     ],
     "probs_raw": [
      0.5980427265167236,
      0.029331453144550323,
      0.37262576818466187
     ],
     "probs": [
      0.5466614961624146,
      0.06344036757946014,
      0.3898981213569641
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.15676337480545044,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5466614961624146,
      "probabilities": {
       "insufficient": 0.5466614961624146,
       "B": 0.06344036757946014,
       "A": 0.3898981213569641
      }
     }
    },
    "usage": {
     "input_tokens": 62,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_834f3b3e10cc61b33d25",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 61,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 61,
     "positions": 61,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      59063,
      3498,
      19459,
      42074,
      788,
      856,
      4175,
      523,
      50432,
      60468,
      856,
      22922,
      49550,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      19459,
      60468,
      856,
      4175,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      29,
      39,
      50
     ],
     "logits_raw": [
      -2.509235382080078,
      1.4758033752441406,
      1.9639065265655518
     ],
     "probs_raw": [
      0.00702153705060482,
      0.37766996026039124,
      0.6153085231781006
     ],
     "probs": [
      0.023447468876838684,
      0.4040045142173767,
      0.572547972202301
     ],
     "argmax_index": 2,
     "argmax": "contradicted",
     "top2_margin": 0.16854345798492432,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.862645149230957e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.572547972202301,
      "probabilities": {
       "supported": 0.023447468876838684,
       "insufficient": 0.4040045142173767,
       "contradicted": 0.572547972202301
      }
     }
    },
    "usage": {
     "input_tokens": 61,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_843185706adff0c60f8b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 69,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 69,
     "positions": 69,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      10449,
      6835,
      3956,
      8649,
      810,
      768,
      5196,
      8170,
      521,
      1203,
      779,
      11746,
      3084,
      6602,
      1295,
      2203,
      1337,
      4021,
      8688,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      11860,
      2090,
      1321,
      856,
      5156,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      37,
      48,
      58
     ],
     "logits_raw": [
      0.3888823986053467,
      -8.769230842590332,
      1.5510286092758179
     ],
     "probs_raw": [
      0.2382715493440628,
      2.5104656742769293e-05,
      0.761703372001648
     ],
     "probs": [
      0.30347737669944763,
      0.00043750001350417733,
      0.6960850954055786
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.392607718706131,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9103830456733704e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6960850954055786,
      "probabilities": {
       "contradicted": 0.30347737669944763,
       "supported": 0.00043750001350417733,
       "insufficient": 0.6960850954055786
      }
     }
    },
    "usage": {
     "input_tokens": 69,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_86a15618ba4cb90a307e",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 64,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 64,
     "positions": 64,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3059,
      12930,
      12874,
      875,
      45853,
      779,
      61026,
      810,
      22922,
      3498,
      1295,
      45853,
      1178,
      1381,
      5813,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      61026,
      1178,
      1381,
      45943,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      32,
      43,
      54
     ],
     "logits_raw": [
      1.7928204536437988,
      1.682106852531433,
      -5.491433143615723
     ],
     "probs_raw": [
      0.5274591445922852,
      0.4721788465976715,
      0.0003619742637965828
     ],
     "probs": [
      0.5182804465293884,
      0.4788702130317688,
      0.002849374897778034
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.03941023349761963,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 2.3283064365386963e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.5182804465293884,
      "probabilities": {
       "contradicted": 0.5182804465293884,
       "insufficient": 0.4788702130317688,
       "supported": 0.002849374897778034
      }
     }
    },
    "usage": {
     "input_tokens": 64,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_8a4c3de28be95c105ca1",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      22773,
      2396,
      2240,
      3525,
      768,
      6643,
      59534,
      1203,
      4943,
      876,
      916,
      4289,
      592,
      17784,
      875,
      3507,
      3893,
      523,
      18,
      31678,
      3893,
      856,
      6767,
      1550,
      1720,
      3507,
      25252,
      980,
      17784,
      523,
      3354,
      936,
      6305,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      38,
      49,
      66
     ],
     "logits_raw": [
      -1.75178062915802,
      2.2357535362243652,
      0.6280560493469238
     ],
     "probs_raw": [
      0.015214934945106506,
      0.8204160332679749,
      0.16436903178691864
     ],
     "probs": [
      0.04213201254606247,
      0.7272379398345947,
      0.230630025267601
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.4966079145669937,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7272379398345947,
      "probabilities": {
       "permitted": 0.04213201254606247,
       "insufficient": 0.7272379398345947,
       "prohibited": 0.230630025267601
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_8c992e17e4fb54b044f5",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      28356,
      893,
      44987,
      896,
      779,
      52414,
      1079,
      953,
      14465,
      523,
      46050,
      931,
      879,
      535,
      941,
      28356,
      893,
      7637,
      811,
      5514,
      768,
      52414,
      1079,
      27968,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      7850,
      11746,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      41,
      49,
      57
     ],
     "logits_raw": [
      1.3221626281738281,
      0.9677733778953552,
      -2.26184344291687
     ],
     "probs_raw": [
      0.5782465934753418,
      0.40569886565208435,
      0.016054581850767136
     ],
     "probs": [
      0.5394794940948486,
      0.41882413625717163,
      0.041696369647979736
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.120655357837677,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.5394794940948486,
      "probabilities": {
       "B": 0.5394794940948486,
       "A": 0.41882413625717163,
       "insufficient": 0.041696369647979736
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_8e8c3804a3c15ebb31e3",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 73,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 73,
     "positions": 73,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      21522,
      16461,
      521,
      997,
      1098,
      21008,
      3498,
      25583,
      1706,
      938,
      21706,
      523,
      859,
      11667,
      535,
      7448,
      25583,
      1706,
      1166,
      4617,
      2013,
      1727,
      5776,
      2927,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      21522,
      38419,
      779,
      50624,
      8294,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      41,
      52,
      62
     ],
     "logits_raw": [
      -0.09096686542034149,
      -3.5231728553771973,
      1.6575649976730347
     ],
     "probs_raw": [
      0.14752578735351562,
      0.0047673797234892845,
      0.8477067947387695
     ],
     "probs": [
      0.21866793930530548,
      0.018836576491594315,
      0.7624955177307129
     ],
     "argmax_index": 2,
     "argmax": "contradicted",
     "top2_margin": 0.5438275784254074,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.862645149230957e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.7624955177307129,
      "probabilities": {
       "insufficient": 0.21866793930530548,
       "supported": 0.018836576491594315,
       "contradicted": 0.7624955177307129
      }
     }
    },
    "usage": {
     "input_tokens": 73,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_90494c76078ffc3c717a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 90,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 90,
     "positions": 90,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8414,
      22654,
      1178,
      768,
      12803,
      13789,
      810,
      779,
      28332,
      44987,
      1295,
      41095,
      953,
      5231,
      523,
      18,
      41914,
      1587,
      22879,
      8414,
      5799,
      523,
      45681,
      936,
      875,
      768,
      12803,
      13789,
      521,
      6794,
      896,
      5231,
      41095,
      3469,
      4201,
      936,
      797,
      808,
      16979,
      523,
      3354,
      1033,
      1789,
      874,
      11086,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      50,
      61,
      78
     ],
     "logits_raw": [
      3.778726100921631,
      1.7921221256256104,
      0.7049117684364319
     ],
     "probs_raw": [
      0.8450194001197815,
      0.11590322852134705,
      0.039077434688806534
     ],
     "probs": [
      0.7389857172966003,
      0.17878293991088867,
      0.08223139494657516
     ],
     "argmax_index": 0,
     "argmax": "permitted",
     "top2_margin": 0.5602027773857117,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.7389857172966003,
      "probabilities": {
       "permitted": 0.7389857172966003,
       "insufficient": 0.17878293991088867,
       "prohibited": 0.08223139494657516
      }
     }
    },
    "usage": {
     "input_tokens": 90,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_92ec2ffa3dd46ada1335",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 91,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 91,
     "positions": 91,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1789,
      5792,
      44117,
      60706,
      43675,
      9618,
      8414,
      25885,
      521,
      1203,
      22922,
      13748,
      774,
      1883,
      5896,
      1919,
      1403,
      779,
      29114,
      856,
      60706,
      523,
      1470,
      8414,
      29114,
      856,
      60706,
      1929,
      1883,
      11851,
      936,
      523,
      3253,
      1312,
      3360,
      856,
      2251,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1033,
      29114,
      4357,
      10812,
      875,
      1789,
      1490,
      779,
      9519,
      5792,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      59,
      70,
      81
     ],
     "logits_raw": [
      1.858945608139038,
      1.736254096031189,
      0.3094581365585327
     ],
     "probs_raw": [
      0.4768959879875183,
      0.4218319058418274,
      0.1012720987200737
     ],
     "probs": [
      0.4451003968715668,
      0.40775102376937866,
      0.14714854955673218
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.03734937310218811,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.4451003968715668,
      "probabilities": {
       "contradicted": 0.4451003968715668,
       "insufficient": 0.40775102376937866,
       "supported": 0.14714854955673218
      }
     }
    },
    "usage": {
     "input_tokens": 91,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_942ec65dac0bfb9874f9",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 62,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 62,
     "positions": 62,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2525,
      7867,
      8269,
      768,
      28623,
      22820,
      1203,
      1295,
      2565,
      933,
      27742,
      5742,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      16400,
      31286,
      1052,
      1381,
      25539,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      30,
      41,
      51
     ],
     "logits_raw": [
      3.149710178375244,
      -5.195099830627441,
      -0.9152504205703735
     ],
     "probs_raw": [
      0.9828963875770569,
      0.00023356221208814532,
      0.016870107501745224
     ],
     "probs": [
      0.945720911026001,
      0.0024374090135097504,
      0.051841605454683304
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.8938793055713177,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.945720911026001,
      "probabilities": {
       "insufficient": 0.945720911026001,
       "supported": 0.0024374090135097504,
       "contradicted": 0.051841605454683304
      }
     }
    },
    "usage": {
     "input_tokens": 62,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_9473603b44af63beab79",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8170,
      7850,
      1509,
      9764,
      833,
      4611,
      1203,
      1014,
      11820,
      37414,
      9007,
      4611,
      523,
      18,
      1098,
      1509,
      9764,
      833,
      59534,
      6835,
      1550,
      1509,
      9764,
      833,
      4611,
      523,
      3354,
      1033,
      8170,
      6984,
      936,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      36,
      47,
      64
     ],
     "logits_raw": [
      0.09622183442115784,
      2.969804525375366,
      -1.9030358791351318
     ],
     "probs_raw": [
      0.053090523928403854,
      0.9397191405296326,
      0.007190356031060219
     ],
     "probs": [
      0.11075764894485474,
      0.8626878261566162,
      0.026554502546787262
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.7519301772117615,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8626878261566162,
      "probabilities": {
       "permitted": 0.11075764894485474,
       "insufficient": 0.8626878261566162,
       "prohibited": 0.026554502546787262
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_960a4bcd27363e6db4b9",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      835,
      15001,
      1534,
      856,
      8815,
      1203,
      1352,
      4450,
      810,
      2919,
      3807,
      938,
      10144,
      523,
      46050,
      931,
      879,
      535,
      941,
      3308,
      2154,
      856,
      21706,
      2630,
      15001,
      3129,
      523,
      18,
      15534,
      779,
      13719,
      916,
      6654,
      6612,
      32340,
      15001,
      536,
      768,
      28687,
      32491,
      1706,
      902,
      7045,
      8825,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      52,
      60,
      68
     ],
     "logits_raw": [
      2.380281925201416,
      3.187776803970337,
      1.8507490158081055
     ],
     "probs_raw": [
      0.2610172927379608,
      0.585274875164032,
      0.15370787680149078
     ],
     "probs": [
      0.2885650098323822,
      0.5137538909912109,
      0.19768109917640686
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.22518888115882874,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.5137538909912109,
      "probabilities": {
       "B": 0.2885650098323822,
       "A": 0.5137538909912109,
       "insufficient": 0.19768109917640686
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_97a35499bd8717f3e786",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 92,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 92,
     "positions": 92,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2681,
      28117,
      1509,
      17436,
      1530,
      793,
      9544,
      521,
      1757,
      22922,
      15011,
      936,
      2076,
      779,
      5089,
      523,
      3253,
      2919,
      6177,
      9647,
      523,
      18,
      14875,
      779,
      2681,
      28117,
      1090,
      10500,
      15001,
      6177,
      536,
      768,
      28687,
      32491,
      1706,
      779,
      7045,
      8825,
      523,
      3354,
      779,
      8153,
      777,
      7157,
      779,
      1530,
      793,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      52,
      64,
      81
     ],
     "logits_raw": [
      1.8441698551177979,
      2.0381903648376465,
      1.5088130235671997
     ],
     "probs_raw": [
      0.34138965606689453,
      0.41448840498924255,
      0.24412190914154053
     ],
     "probs": [
      0.34064170718193054,
      0.39128151535987854,
      0.26807668805122375
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.050639808177948,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.39128151535987854,
      "probabilities": {
       "prohibited": 0.34064170718193054,
       "insufficient": 0.39128151535987854,
       "permitted": 0.26807668805122375
      }
     }
    },
    "usage": {
     "input_tokens": 92,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_97d2ed2591fc7638806c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 86,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 86,
     "positions": 86,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1789,
      5792,
      44117,
      60706,
      43675,
      9618,
      8414,
      25885,
      1203,
      13748,
      774,
      1883,
      5896,
      1919,
      1403,
      60706,
      523,
      941,
      29114,
      856,
      60706,
      521,
      1203,
      779,
      3360,
      810,
      6771,
      803,
      1883,
      5896,
      938,
      49033,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1033,
      29114,
      4357,
      10812,
      875,
      1789,
      1490,
      779,
      9519,
      5792,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      54,
      65,
      75
     ],
     "logits_raw": [
      1.4517431259155273,
      -2.3006997108459473,
      0.18750600516796112
     ],
     "probs_raw": [
      0.765746533870697,
      0.017964694648981094,
      0.21628868579864502
     ],
     "probs": [
      0.6784985661506653,
      0.04649622365832329,
      0.2750052809715271
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.4034932851791382,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6784985661506653,
      "probabilities": {
       "insufficient": 0.6784985661506653,
       "supported": 0.04649622365832329,
       "contradicted": 0.2750052809715271
      }
     }
    },
    "usage": {
     "input_tokens": 86,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_980e76145c925747220c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 98,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 98,
     "positions": 98,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      53076,
      7944,
      779,
      7494,
      28553,
      906,
      10274,
      523,
      941,
      10423,
      4462,
      1856,
      7944,
      936,
      906,
      946,
      38008,
      523,
      9758,
      7944,
      938,
      2835,
      523,
      18,
      35516,
      7494,
      8544,
      1403,
      779,
      28553,
      856,
      8112,
      10274,
      536,
      1490,
      1033,
      8530,
      13061,
      521,
      779,
      26862,
      53076,
      1391,
      58340,
      779,
      4462,
      1856,
      523,
      3354,
      936,
      874,
      5789,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      58,
      75,
      87
     ],
     "logits_raw": [
      2.910322427749634,
      2.614384651184082,
      2.6769022941589355
     ],
     "probs_raw": [
      0.3943755030632019,
      0.2933498024940491,
      0.3122747242450714
     ],
     "probs": [
      0.37652453780174255,
      0.30477800965309143,
      0.3186974823474884
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.05782705545425415,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.37652453780174255,
      "probabilities": {
       "insufficient": 0.37652453780174255,
       "prohibited": 0.30477800965309143,
       "permitted": 0.3186974823474884
      }
     }
    },
    "usage": {
     "input_tokens": 98,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_9ac5cc82c4126c91174a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 89,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 89,
     "positions": 89,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8414,
      22654,
      856,
      57077,
      521,
      1203,
      3984,
      41095,
      953,
      5231,
      856,
      1014,
      2538,
      523,
      18,
      41914,
      1587,
      22879,
      8414,
      5799,
      523,
      45681,
      936,
      875,
      768,
      12803,
      13789,
      521,
      6794,
      896,
      5231,
      41095,
      3469,
      4201,
      936,
      797,
      808,
      16979,
      523,
      3354,
      1033,
      1789,
      874,
      11086,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      49,
      60,
      77
     ],
     "logits_raw": [
      1.1729261875152588,
      2.151684284210205,
      0.8149329423904419
     ],
     "probs_raw": [
      0.22934584319591522,
      0.6103235483169556,
      0.16033059358596802
     ],
     "probs": [
      0.2641015648841858,
      0.5313907265663147,
      0.2045077234506607
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.2672891616821289,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5313907265663147,
      "probabilities": {
       "permitted": 0.2641015648841858,
       "insufficient": 0.5313907265663147,
       "prohibited": 0.2045077234506607
      }
     }
    },
    "usage": {
     "input_tokens": 89,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a0128e551d095f3449f8",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7576,
      6602,
      8376,
      2314,
      10562,
      811,
      51132,
      16993,
      18577,
      536,
      779,
      3084,
      44987,
      1033,
      8825,
      856,
      6612,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      8376,
      1178,
      10562,
      811,
      51132,
      16993,
      18577,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      36,
      46,
      57
     ],
     "logits_raw": [
      6.080697059631348,
      0.6055939793586731,
      -2.5855391025543213
     ],
     "probs_raw": [
      0.9956568479537964,
      0.004171600099653006,
      0.0001715581165626645
     ],
     "probs": [
      0.9784088134765625,
      0.01958679035305977,
      0.0020043274853378534
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.9588220231235027,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9784088134765625,
      "probabilities": {
       "supported": 0.9784088134765625,
       "insufficient": 0.01958679035305977,
       "contradicted": 0.0020043274853378534
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a02fd582d2881cc462b2",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 85,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 85,
     "positions": 85,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      33648,
      1090,
      37181,
      6175,
      13546,
      779,
      30122,
      9041,
      810,
      4943,
      876,
      13748,
      774,
      779,
      1232,
      33388,
      920,
      2543,
      10074,
      523,
      18,
      542,
      37181,
      6175,
      22558,
      9544,
      8544,
      811,
      779,
      3508,
      936,
      13546,
      523,
      3354,
      779,
      33648,
      7157,
      779,
      30122,
      9041,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      45,
      57,
      74
     ],
     "logits_raw": [
      2.111523151397705,
      1.9371681213378906,
      3.128459930419922
     ],
     "probs_raw": [
      0.2171688675880432,
      0.18242156505584717,
      0.6004095673561096
     ],
     "probs": [
      0.25312647223472595,
      0.22348424792289734,
      0.5233892798423767
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.27026280760765076,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.5233892798423767,
      "probabilities": {
       "prohibited": 0.25312647223472595,
       "insufficient": 0.22348424792289734,
       "permitted": 0.5233892798423767
      }
     }
    },
    "usage": {
     "input_tokens": 85,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a0a39ace7e5337d038a7",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 83,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 83,
     "positions": 83,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3863,
      12153,
      22558,
      9365,
      779,
      6332,
      2154,
      1203,
      4943,
      876,
      23369,
      2796,
      9365,
      779,
      11256,
      8355,
      523,
      18,
      18693,
      902,
      3863,
      2853,
      1550,
      1403,
      779,
      12153,
      3575,
      6229,
      9365,
      896,
      2853,
      523,
      3354,
      779,
      11256,
      8355,
      874,
      7093,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      43,
      55,
      66
     ],
     "logits_raw": [
      4.502979278564453,
      2.016558885574341,
      1.5348275899887085
     ],
     "probs_raw": [
      0.881363570690155,
      0.07333587110042572,
      0.045300524681806564
     ],
     "probs": [
      0.7756208181381226,
      0.13130463659763336,
      0.09307458251714706
     ],
     "argmax_index": 0,
     "argmax": "prohibited",
     "top2_margin": 0.6443161815404892,
     "near_tie": false,
     "gold": "prohibited",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.7756208181381226,
      "probabilities": {
       "prohibited": 0.7756208181381226,
       "permitted": 0.13130463659763336,
       "insufficient": 0.09307458251714706
      }
     }
    },
    "usage": {
     "input_tokens": 83,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a22dab8ba073ff6cdb8d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 94,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 94,
     "positions": 94,
     "bucket": 256,
     "ids": [
      1,
      17,
      49290,
      5019,
      11528,
      6261,
      875,
      20155,
      1230,
      1798,
      521,
      6794,
      896,
      10144,
      39208,
      3469,
      1134,
      15419,
      2563,
      523,
      46050,
      931,
      835,
      535,
      4411,
      5019,
      1230,
      1047,
      8896,
      38120,
      39273,
      779,
      19214,
      523,
      46050,
      931,
      879,
      535,
      4411,
      5019,
      1230,
      1047,
      8896,
      38120,
      1178,
      779,
      19214,
      523,
      9758,
      1230,
      1798,
      938,
      20155,
      523,
      18,
      15534,
      779,
      38120,
      46102,
      1490,
      779,
      9519,
      11287,
      810,
      32294,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      67,
      78,
      86
     ],
     "logits_raw": [
      2.378375768661499,
      0.7194362282752991,
      1.3831477165222168
     ],
     "probs_raw": [
      0.6410338878631592,
      0.1220148503780365,
      0.23695123195648193
     ],
     "probs": [
      0.5565075874328613,
      0.1701427549123764,
      0.27334970235824585
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.2831578850746155,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5565075874328613,
      "probabilities": {
       "insufficient": 0.5565075874328613,
       "B": 0.1701427549123764,
       "A": 0.27334970235824585
      }
     }
    },
    "usage": {
     "input_tokens": 94,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a3f18f3a63d45345942b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 70,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 70,
     "positions": 70,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2525,
      7867,
      13055,
      16400,
      31286,
      523,
      941,
      23120,
      44987,
      1186,
      1052,
      1014,
      4763,
      1381,
      25539,
      811,
      779,
      13608,
      1090,
      28623,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      16400,
      31286,
      1052,
      1381,
      25539,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      38,
      48,
      59
     ],
     "logits_raw": [
      -10.35418701171875,
      2.370882034301758,
      1.7727797031402588
     ],
     "probs_raw": [
      1.919910801007063e-06,
      0.6452207565307617,
      0.3547772765159607
     ],
     "probs": [
      6.825674063293263e-05,
      0.605174720287323,
      0.394756942987442
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.21041777729988098,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.605174720287323,
      "probabilities": {
       "supported": 6.825674063293263e-05,
       "insufficient": 0.605174720287323,
       "contradicted": 0.394756942987442
      }
     }
    },
    "usage": {
     "input_tokens": 70,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_a9230edfc6ec9e524af0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      542,
      5156,
      13509,
      13443,
      6835,
      2059,
      768,
      39037,
      810,
      768,
      33407,
      2312,
      523,
      46050,
      931,
      835,
      535,
      9758,
      10621,
      938,
      12930,
      523,
      46050,
      931,
      879,
      535,
      11005,
      779,
      39037,
      856,
      12930,
      536,
      779,
      33407,
      2312,
      856,
      26117,
      523,
      18,
      15534,
      779,
      5156,
      13509,
      13443,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      48,
      56,
      67
     ],
     "logits_raw": [
      -0.8022832274436951,
      0.6754550933837891,
      0.012994378805160522
     ],
     "probs_raw": [
      0.1308416724205017,
      0.5734819173812866,
      0.29567641019821167
     ],
     "probs": [
      0.17655335366725922,
      0.5073632001876831,
      0.31608352065086365
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.19127967953681946,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5073632001876831,
      "probabilities": {
       "B": 0.17655335366725922,
       "insufficient": 0.5073632001876831,
       "A": 0.31608352065086365
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_acacd8b46acd25cb954b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 94,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 94,
     "positions": 94,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      4887,
      11917,
      856,
      11649,
      58754,
      521,
      2158,
      779,
      2254,
      856,
      3161,
      810,
      11500,
      523,
      18,
      21509,
      779,
      2254,
      1550,
      906,
      768,
      3981,
      3321,
      1403,
      779,
      4887,
      11917,
      856,
      58754,
      933,
      4994,
      3972,
      1108,
      536,
      9141,
      2643,
      779,
      4887,
      11917,
      523,
      3354,
      779,
      2254,
      874,
      2643,
      791,
      1490,
      1033,
      5792,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      54,
      71,
      82
     ],
     "logits_raw": [
      1.6015324592590332,
      4.51754903793335,
      1.0473811626434326
     ],
     "probs_raw": [
      0.04989488422870636,
      0.9214375615119934,
      0.028667587786912918
     ],
     "probs": [
      0.10307374596595764,
      0.8275465965270996,
      0.06937965005636215
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.724472850561142,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.8275465965270996,
      "probabilities": {
       "insufficient": 0.10307374596595764,
       "permitted": 0.8275465965270996,
       "prohibited": 0.06937965005636215
      }
     }
    },
    "usage": {
     "input_tokens": 94,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b45c63fb92729af27f05",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 83,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 83,
     "positions": 83,
     "bucket": 256,
     "ids": [
      1,
      17,
      3312,
      17980,
      779,
      12153,
      30403,
      33300,
      1450,
      52518,
      523,
      2357,
      22092,
      1689,
      22922,
      46159,
      896,
      52847,
      523,
      941,
      3084,
      3498,
      17688,
      1486,
      23496,
      7045,
      15001,
      521,
      810,
      1295,
      2919,
      15001,
      953,
      2997,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      15001,
      811,
      19382,
      779,
      52518,
      856,
      6654,
      6612,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      51,
      62,
      72
     ],
     "logits_raw": [
      3.0344765186309814,
      -6.221322059631348,
      1.7618658542633057
     ],
     "probs_raw": [
      0.7811310291290283,
      7.464173540938646e-05,
      0.2187943458557129
     ],
     "probs": [
      0.7121281027793884,
      0.0009574248106218874,
      0.2869144380092621
     ],
     "argmax_index": 0,
     "argmax": "contradicted",
     "top2_margin": 0.42521366477012634,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 5.820766091346741e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.7121281027793884,
      "probabilities": {
       "contradicted": 0.7121281027793884,
       "supported": 0.0009574248106218874,
       "insufficient": 0.2869144380092621
      }
     }
    },
    "usage": {
     "input_tokens": 83,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b49ecda75086d8c68d2a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 61,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 61,
     "positions": 61,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      28356,
      893,
      6602,
      902,
      4332,
      523,
      46050,
      931,
      879,
      535,
      941,
      52414,
      1079,
      6034,
      809,
      814,
      969,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      7850,
      11746,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      34,
      42,
      50
     ],
     "logits_raw": [
      1.6683297157287598,
      -2.664180040359497,
      -0.32061368227005005
     ],
     "probs_raw": [
      0.8695845007896423,
      0.01142159104347229,
      0.11899389326572418
     ],
     "probs": [
      0.7771173715591431,
      0.035188402980566025,
      0.1876942366361618
     ],
     "argmax_index": 0,
     "argmax": "B",
     "top2_margin": 0.5894231349229813,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.7771173715591431,
      "probabilities": {
       "B": 0.7771173715591431,
       "A": 0.035188402980566025,
       "insufficient": 0.1876942366361618
      }
     }
    },
    "usage": {
     "input_tokens": 61,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b531dd603bf080dab820",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 84,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 84,
     "positions": 84,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      49104,
      21706,
      779,
      19089,
      16687,
      1203,
      22922,
      2172,
      1014,
      7018,
      779,
      37610,
      1126,
      523,
      18,
      8475,
      32144,
      2524,
      2090,
      1321,
      8544,
      1403,
      2059,
      779,
      19089,
      16687,
      810,
      37610,
      1126,
      1052,
      1381,
      21706,
      523,
      3354,
      2524,
      2090,
      1321,
      874,
      12043,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      44,
      56,
      67
     ],
     "logits_raw": [
      -0.07308841496706009,
      2.8655457496643066,
      3.741370916366577
     ],
     "probs_raw": [
      0.015327489003539085,
      0.2895366847515106,
      0.695135772228241
     ],
     "probs": [
      0.04096178337931633,
      0.33422577381134033,
      0.6248124837875366
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.2905867099761963,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6248124837875366,
      "probabilities": {
       "prohibited": 0.04096178337931633,
       "permitted": 0.33422577381134033,
       "insufficient": 0.6248124837875366
      }
     }
    },
    "usage": {
     "input_tokens": 84,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b5e6788d95b8395d2888",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      11860,
      2090,
      1321,
      6835,
      2059,
      779,
      3956,
      8649,
      810,
      779,
      5196,
      8170,
      523,
      941,
      11746,
      3084,
      44987,
      779,
      3956,
      8649,
      11423,
      810,
      22922,
      3498,
      779,
      8170,
      953,
      1014,
      14465,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      3956,
      8649,
      1166,
      14465,
      523,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      46,
      57,
      68
     ],
     "logits_raw": [
      0.7756969332695007,
      0.1709093302488327,
      -0.5731619596481323
     ],
     "probs_raw": [
      0.5537936687469482,
      0.30247679352760315,
      0.1437295377254486
     ],
     "probs": [
      0.49243271350860596,
      0.3196847140789032,
      0.18788257241249084
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.17274799942970276,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.49243271350860596,
      "probabilities": {
       "insufficient": 0.49243271350860596,
       "contradicted": 0.3196847140789032,
       "supported": 0.18788257241249084
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b7c4e62d07986286fce0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      2797,
      6765,
      1090,
      15001,
      7627,
      9647,
      521,
      1203,
      779,
      5582,
      936,
      6007,
      938,
      49033,
      523,
      18,
      560,
      965,
      16711,
      1550,
      916,
      779,
      6765,
      1090,
      11599,
      15001,
      523,
      3354,
      16711,
      874,
      4997,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      35,
      52,
      63
     ],
     "logits_raw": [
      2.9559473991394043,
      -0.06785887479782104,
      0.48010173439979553
     ],
     "probs_raw": [
      0.8828402757644653,
      0.04292000085115433,
      0.07423967868089676
     ],
     "probs": [
      0.7776677012443542,
      0.08968294411897659,
      0.13264939188957214
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.6450183093547821,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7776677012443542,
      "probabilities": {
       "insufficient": 0.7776677012443542,
       "permitted": 0.08968294411897659,
       "prohibited": 0.13264939188957214
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b8091be929b1e489fcc4",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      10720,
      768,
      25473,
      813,
      1914,
      521,
      768,
      3338,
      7061,
      1165,
      779,
      12037,
      803,
      768,
      33390,
      7267,
      523,
      941,
      3893,
      6634,
      4478,
      1033,
      856,
      37640,
      810,
      1295,
      2151,
      7267,
      953,
      7935,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      768,
      2151,
      7267,
      953,
      7935,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      46,
      56,
      67
     ],
     "logits_raw": [
      -9.860912322998047,
      4.097040176391602,
      1.2136549949645996
     ],
     "probs_raw": [
      8.212900866055861e-07,
      0.9470182657241821,
      0.05298099294304848
     ],
     "probs": [
      4.146222636336461e-05,
      0.8868883848190308,
      0.11307018250226974
     ],
     "argmax_index": 1,
     "argmax": "contradicted",
     "top2_margin": 0.773818202316761,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.8868883848190308,
      "probabilities": {
       "supported": 4.146222636336461e-05,
       "contradicted": 0.8868883848190308,
       "insufficient": 0.11307018250226974
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b863998d4be2721a5dfd",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 92,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 92,
     "positions": 92,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      45609,
      7330,
      9938,
      592,
      38183,
      811,
      779,
      797,
      41722,
      23120,
      1403,
      936,
      1011,
      2463,
      779,
      1336,
      536,
      1517,
      779,
      5329,
      23120,
      1550,
      1403,
      779,
      797,
      41722,
      23120,
      4584,
      523,
      9758,
      29142,
      10547,
      1186,
      1011,
      2463,
      1033,
      12940,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      45609,
      7330,
      9938,
      592,
      1033,
      12940,
      811,
      779,
      5329,
      23120,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      60,
      70,
      81
     ],
     "logits_raw": [
      -0.9867814779281616,
      0.012546658515930176,
      -0.5340939164161682
     ],
     "probs_raw": [
      0.18907205760478973,
      0.5136059522628784,
      0.29732200503349304
     ],
     "probs": [
      0.22605858743190765,
      0.4615784287452698,
      0.31236302852630615
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.14921540021896362,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4615784287452698,
      "probabilities": {
       "supported": 0.22605858743190765,
       "insufficient": 0.4615784287452698,
       "contradicted": 0.31236302852630615
      }
     }
    },
    "usage": {
     "input_tokens": 92,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b8c5b9b285cbdddc9f8d",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      856,
      779,
      16495,
      57490,
      1203,
      856,
      11649,
      12359,
      811,
      9936,
      1033,
      3869,
      523,
      46050,
      931,
      879,
      856,
      779,
      3981,
      3321,
      810,
      856,
      11649,
      3222,
      523,
      18,
      52739,
      835,
      19148,
      835,
      856,
      3222,
      536,
      7056,
      879,
      1550,
      1403,
      835,
      856,
      12359,
      810,
      879,
      856,
      3222,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      51,
      59,
      70
     ],
     "logits_raw": [
      0.475014865398407,
      1.241917371749878,
      2.9685797691345215
     ],
     "probs_raw": [
      0.0655418112874031,
      0.1411171853542328,
      0.7933409810066223
     ],
     "probs": [
      0.11538352072238922,
      0.19955480098724365,
      0.6850616335868835
     ],
     "argmax_index": 2,
     "argmax": "B",
     "top2_margin": 0.4855068325996399,
     "near_tie": false,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.6850616335868835,
      "probabilities": {
       "A": 0.11538352072238922,
       "insufficient": 0.19955480098724365,
       "B": 0.6850616335868835
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_b8e2750307df9761e716",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      49104,
      21706,
      2059,
      779,
      19089,
      16687,
      810,
      779,
      37610,
      1126,
      523,
      18,
      8475,
      32144,
      2524,
      2090,
      1321,
      8544,
      1403,
      2059,
      779,
      19089,
      16687,
      810,
      37610,
      1126,
      1052,
      1381,
      21706,
      523,
      3354,
      2524,
      2090,
      1321,
      874,
      12043,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      41,
      52,
      64
     ],
     "logits_raw": [
      6.782066822052002,
      -1.3062357902526855,
      3.482503652572632
     ],
     "probs_raw": [
      0.9641282558441162,
      0.0002960940182674676,
      0.03557564318180084
     ],
     "probs": [
      0.9109132885932922,
      0.0028198061045259237,
      0.08626683801412582
     ],
     "argmax_index": 0,
     "argmax": "permitted",
     "top2_margin": 0.8246464505791664,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.9109132885932922,
      "probabilities": {
       "permitted": 0.9109132885932922,
       "prohibited": 0.0028198061045259237,
       "insufficient": 0.08626683801412582
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_be2dd67bc733c1213bef",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      49104,
      21706,
      779,
      19089,
      16687,
      1203,
      22922,
      2172,
      1014,
      7018,
      779,
      37610,
      1126,
      523,
      18,
      8475,
      32144,
      779,
      14144,
      58931,
      1403,
      779,
      16687,
      1178,
      1381,
      21706,
      536,
      779,
      37610,
      1126,
      856,
      1014,
      4225,
      875,
      1033,
      58931,
      523,
      3354,
      1033,
      58931,
      874,
      12043,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      47,
      59,
      70
     ],
     "logits_raw": [
      0.8967389464378357,
      1.5261001586914062,
      3.427116632461548
     ],
     "probs_raw": [
      0.06478925049304962,
      0.12157130241394043,
      0.8136394619941711
     ],
     "probs": [
      0.11543203890323639,
      0.18095681071281433,
      0.7036111950874329
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.5226543843746185,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7036111950874329,
      "probabilities": {
       "prohibited": 0.11543203890323639,
       "permitted": 0.18095681071281433,
       "insufficient": 0.7036111950874329
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_be866e1dbec5d42d9423",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      1789,
      5792,
      44117,
      60706,
      43675,
      9618,
      8414,
      25885,
      521,
      1203,
      13748,
      774,
      1883,
      5896,
      1919,
      1403,
      60706,
      523,
      1470,
      8414,
      29114,
      1178,
      768,
      12803,
      45836,
      15868,
      521,
      916,
      1295,
      1883,
      5896,
      933,
      1312,
      32294,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1033,
      29114,
      4357,
      10812,
      875,
      1789,
      1490,
      779,
      9519,
      5792,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      56,
      67,
      78
     ],
     "logits_raw": [
      1.425513744354248,
      1.8368489742279053,
      0.7128006219863892
     ],
     "probs_raw": [
      0.3334285318851471,
      0.5030873417854309,
      0.16348405182361603
     ],
     "probs": [
      0.33983734250068665,
      0.455911785364151,
      0.20425090193748474
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.11607444286346436,
     "near_tie": false,
     "gold": "supported",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.455911785364151,
      "probabilities": {
       "contradicted": 0.33983734250068665,
       "insufficient": 0.455911785364151,
       "supported": 0.20425090193748474
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_c100e84585149a04be49",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      542,
      5156,
      13509,
      13443,
      6835,
      2059,
      768,
      39037,
      810,
      768,
      33407,
      2312,
      523,
      46050,
      931,
      835,
      535,
      9758,
      10621,
      938,
      12930,
      523,
      46050,
      931,
      879,
      535,
      11005,
      779,
      39037,
      856,
      12930,
      536,
      779,
      33407,
      2312,
      856,
      26117,
      523,
      18,
      15534,
      779,
      13443,
      8960,
      779,
      39037,
      1203,
      10144,
      779,
      33407,
      2312,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      54,
      62,
      70
     ],
     "logits_raw": [
      -0.07913510501384735,
      -0.2963172197341919,
      0.7918941378593445
     ],
     "probs_raw": [
      0.23842720687389374,
      0.19188226759433746,
      0.5696905255317688
     ],
     "probs": [
      0.2688654065132141,
      0.23022787272930145,
      0.5009067058563232
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.23204129934310913,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5009067058563232,
      "probabilities": {
       "B": 0.2688654065132141,
       "A": 0.23022787272930145,
       "insufficient": 0.5009067058563232
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_c4e9bd463b6fde5be743",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      542,
      5156,
      13443,
      6835,
      2059,
      10621,
      523,
      46050,
      931,
      835,
      810,
      46050,
      931,
      879,
      3619,
      1550,
      13443,
      20317,
      810,
      1295,
      28261,
      2203,
      523,
      18,
      15534,
      779,
      5156,
      13509,
      13443,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      33,
      41,
      52
     ],
     "logits_raw": [
      2.186413526535034,
      1.998923420906067,
      2.937082290649414
     ],
     "probs_raw": [
      0.253327876329422,
      0.21001826226711273,
      0.5366538763046265
     ],
     "probs": [
      0.27900218963623047,
      0.24402931332588196,
      0.47696855664253235
     ],
     "argmax_index": 2,
     "argmax": "A",
     "top2_margin": 0.19796636700630188,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.47696855664253235,
      "probabilities": {
       "B": 0.27900218963623047,
       "insufficient": 0.24402931332588196,
       "A": 0.47696855664253235
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_c6a411e7f2bbdf377173",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 92,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 92,
     "positions": 92,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7576,
      6516,
      903,
      1653,
      10562,
      811,
      51132,
      16993,
      18577,
      810,
      22922,
      23369,
      1339,
      3759,
      53859,
      523,
      903,
      1653,
      1757,
      5689,
      8376,
      811,
      51132,
      1377,
      523,
      941,
      7330,
      4478,
      902,
      18135,
      42183,
      53859,
      25274,
      1295,
      10562,
      536,
      8376,
      1178,
      1295,
      1312,
      8825,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      8376,
      1178,
      10562,
      811,
      51132,
      16993,
      18577,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      60,
      71,
      82
     ],
     "logits_raw": [
      1.1703366041183472,
      1.2382922172546387,
      -0.3189012110233307
     ],
     "probs_raw": [
      0.4355661869049072,
      0.4661942422389984,
      0.09823955595493317
     ],
     "probs": [
      0.41755813360214233,
      0.438327819108963,
      0.14411407709121704
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.02076968550682068,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.438327819108963,
      "probabilities": {
       "contradicted": 0.41755813360214233,
       "insufficient": 0.438327819108963,
       "supported": 0.14411407709121704
      }
     }
    },
    "usage": {
     "input_tokens": 92,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_c89b86e3f55123bfa506",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      6634,
      11854,
      768,
      2151,
      5854,
      7267,
      7935,
      5329,
      768,
      4795,
      810,
      4943,
      876,
      3498,
      779,
      12037,
      953,
      1014,
      37640,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      7267,
      12037,
      953,
      37640,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      36,
      46,
      57
     ],
     "logits_raw": [
      -8.718061447143555,
      3.4809868335723877,
      1.940075397491455
     ],
     "probs_raw": [
      4.146997525822371e-06,
      0.8235937356948853,
      0.17640209197998047
     ],
     "probs": [
      0.0001232229551533237,
      0.7503045797348022,
      0.24957218766212463
     ],
     "argmax_index": 1,
     "argmax": "contradicted",
     "top2_margin": 0.5007323920726776,
     "near_tie": false,
     "gold": "contradicted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4551915228366852e-11,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "contradicted",
      "confidence": 0.7503045797348022,
      "probabilities": {
       "supported": 0.0001232229551533237,
       "contradicted": 0.7503045797348022,
       "insufficient": 0.24957218766212463
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_caf773b23934fa7362a1",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      12153,
      30403,
      33300,
      779,
      52518,
      884,
      17980,
      523,
      941,
      5156,
      15001,
      3084,
      44987,
      896,
      1689,
      1178,
      1014,
      47927,
      936,
      810,
      936,
      6728,
      6612,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      15001,
      811,
      19382,
      779,
      52518,
      856,
      6654,
      6612,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      43,
      53,
      64
     ],
     "logits_raw": [
      3.9008922576904297,
      -2.43294620513916,
      0.15340781211853027
     ],
     "probs_raw": [
      0.9752746224403381,
      0.0017313139978796244,
      0.0229940302670002
     ],
     "probs": [
      0.9262592196464539,
      0.010040778666734695,
      0.06370002031326294
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.8625591993331909,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9262592196464539,
      "probabilities": {
       "supported": 0.9262592196464539,
       "contradicted": 0.010040778666734695,
       "insufficient": 0.06370002031326294
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_cdf7f428857e1b841cb3",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 84,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 84,
     "positions": 84,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7330,
      22558,
      2314,
      53859,
      968,
      779,
      7576,
      1203,
      23369,
      2796,
      3759,
      53859,
      523,
      46050,
      931,
      835,
      535,
      4411,
      1550,
      13289,
      10562,
      4010,
      988,
      879,
      523,
      46050,
      931,
      879,
      535,
      941,
      7576,
      5828,
      15011,
      1123,
      3346,
      17784,
      10562,
      523,
      18,
      15534,
      779,
      13719,
      12467,
      6612,
      3346,
      17784,
      10562,
      1490,
      779,
      9519,
      53859,
      7330,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      57,
      65,
      76
     ],
     "logits_raw": [
      2.717790365219116,
      1.3965704441070557,
      2.6717827320098877
     ],
     "probs_raw": [
      0.45007655024528503,
      0.120084747672081,
      0.42983871698379517
     ],
     "probs": [
      0.4243011772632599,
      0.16511568427085876,
      0.41058313846588135
     ],
     "argmax_index": 0,
     "argmax": "A",
     "top2_margin": 0.01371803879737854,
     "near_tie": true,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.4243011772632599,
      "probabilities": {
       "A": 0.4243011772632599,
       "insufficient": 0.16511568427085876,
       "B": 0.41058313846588135
      }
     }
    },
    "usage": {
     "input_tokens": 84,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_cdfa9eed51fafcf09b72",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 82,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 82,
     "positions": 82,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      16396,
      5828,
      3575,
      6229,
      779,
      16701,
      811,
      51132,
      41955,
      523,
      18,
      1098,
      5937,
      11430,
      1400,
      51132,
      41955,
      1203,
      1400,
      1014,
      40500,
      896,
      10562,
      523,
      11005,
      30403,
      2396,
      2053,
      1400,
      51132,
      523,
      3354,
      779,
      16701,
      51132,
      768,
      3346,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      42,
      59,
      71
     ],
     "logits_raw": [
      1.991889238357544,
      0.2586843967437744,
      4.35775089263916
     ],
     "probs_raw": [
      0.08453133702278137,
      0.014938133768737316,
      0.9005304574966431
     ],
     "probs": [
      0.1490413397550583,
      0.043212514370679855,
      0.807746171951294
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.6587048321962357,
     "near_tie": false,
     "gold": "permitted",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.807746171951294,
      "probabilities": {
       "insufficient": 0.1490413397550583,
       "prohibited": 0.043212514370679855,
       "permitted": 0.807746171951294
      }
     }
    },
    "usage": {
     "input_tokens": 82,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d0e36c4cb10a29b3191c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      33648,
      1178,
      768,
      37181,
      6175,
      1203,
      1352,
      8594,
      3508,
      938,
      1014,
      21706,
      523,
      18,
      542,
      37181,
      6175,
      22558,
      9544,
      8544,
      811,
      779,
      3508,
      936,
      13546,
      523,
      3354,
      779,
      33648,
      7157,
      779,
      30122,
      9041,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      38,
      55,
      66
     ],
     "logits_raw": [
      3.2459259033203125,
      3.2828853130340576,
      1.2678097486495972
     ],
     "probs_raw": [
      0.45956292748451233,
      0.47686588764190674,
      0.06357114017009735
     ],
     "probs": [
      0.44049862027168274,
      0.4522833526134491,
      0.10721801966428757
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.011784732341766357,
     "near_tie": true,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.4522833526134491,
      "probabilities": {
       "insufficient": 0.44049862027168274,
       "permitted": 0.4522833526134491,
       "prohibited": 0.10721801966428757
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d167e51bb1f499099645",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      10423,
      57119,
      12199,
      19980,
      523,
      46050,
      931,
      835,
      535,
      941,
      57119,
      3498,
      8177,
      810,
      22537,
      3498,
      1014,
      8177,
      523,
      46050,
      931,
      879,
      535,
      941,
      57119,
      3498,
      1014,
      8177,
      810,
      22537,
      3498,
      8177,
      523,
      18,
      15534,
      779,
      13719,
      8177,
      4262,
      811,
      779,
      19670,
      10423,
      57119,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      49,
      57,
      65
     ],
     "logits_raw": [
      0.7584027051925659,
      0.9608060121536255,
      1.1365561485290527
     ],
     "probs_raw": [
      0.27144932746887207,
      0.33234673738479614,
      0.3962039053440094
     ],
     "probs": [
      0.28854256868362427,
      0.3334279954433441,
      0.37802937626838684
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.044601380825042725,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.37802937626838684,
      "probabilities": {
       "A": 0.28854256868362427,
       "B": 0.3334279954433441,
       "insufficient": 0.37802937626838684
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d41072dc35d8cd14c056",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 82,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 82,
     "positions": 82,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8170,
      7850,
      1509,
      9764,
      833,
      4611,
      1203,
      1014,
      11820,
      37414,
      9007,
      4611,
      523,
      18,
      15246,
      779,
      8530,
      7330,
      521,
      779,
      59534,
      6835,
      2059,
      1509,
      9764,
      833,
      810,
      11820,
      37414,
      9007,
      4611,
      523,
      3354,
      1033,
      8170,
      6984,
      936,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      42,
      53,
      65
     ],
     "logits_raw": [
      0.1964302361011505,
      0.45343083143234253,
      1.8850579261779785
     ],
     "probs_raw": [
      0.12978427112102509,
      0.16781702637672424,
      0.7023987770080566
     ],
     "probs": [
      0.18042559921741486,
      0.21678446233272552,
      0.6027899384498596
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.3860054761171341,
     "near_tie": false,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6027899384498596,
      "probabilities": {
       "permitted": 0.18042559921741486,
       "prohibited": 0.21678446233272552,
       "insufficient": 0.6027899384498596
      }
     }
    },
    "usage": {
     "input_tokens": 82,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d51b05bfb9d6d7f09331",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      2797,
      16701,
      4117,
      884,
      779,
      3893,
      521,
      1203,
      1295,
      17784,
      25274,
      933,
      53859,
      3129,
      938,
      4617,
      523,
      18,
      1098,
      5937,
      11430,
      1400,
      51132,
      41955,
      1203,
      1400,
      1014,
      40500,
      896,
      10562,
      523,
      11005,
      30403,
      2396,
      2053,
      1400,
      51132,
      523,
      3354,
      779,
      16701,
      51132,
      768,
      3346,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      48,
      60,
      77
     ],
     "logits_raw": [
      1.244246482849121,
      2.568499803543091,
      2.9976813793182373
     ],
     "probs_raw": [
      0.0949326828122139,
      0.3568877577781677,
      0.548179566860199
     ],
     "probs": [
      0.14135213196277618,
      0.3640234172344208,
      0.49462443590164185
     ],
     "argmax_index": 2,
     "argmax": "permitted",
     "top2_margin": 0.13060101866722107,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.49462443590164185,
      "probabilities": {
       "prohibited": 0.14135213196277618,
       "insufficient": 0.3640234172344208,
       "permitted": 0.49462443590164185
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d6de731cca52359b1c87",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      42271,
      6602,
      1352,
      2154,
      523,
      46050,
      931,
      879,
      535,
      941,
      5771,
      1178,
      768,
      7392,
      2080,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      14708,
      8825,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      33,
      44,
      52
     ],
     "logits_raw": [
      -0.32768940925598145,
      -2.296766757965088,
      -3.953437328338623
     ],
     "probs_raw": [
      0.8574749231338501,
      0.1196911409497261,
      0.022833848372101784
     ],
     "probs": [
      0.7575771808624268,
      0.18558989465236664,
      0.05683298781514168
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.5719872862100601,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7575771808624268,
      "probabilities": {
       "insufficient": 0.7575771808624268,
       "A": 0.18558989465236664,
       "B": 0.05683298781514168
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d7e84e4245a29817da1f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      12153,
      21090,
      811,
      1039,
      578,
      786,
      779,
      7605,
      8449,
      2158,
      10512,
      1352,
      6212,
      523,
      46050,
      931,
      879,
      535,
      941,
      12153,
      21090,
      811,
      7827,
      779,
      8449,
      810,
      6812,
      768,
      5156,
      16400,
      523,
      18,
      52739,
      779,
      13719,
      14065,
      875,
      5156,
      16400,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      48,
      59,
      67
     ],
     "logits_raw": [
      -1.3053580522537231,
      -1.3997042179107666,
      -3.440410614013672
     ],
     "probs_raw": [
      0.4930465519428253,
      0.44865643978118896,
      0.058297015726566315
     ],
     "probs": [
      0.4645945131778717,
      0.4343149662017822,
      0.10109055042266846
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.030279546976089478,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4645945131778717,
      "probabilities": {
       "insufficient": 0.4645945131778717,
       "A": 0.4343149662017822,
       "B": 0.10109055042266846
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_d880e65fef5e2993fdd0",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 88,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 88,
     "positions": 88,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      8414,
      5799,
      22654,
      1178,
      768,
      12803,
      13789,
      810,
      1236,
      6034,
      5231,
      41095,
      523,
      9758,
      11445,
      938,
      11649,
      523,
      18,
      49290,
      57077,
      5799,
      1919,
      1288,
      8414,
      933,
      10372,
      916,
      41095,
      536,
      13834,
      4576,
      1391,
      58340,
      779,
      13789,
      11287,
      523,
      3354,
      1033,
      1789,
      874,
      11086,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      48,
      59,
      76
     ],
     "logits_raw": [
      1.5926122665405273,
      1.731564998626709,
      0.0559811033308506
     ],
     "probs_raw": [
      0.4229806661605835,
      0.4860343039035797,
      0.09098505973815918
     ],
     "probs": [
      0.4101724624633789,
      0.45297500491142273,
      0.13685256242752075
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.04280254244804382,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.45297500491142273,
      "probabilities": {
       "permitted": 0.4101724624633789,
       "insufficient": 0.45297500491142273,
       "prohibited": 0.13685256242752075
      }
     }
    },
    "usage": {
     "input_tokens": 88,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_def9fe59d262357a673a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 61,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 61,
     "positions": 61,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      18779,
      856,
      3460,
      523,
      46050,
      931,
      879,
      535,
      941,
      1927,
      770,
      856,
      797,
      768,
      64098,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      6023,
      1927,
      1725,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      34,
      45,
      53
     ],
     "logits_raw": [
      -0.8470367193222046,
      1.5626904964447021,
      -4.1368608474731445
     ],
     "probs_raw": [
      0.08218152821063995,
      0.9147562980651855,
      0.0030621157493442297
     ],
     "probs": [
      0.1495330035686493,
      0.83620685338974,
      0.0142601877450943
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.6866738498210907,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.83620685338974,
      "probabilities": {
       "insufficient": 0.1495330035686493,
       "A": 0.83620685338974,
       "B": 0.0142601877450943
      }
     }
    },
    "usage": {
     "input_tokens": 61,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_e0c140e9222d1bce5647",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      21522,
      16461,
      521,
      997,
      1098,
      21008,
      3498,
      25583,
      1706,
      938,
      21706,
      523,
      859,
      11667,
      535,
      7448,
      25583,
      1706,
      1166,
      4617,
      2013,
      1727,
      5776,
      2927,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      21522,
      30261,
      774,
      779,
      21008,
      1090,
      50624,
      8294,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      44,
      54,
      65
     ],
     "logits_raw": [
      4.24207878112793,
      -1.3785492181777954,
      0.46038830280303955
     ],
     "probs_raw": [
      0.9742728471755981,
      0.003529172856360674,
      0.022197969257831573
     ],
     "probs": [
      0.9215284585952759,
      0.01662665419280529,
      0.06184489652514458
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.8596835620701313,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.9215284585952759,
      "probabilities": {
       "supported": 0.9215284585952759,
       "contradicted": 0.01662665419280529,
       "insufficient": 0.06184489652514458
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_e1d610bd14e3d16c09b9",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      10046,
      31494,
      768,
      18958,
      875,
      768,
      4025,
      17686,
      1550,
      811,
      17575,
      2723,
      936,
      523,
      46050,
      931,
      879,
      535,
      941,
      10046,
      30261,
      774,
      26677,
      768,
      4025,
      17686,
      523,
      18,
      52739,
      779,
      13719,
      36154,
      768,
      4025,
      17686,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      44,
      52,
      63
     ],
     "logits_raw": [
      0.24006280303001404,
      0.42934364080429077,
      -0.9660718441009521
     ],
     "probs_raw": [
      0.3987666666507721,
      0.48186177015304565,
      0.11937165260314941
     ],
     "probs": [
      0.389519065618515,
      0.4459127187728882,
      0.16456826031208038
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.05639365315437317,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.4459127187728882,
      "probabilities": {
       "A": 0.389519065618515,
       "insufficient": 0.4459127187728882,
       "B": 0.16456826031208038
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_e332aa927d0c2ed044aa",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 93,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 93,
     "positions": 93,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7576,
      6516,
      903,
      1653,
      10562,
      811,
      51132,
      16993,
      18577,
      810,
      22922,
      23369,
      1339,
      3759,
      53859,
      523,
      903,
      1653,
      1757,
      5689,
      8376,
      811,
      51132,
      1377,
      523,
      941,
      7330,
      4478,
      902,
      18135,
      42183,
      53859,
      25274,
      1295,
      10562,
      536,
      8376,
      1178,
      1295,
      1312,
      8825,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      903,
      1653,
      1178,
      10562,
      811,
      51132,
      16993,
      18577,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      61,
      72,
      83
     ],
     "logits_raw": [
      0.6507751941680908,
      0.9516758918762207,
      1.8932214975357056
     ],
     "probs_raw": [
      0.17196454107761383,
      0.23233702778816223,
      0.5956984758377075
     ],
     "probs": [
      0.21418289840221405,
      0.26554253697395325,
      0.5202745199203491
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.2547319829463959,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.5202745199203491,
      "probabilities": {
       "contradicted": 0.21418289840221405,
       "insufficient": 0.26554253697395325,
       "supported": 0.5202745199203491
      }
     }
    },
    "usage": {
     "input_tokens": 93,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_e6155431ab7b0f8170d4",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 60,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 60,
     "positions": 60,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      859,
      1011,
      5514,
      768,
      52414,
      1079,
      27968,
      523,
      46050,
      931,
      879,
      535,
      941,
      52414,
      1079,
      953,
      14465,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      7850,
      11746,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      33,
      44,
      52
     ],
     "logits_raw": [
      -0.8405516147613525,
      0.2589983344078064,
      -0.3213144540786743
     ],
     "probs_raw": [
      0.17594608664512634,
      0.5283334255218506,
      0.29572051763534546
     ],
     "probs": [
      0.21540363132953644,
      0.47246497869491577,
      0.31213143467903137
     ],
     "argmax_index": 1,
     "argmax": "B",
     "top2_margin": 0.1603335440158844,
     "near_tie": false,
     "gold": "B",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "B",
      "confidence": 0.47246497869491577,
      "probabilities": {
       "insufficient": 0.21540363132953644,
       "B": 0.47246497869491577,
       "A": 0.31213143467903137
      }
     }
    },
    "usage": {
     "input_tokens": 60,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ea7ec66a5e8eac886548",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 80,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 80,
     "positions": 80,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      22773,
      2396,
      2240,
      3525,
      768,
      6643,
      59534,
      1203,
      4943,
      876,
      916,
      4289,
      592,
      17784,
      875,
      3507,
      3893,
      523,
      18,
      542,
      6643,
      1400,
      874,
      2047,
      1403,
      6643,
      3893,
      856,
      12043,
      523,
      3354,
      779,
      6643,
      874,
      2047,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      40,
      51,
      68
     ],
     "logits_raw": [
      -0.49817439913749695,
      1.6548174619674683,
      1.9368813037872314
     ],
     "probs_raw": [
      0.04755781963467598,
      0.40950050950050354,
      0.542941689491272
     ],
     "probs": [
      0.08811099082231522,
      0.410165399312973,
      0.5017235279083252
     ],
     "argmax_index": 2,
     "argmax": "prohibited",
     "top2_margin": 0.09155812859535217,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.5017235279083252,
      "probabilities": {
       "permitted": 0.08811099082231522,
       "insufficient": 0.410165399312973,
       "prohibited": 0.5017235279083252
      }
     }
    },
    "usage": {
     "input_tokens": 80,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_eac5121a4a1bf391a08c",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 58,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 58,
     "positions": 58,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      59063,
      15994,
      2059,
      55501,
      1203,
      4654,
      1512,
      1149,
      7845,
      5742,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      19459,
      60468,
      856,
      4175,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      26,
      37,
      48
     ],
     "logits_raw": [
      2.0061943531036377,
      3.7291605472564697,
      -3.016935348510742
     ],
     "probs_raw": [
      0.15133854746818542,
      0.8476650714874268,
      0.0009963964112102985
     ],
     "probs": [
      0.2246410846710205,
      0.7691481113433838,
      0.006210819352418184
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.5445070266723633,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7691481113433838,
      "probabilities": {
       "contradicted": 0.2246410846710205,
       "insufficient": 0.7691481113433838,
       "supported": 0.006210819352418184
      }
     }
    },
    "usage": {
     "input_tokens": 58,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_eafc22c8c40df3932a8e",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 77,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 77,
     "positions": 77,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      13061,
      6602,
      779,
      30174,
      17150,
      521,
      1203,
      779,
      2835,
      30174,
      810,
      26862,
      2811,
      952,
      21148,
      875,
      779,
      2391,
      931,
      938,
      49033,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1490,
      779,
      21706,
      13061,
      521,
      779,
      2391,
      931,
      856,
      6654,
      797,
      8784,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      21
     ],
     "markers": [
      45,
      55,
      66
     ],
     "logits_raw": [
      -2.8015472888946533,
      0.15241731703281403,
      2.426419258117676
     ],
     "probs_raw": [
      0.004840384237468243,
      0.09284752607345581,
      0.9023120999336243
     ],
     "probs": [
      0.01956266537308693,
      0.16137823462486267,
      0.8190590739250183
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.6576808393001556,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8190590739250183,
      "probabilities": {
       "supported": 0.01956266537308693,
       "contradicted": 0.16137823462486267,
       "insufficient": 0.8190590739250183
      }
     }
    },
    "usage": {
     "input_tokens": 77,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ebd74aa05324a1556288",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 77,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 77,
     "positions": 77,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      12153,
      21090,
      811,
      1039,
      578,
      786,
      779,
      7605,
      8449,
      2158,
      10512,
      1352,
      6212,
      523,
      46050,
      931,
      879,
      535,
      941,
      12153,
      21090,
      811,
      7827,
      779,
      8449,
      810,
      6812,
      768,
      5156,
      16400,
      523,
      18,
      52739,
      779,
      13719,
      14065,
      811,
      12940,
      779,
      7605,
      8449,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      50,
      61,
      69
     ],
     "logits_raw": [
      -1.8500902652740479,
      -1.6692605018615723,
      -4.712647914886475
     ],
     "probs_raw": [
      0.44339337944984436,
      0.5312789082527161,
      0.02532772719860077
     ],
     "probs": [
      0.4410553276538849,
      0.5018713474273682,
      0.05707336589694023
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.060816019773483276,
     "near_tie": false,
     "gold": "A",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.5018713474273682,
      "probabilities": {
       "insufficient": 0.4410553276538849,
       "A": 0.5018713474273682,
       "B": 0.05707336589694023
      }
     }
    },
    "usage": {
     "input_tokens": 77,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ebea4681e3dcc7d92ca2",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      856,
      779,
      16495,
      57490,
      810,
      856,
      11649,
      3222,
      811,
      9936,
      1033,
      3869,
      523,
      46050,
      931,
      879,
      856,
      779,
      3981,
      3321,
      57490,
      810,
      856,
      1236,
      3222,
      811,
      9936,
      936,
      523,
      18,
      15246,
      779,
      8530,
      17150,
      521,
      7056,
      879,
      19148,
      879,
      856,
      3222,
      536,
      7056,
      835,
      1550,
      1403,
      879,
      856,
      12359,
      810,
      835,
      856,
      3222,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      21
     ],
     "markers": [
      60,
      71,
      79
     ],
     "logits_raw": [
      2.0848593711853027,
      1.2152581214904785,
      2.261540174484253
     ],
     "probs_raw": [
      0.3827943205833435,
      0.1604362428188324,
      0.4567693769931793
     ],
     "probs": [
      0.37427595257759094,
      0.2011004239320755,
      0.42462357878685
     ],
     "argmax_index": 2,
     "argmax": "A",
     "top2_margin": 0.05034762620925903,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.42462357878685,
      "probabilities": {
       "insufficient": 0.37427595257759094,
       "B": 0.2011004239320755,
       "A": 0.42462357878685
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ec94ef909f7918695ab3",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 67,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 67,
     "positions": 67,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      40348,
      3575,
      5405,
      592,
      779,
      5562,
      811,
      4963,
      810,
      4943,
      876,
      22879,
      592,
      779,
      7045,
      5105,
      5562,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      779,
      35101,
      5562,
      856,
      797,
      5105,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      35,
      45,
      56
     ],
     "logits_raw": [
      -11.032297134399414,
      0.18279242515563965,
      -0.3950861096382141
     ],
     "probs_raw": [
      8.628144314570818e-06,
      0.6405735611915588,
      0.35941776633262634
     ],
     "probs": [
      0.0001995474158320576,
      0.6016390919685364,
      0.3981613516807556
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.20347774028778076,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6016390919685364,
      "probabilities": {
       "supported": 0.0001995474158320576,
       "insufficient": 0.6016390919685364,
       "contradicted": 0.3981613516807556
      }
     }
    },
    "usage": {
     "input_tokens": 67,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_edb708bd0fe5fd67d6ee",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 99,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 99,
     "positions": 99,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2681,
      28117,
      1509,
      6767,
      768,
      8153,
      777,
      811,
      7157,
      779,
      1530,
      793,
      523,
      13629,
      779,
      5089,
      1689,
      22922,
      46159,
      896,
      15001,
      523,
      3253,
      2919,
      6177,
      9647,
      523,
      18,
      3111,
      1033,
      37640,
      13061,
      521,
      779,
      1509,
      8112,
      6177,
      856,
      13827,
      810,
      2919,
      17688,
      1486,
      938,
      21867,
      523,
      3354,
      779,
      8153,
      777,
      7157,
      779,
      1530,
      793,
      540,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      59,
      70,
      82
     ],
     "logits_raw": [
      0.6859942078590393,
      0.7516666054725647,
      2.53814697265625
     ],
     "probs_raw": [
      0.11846372485160828,
      0.12650465965270996,
      0.7550315856933594
     ],
     "probs": [
      0.17232663929462433,
      0.1806035041809082,
      0.6470698714256287
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.46646636724472046,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.6470698714256287,
      "probabilities": {
       "permitted": 0.17232663929462433,
       "prohibited": 0.1806035041809082,
       "insufficient": 0.6470698714256287
      }
     }
    },
    "usage": {
     "input_tokens": 99,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_eeae283841ce75dd25bf",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 83,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 83,
     "positions": 83,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3863,
      12153,
      22558,
      9365,
      779,
      6332,
      2154,
      1203,
      4943,
      876,
      23369,
      2796,
      9365,
      779,
      11256,
      8355,
      523,
      18,
      18693,
      902,
      3863,
      2853,
      1550,
      1403,
      779,
      12153,
      3575,
      6229,
      9365,
      896,
      2853,
      523,
      3354,
      779,
      6332,
      2154,
      874,
      7093,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      43,
      60,
      71
     ],
     "logits_raw": [
      1.9277441501617432,
      1.6321382522583008,
      3.0559921264648438
     ],
     "probs_raw": [
      0.20685440301895142,
      0.1539163440465927,
      0.6392292380332947
     ],
     "probs": [
      0.24700720608234406,
      0.199987530708313,
      0.5530052781105042
     ],
     "argmax_index": 2,
     "argmax": "prohibited",
     "top2_margin": 0.3059980720281601,
     "near_tie": false,
     "gold": "permitted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.5530052781105042,
      "probabilities": {
       "insufficient": 0.24700720608234406,
       "permitted": 0.199987530708313,
       "prohibited": 0.5530052781105042
      }
     }
    },
    "usage": {
     "input_tokens": 83,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ef3e0f8e657bf59f8a29",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 90,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 90,
     "positions": 90,
     "bucket": 256,
     "ids": [
      1,
      17,
      3111,
      1033,
      4889,
      521,
      779,
      2835,
      10423,
      57119,
      12199,
      52010,
      1403,
      936,
      19980,
      916,
      22537,
      523,
      46050,
      931,
      835,
      535,
      40831,
      3498,
      8177,
      521,
      1203,
      779,
      10423,
      57119,
      3498,
      1014,
      8177,
      523,
      46050,
      931,
      879,
      535,
      40831,
      3498,
      1014,
      8177,
      521,
      1203,
      779,
      10423,
      57119,
      3498,
      8177,
      523,
      18,
      15534,
      779,
      13719,
      8177,
      4262,
      811,
      779,
      19670,
      10423,
      57119,
      523,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      63,
      71,
      82
     ],
     "logits_raw": [
      -0.09687026590108871,
      1.2967870235443115,
      0.46955496072769165
     ],
     "probs_raw": [
      0.1472424864768982,
      0.5933225154876709,
      0.2594349682331085
     ],
     "probs": [
      0.19212587177753448,
      0.5199291706085205,
      0.2879450023174286
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.23198416829109192,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5199291706085205,
      "probabilities": {
       "A": 0.19212587177753448,
       "insufficient": 0.5199291706085205,
       "B": 0.2879450023174286
      }
     }
    },
    "usage": {
     "input_tokens": 90,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f2b4ec4930322fe45118",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 80,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 80,
     "positions": 80,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      6765,
      22558,
      10004,
      2782,
      768,
      2984,
      797,
      902,
      7477,
      4611,
      62248,
      810,
      4943,
      876,
      23369,
      2796,
      18109,
      16711,
      523,
      18,
      560,
      965,
      16711,
      1550,
      916,
      779,
      6765,
      1090,
      11599,
      15001,
      523,
      3354,
      16711,
      874,
      4997,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      40,
      57,
      69
     ],
     "logits_raw": [
      2.063524007797241,
      3.0282185077667236,
      0.5558172464370728
     ],
     "probs_raw": [
      0.26005077362060547,
      0.6823695302009583,
      0.057579707354307175
     ],
     "probs": [
      0.30006879568099976,
      0.597724199295044,
      0.1022069901227951
     ],
     "argmax_index": 1,
     "argmax": "prohibited",
     "top2_margin": 0.2976554036140442,
     "near_tie": false,
     "gold": "prohibited",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.597724199295044,
      "probabilities": {
       "insufficient": 0.30006879568099976,
       "prohibited": 0.597724199295044,
       "permitted": 0.1022069901227951
      }
     }
    },
    "usage": {
     "input_tokens": 80,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f3d13cf184f916677def",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 95,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 95,
     "positions": 95,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      26862,
      3446,
      3498,
      779,
      2391,
      931,
      856,
      797,
      8784,
      523,
      941,
      51746,
      30174,
      3498,
      936,
      1178,
      3191,
      8784,
      523,
      9758,
      7944,
      938,
      2835,
      523,
      941,
      21706,
      13061,
      3498,
      779,
      51746,
      30174,
      12199,
      6249,
      9448,
      19148,
      1518,
      1547,
      5617,
      9171,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      1490,
      779,
      21706,
      13061,
      521,
      779,
      2391,
      931,
      856,
      6654,
      797,
      8784,
      523,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      21
     ],
     "markers": [
      63,
      74,
      85
     ],
     "logits_raw": [
      1.1147890090942383,
      1.3649086952209473,
      3.313030481338501
     ],
     "probs_raw": [
      0.08854779601097107,
      0.11371122300624847,
      0.7977409362792969
     ],
     "probs": [
      0.14278262853622437,
      0.1707146316766739,
      0.6865026950836182
     ],
     "argmax_index": 2,
     "argmax": "supported",
     "top2_margin": 0.5157880634069443,
     "near_tie": false,
     "gold": "contradicted",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.6865026950836182,
      "probabilities": {
       "contradicted": 0.14278262853622437,
       "insufficient": 0.1707146316766739,
       "supported": 0.6865026950836182
      }
     }
    },
    "usage": {
     "input_tokens": 95,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f40beba9088c8db8bbd6",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      2525,
      7867,
      13055,
      16400,
      31286,
      523,
      941,
      23120,
      44987,
      1186,
      1052,
      1014,
      4763,
      1381,
      25539,
      811,
      779,
      13608,
      1090,
      28623,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      16400,
      31286,
      1166,
      13055,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      36,
      46,
      57
     ],
     "logits_raw": [
      9.068342208862305,
      0.18565919995307922,
      -5.66766357421875
     ],
     "probs_raw": [
      0.9998608827590942,
      0.00013875206059310585,
      3.9826630882089376e-07
     ],
     "probs": [
      0.998221218585968,
      0.0017519677057862282,
      2.6769057512865402e-05
     ],
     "argmax_index": 0,
     "argmax": "supported",
     "top2_margin": 0.9964692508801818,
     "near_tie": false,
     "gold": "supported",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "supported",
      "confidence": 0.998221218585968,
      "probabilities": {
       "supported": 0.998221218585968,
       "insufficient": 0.0017519677057862282,
       "contradicted": 2.6769057512865402e-05
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f41ae66b3c09630c7a42",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 68,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 68,
     "positions": 68,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      27022,
      779,
      8449,
      810,
      5466,
      768,
      16400,
      523,
      46050,
      931,
      879,
      535,
      2382,
      578,
      786,
      779,
      7605,
      8449,
      810,
      3089,
      1352,
      6212,
      523,
      18,
      52739,
      779,
      13719,
      14065,
      811,
      12940,
      779,
      7605,
      8449,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      41,
      52,
      60
     ],
     "logits_raw": [
      0.4962294399738312,
      -0.13544343411922455,
      0.3153257966041565
     ],
     "probs_raw": [
      0.4226154386997223,
      0.2247052937746048,
      0.3526792526245117
     ],
     "probs": [
      0.3975161612033844,
      0.25315651297569275,
      0.34932732582092285
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.04818883538246155,
     "near_tie": false,
     "gold": "B",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.3975161612033844,
      "probabilities": {
       "insufficient": 0.3975161612033844,
       "A": 0.25315651297569275,
       "B": 0.34932732582092285
      }
     }
    },
    "usage": {
     "input_tokens": 68,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f46f392ef9e9e9df564b",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 73,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 73,
     "positions": 73,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      12153,
      30403,
      33300,
      884,
      17980,
      523,
      941,
      21706,
      3084,
      4654,
      1512,
      1113,
      2919,
      15001,
      3807,
      810,
      2177,
      1014,
      6122,
      2835,
      28195,
      523,
      18,
      9886,
      894,
      779,
      4673,
      535,
      15001,
      811,
      19382,
      779,
      52518,
      856,
      6654,
      6612,
      523,
      19,
      16,
      7951,
      535,
      941,
      4589,
      48308,
      779,
      4673,
      20,
      19,
      16,
      28441,
      535,
      941,
      4589,
      2177,
      1014,
      6122,
      4021,
      20,
      19,
      16,
      18197,
      13735,
      535,
      941,
      4589,
      48308,
      779,
      11977,
      20,
      21
     ],
     "markers": [
      41,
      51,
      62
     ],
     "logits_raw": [
      -1.2643077373504639,
      2.459071397781372,
      -1.124413251876831
     ],
     "probs_raw": [
      0.022959887981414795,
      0.9506327509880066,
      0.02640737034380436
     ],
     "probs": [
      0.060983750969171524,
      0.8716233968734741,
      0.06739288568496704
     ],
     "argmax_index": 1,
     "argmax": "insufficient",
     "top2_margin": 0.8042305111885071,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.8716233968734741,
      "probabilities": {
       "supported": 0.060983750969171524,
       "insufficient": 0.8716233968734741,
       "contradicted": 0.06739288568496704
      }
     }
    },
    "usage": {
     "input_tokens": 73,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f6d68126b43f8c9ed183",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      6634,
      8269,
      896,
      779,
      7850,
      33037,
      1927,
      7410,
      523,
      46050,
      931,
      879,
      535,
      941,
      6634,
      15994,
      33037,
      29067,
      7441,
      1203,
      3498,
      1295,
      1927,
      953,
      2685,
      523,
      18,
      52739,
      779,
      13719,
      15603,
      902,
      6023,
      1927,
      1725,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      44,
      55,
      63
     ],
     "logits_raw": [
      -1.3054568767547607,
      -1.7994532585144043,
      -3.502875328063965
     ],
     "probs_raw": [
      0.580965518951416,
      0.3544952869415283,
      0.06453920900821686
     ],
     "probs": [
      0.5233494639396667,
      0.36773747205734253,
      0.10891313850879669
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.15561199188232422,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5233494639396667,
      "probabilities": {
       "insufficient": 0.5233494639396667,
       "A": 0.36773747205734253,
       "B": 0.10891313850879669
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_f756e1cb04e11766341e",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 96,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 96,
     "positions": 96,
     "bucket": 256,
     "ids": [
      1,
      17,
      3111,
      1033,
      4889,
      521,
      779,
      2835,
      10423,
      57119,
      12199,
      52010,
      1403,
      936,
      19980,
      916,
      22537,
      523,
      46050,
      931,
      835,
      535,
      40831,
      3498,
      8177,
      521,
      1203,
      779,
      10423,
      57119,
      3498,
      1014,
      8177,
      523,
      46050,
      931,
      879,
      535,
      40831,
      3498,
      1014,
      8177,
      521,
      1203,
      779,
      10423,
      57119,
      3498,
      8177,
      523,
      18,
      15246,
      779,
      8530,
      13061,
      521,
      22537,
      12199,
      19980,
      523,
      22269,
      779,
      13719,
      8177,
      4262,
      811,
      22537,
      523,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      21
     ],
     "markers": [
      69,
      80,
      88
     ],
     "logits_raw": [
      -0.13149817287921906,
      -2.9378244876861572,
      -0.3618924617767334
     ],
     "probs_raw": [
      0.5391861796379089,
      0.03258117288351059,
      0.42823266983032227
     ],
     "probs": [
      0.5042976140975952,
      0.0679316371679306,
      0.4277706742286682
     ],
     "argmax_index": 0,
     "argmax": "insufficient",
     "top2_margin": 0.076526939868927,
     "near_tie": false,
     "gold": "A",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.5042976140975952,
      "probabilities": {
       "insufficient": 0.5042976140975952,
       "A": 0.0679316371679306,
       "B": 0.4277706742286682
      }
     }
    },
    "usage": {
     "input_tokens": 96,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_fb73a85d386cf59b0c51",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 71,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 71,
     "positions": 71,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      22773,
      12800,
      768,
      6177,
      1203,
      1352,
      3893,
      12757,
      856,
      58754,
      523,
      18,
      31678,
      3893,
      856,
      6767,
      1550,
      1720,
      3507,
      25252,
      980,
      17784,
      523,
      3354,
      936,
      6305,
      540,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      31,
      48,
      59
     ],
     "logits_raw": [
      1.5704766511917114,
      -1.7801976203918457,
      2.1247355937957764
     ],
     "probs_raw": [
      0.3602679967880249,
      0.0126312505453825,
      0.6271007061004639
     ],
     "probs": [
      0.38803747296333313,
      0.035431068390607834,
      0.5765314698219299
     ],
     "argmax_index": 2,
     "argmax": "prohibited",
     "top2_margin": 0.1884939968585968,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "prohibited",
      "confidence": 0.5765314698219299,
      "probabilities": {
       "insufficient": 0.38803747296333313,
       "permitted": 0.035431068390607834,
       "prohibited": 0.5765314698219299
      }
     }
    },
    "usage": {
     "input_tokens": 71,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_fda0ca94652c6d739ecf",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 84,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 84,
     "positions": 84,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      37181,
      6175,
      4943,
      876,
      13748,
      774,
      779,
      30122,
      9041,
      810,
      13546,
      1550,
      779,
      1232,
      33388,
      920,
      2543,
      10074,
      523,
      18,
      542,
      37181,
      6175,
      22558,
      9544,
      8544,
      811,
      779,
      3508,
      936,
      13546,
      523,
      3354,
      779,
      33648,
      7157,
      779,
      30122,
      9041,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      44,
      56,
      67
     ],
     "logits_raw": [
      2.2747561931610107,
      1.4293830394744873,
      2.278024196624756
     ],
     "probs_raw": [
      0.41107088327407837,
      0.17651264369487762,
      0.4124164879322052
     ],
     "probs": [
      0.3923072814941406,
      0.21446861326694489,
      0.3932241201400757
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.0009168386459350586,
     "near_tie": true,
     "gold": "prohibited",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.3932241201400757,
      "probabilities": {
       "prohibited": 0.3923072814941406,
       "permitted": 0.21446861326694489,
       "insufficient": 0.3932241201400757
      }
     }
    },
    "usage": {
     "input_tokens": 84,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_fdb9fc214aea2710763a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 58,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 58,
     "positions": 58,
     "bucket": 256,
     "ids": [
      1,
      17,
      55945,
      931,
      835,
      535,
      941,
      13608,
      6602,
      768,
      2154,
      523,
      46050,
      931,
      879,
      535,
      941,
      11019,
      856,
      12534,
      523,
      18,
      52739,
      779,
      13719,
      3575,
      4651,
      41071,
      3665,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      31,
      39,
      47
     ],
     "logits_raw": [
      0.5282842516899109,
      -4.471530914306641,
      1.901053786277771
     ],
     "probs_raw": [
      0.20189766585826874,
      0.0013606270076707006,
      0.7967416644096375
     ],
     "probs": [
      0.2706928849220276,
      0.0076097361743450165,
      0.7216973900794983
     ],
     "argmax_index": 2,
     "argmax": "insufficient",
     "top2_margin": 0.4510045051574707,
     "near_tie": false,
     "gold": "insufficient",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "insufficient",
      "confidence": 0.7216973900794983,
      "probabilities": {
       "B": 0.2706928849220276,
       "A": 0.0076097361743450165,
       "insufficient": 0.7216973900794983
      }
     }
    },
    "usage": {
     "input_tokens": 58,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_fdd902cc6e65fad7cb4f",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 91,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 91,
     "positions": 91,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      7330,
      22558,
      2314,
      53859,
      968,
      779,
      7576,
      1203,
      23369,
      2796,
      1523,
      3759,
      53859,
      523,
      46050,
      931,
      835,
      535,
      859,
      5068,
      3346,
      17784,
      10562,
      5828,
      988,
      779,
      7576,
      523,
      46050,
      931,
      879,
      535,
      859,
      5068,
      936,
      1550,
      988,
      835,
      521,
      810,
      1052,
      1295,
      1312,
      8825,
      523,
      18,
      15534,
      779,
      13719,
      12467,
      6612,
      3346,
      17784,
      10562,
      1490,
      779,
      9519,
      53859,
      7330,
      523,
      19,
      16,
      879,
      535,
      46050,
      931,
      879,
      20,
      19,
      16,
      835,
      535,
      46050,
      931,
      835,
      20,
      19,
      16,
      28441,
      535,
      42749,
      13719,
      14350,
      779,
      21968,
      4589,
      20,
      21
     ],
     "markers": [
      64,
      72,
      80
     ],
     "logits_raw": [
      0.6088454723358154,
      2.894845724105835,
      1.3413251638412476
     ],
     "probs_raw": [
      0.07742483913898468,
      0.7615134716033936,
      0.16106167435646057
     ],
     "probs": [
      0.12809723615646362,
      0.6557409167289734,
      0.21616187691688538
     ],
     "argmax_index": 1,
     "argmax": "A",
     "top2_margin": 0.439579039812088,
     "near_tie": false,
     "gold": "A",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "A",
      "confidence": 0.6557409167289734,
      "probabilities": {
       "B": 0.12809723615646362,
       "A": 0.6557409167289734,
       "insufficient": 0.21616187691688538
      }
     }
    },
    "usage": {
     "input_tokens": 91,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "semif_ffee861474a9665a093a",
   "mode": "text",
   "source": "semif_authored144",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 77,
   "questions": [
    {
     "qid": "decision",
     "type": "choice",
     "K": 3,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 77,
     "positions": 77,
     "bucket": 256,
     "ids": [
      1,
      17,
      1098,
      3434,
      919,
      14350,
      768,
      2154,
      1203,
      1295,
      2681,
      933,
      1336,
      6249,
      523,
      18,
      542,
      12518,
      1090,
      2230,
      6835,
      4624,
      797,
      779,
      20610,
      810,
      1178,
      1295,
      21701,
      27434,
      523,
      3354,
      936,
      874,
      12800,
      540,
      19,
      16,
      33776,
      535,
      941,
      9519,
      7330,
      14480,
      1512,
      779,
      4272,
      20,
      19,
      16,
      20458,
      535,
      941,
      9519,
      7330,
      22558,
      779,
      4272,
      20,
      19,
      16,
      28441,
      535,
      941,
      21706,
      2203,
      2177,
      1014,
      27169,
      3984,
      779,
      7330,
      22558,
      779,
      4272,
      20,
      21
     ],
     "markers": [
      37,
      49,
      60
     ],
     "logits_raw": [
      0.34799861907958984,
      4.5191450119018555,
      2.7875254154205322
     ],
     "probs_raw": [
      0.012943761423230171,
      0.8386222124099731,
      0.14843404293060303
     ],
     "probs": [
      0.03788952901959419,
      0.745668888092041,
      0.2164415717124939
     ],
     "argmax_index": 1,
     "argmax": "permitted",
     "top2_margin": 0.5292273163795471,
     "near_tie": false,
     "gold": "insufficient",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "decision": {
      "type": "choice",
      "choice": "permitted",
      "confidence": 0.745668888092041,
      "probabilities": {
       "prohibited": 0.03788952901959419,
       "permitted": 0.745668888092041,
       "insufficient": 0.2164415717124939
      }
     }
    },
    "usage": {
     "input_tokens": 77,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_000",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 103,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 103,
     "positions": 103,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      53983,
      3609,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      2172,
      1014,
      25764,
      906,
      768,
      13974,
      803,
      779,
      28857,
      7169,
      797,
      779,
      730,
      898,
      881,
      3935,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      941,
      10557,
      7169,
      26807,
      988,
      1994,
      768,
      3674,
      811,
      768,
      3134,
      10282,
      25624,
      523,
      20,
      19,
      16,
      794,
      535,
      1592,
      3761,
      797,
      779,
      28857,
      7169,
      523,
      20,
      19,
      16,
      781,
      535,
      835,
      10636,
      797,
      779,
      10557,
      7169,
      523,
      20,
      19,
      16,
      778,
      535,
      835,
      4320,
      797,
      779,
      2080,
      810,
      5681,
      803,
      14773,
      39451,
      523,
      20,
      21
     ],
     "markers": [
      44,
      63,
      75,
      87
     ],
     "logits_raw": [
      -5.553777694702148,
      -7.747239112854004,
      -5.589899063110352,
      -5.4578471183776855
     ],
     "probs_raw": [
      0.3147885501384735,
      0.035108376294374466,
      0.3036208748817444,
      0.3464823067188263
     ],
     "probs": [
      0.3072996139526367,
      0.06413247436285019,
      0.29947182536125183,
      0.32909613847732544
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.02179652452468872,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.32909613847732544,
      "probabilities": {
       "a": 0.3072996139526367,
       "b": 0.06413247436285019,
       "c": 0.29947182536125183,
       "d": 0.32909613847732544
      }
     }
    },
    "usage": {
     "input_tokens": 103,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_001",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 74,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 74,
     "positions": 74,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      50988,
      4331,
      1878,
      997,
      39420,
      6881,
      997,
      3493,
      856,
      6365,
      2807,
      4791,
      14232,
      4027,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      42144,
      3685,
      803,
      35231,
      6104,
      523,
      20,
      19,
      16,
      794,
      535,
      15858,
      1227,
      41134,
      852,
      523,
      20,
      19,
      16,
      781,
      535,
      5148,
      44771,
      3685,
      803,
      35231,
      6104,
      523,
      20,
      19,
      16,
      778,
      535,
      63401,
      17712,
      30769,
      10254,
      523,
      20,
      21
     ],
     "markers": [
      31,
      42,
      52,
      64
     ],
     "logits_raw": [
      0.6107369661331177,
      -4.443951606750488,
      2.522935152053833,
      -6.117058753967285
     ],
     "probs_raw": [
      0.12860864400863647,
      0.0008204402402043343,
      0.8704168796539307,
      0.00015396632079500705
     ],
     "probs": [
      0.2018294632434845,
      0.0054557411931455135,
      0.7910635471343994,
      0.0016512047732248902
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.5892340838909149,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.7910635471343994,
      "probabilities": {
       "a": 0.2018294632434845,
       "b": 0.0054557411931455135,
       "c": 0.7910635471343994,
       "d": 0.0016512047732248902
      }
     }
    },
    "usage": {
     "input_tokens": 74,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_002",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 107,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 107,
     "positions": 107,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      4167,
      578,
      2499,
      1504,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      856,
      5405,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1030,
      779,
      1787,
      2685,
      521,
      1113,
      23032,
      938,
      9452,
      523,
      20,
      19,
      16,
      794,
      535,
      1030,
      779,
      3290,
      2685,
      521,
      1113,
      23032,
      938,
      9452,
      523,
      20,
      19,
      16,
      781,
      535,
      1030,
      779,
      1787,
      2685,
      521,
      7018,
      856,
      1014,
      3222,
      811,
      8529,
      7408,
      811,
      3807,
      797,
      6154,
      523,
      20,
      19,
      16,
      778,
      535,
      1030,
      779,
      3290,
      2685,
      521,
      7018,
      856,
      3222,
      811,
      8529,
      7408,
      811,
      3807,
      797,
      6154,
      523,
      20,
      21
     ],
     "markers": [
      34,
      49,
      64,
      86
     ],
     "logits_raw": [
      -3.5198915004730225,
      -1.3380160331726074,
      -1.1915581226348877,
      -0.26958832144737244
     ],
     "probs_raw": [
      0.021776100620627403,
      0.1929997056722641,
      0.22344081103801727,
      0.5617833733558655
     ],
     "probs": [
      0.04711969569325447,
      0.2239195704460144,
      0.24861547350883484,
      0.4803452491760254
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.23172977566719055,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.4803452491760254,
      "probabilities": {
       "a": 0.04711969569325447,
       "b": 0.2239195704460144,
       "c": 0.24861547350883484,
       "d": 0.4803452491760254
      }
     }
    },
    "usage": {
     "input_tokens": 107,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_003",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      28009,
      19179,
      1878,
      997,
      39420,
      6881,
      997,
      5711,
      4092,
      896,
      6993,
      7239,
      1914,
      1569,
      3476,
      803,
      62151,
      856,
      896,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      6405,
      62311,
      834,
      1817,
      1277,
      963,
      768,
      12280,
      3209,
      20,
      19,
      16,
      794,
      535,
      2216,
      1112,
      1014,
      3896,
      1842,
      1620,
      856,
      3529,
      884,
      20,
      19,
      16,
      781,
      535,
      1470,
      1400,
      874,
      768,
      15513,
      1723,
      875,
      22344,
      20,
      19,
      16,
      778,
      535,
      6504,
      7485,
      1052,
      2920,
      28195,
      20,
      21
     ],
     "markers": [
      36,
      50,
      64,
      77
     ],
     "logits_raw": [
      -2.4365646839141846,
      -1.0311179161071777,
      -3.46833872795105,
      -1.6626391410827637
     ],
     "probs_raw": [
      0.13154460489749908,
      0.5363531708717346,
      0.046879105269908905,
      0.2852230668067932
     ],
     "probs": [
      0.1681860387325287,
      0.4589926302433014,
      0.08048214763402939,
      0.2923392355442047
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.16665339469909668,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.4589926302433014,
      "probabilities": {
       "a": 0.1681860387325287,
       "b": 0.4589926302433014,
       "c": 0.08048214763402939,
       "d": 0.2923392355442047
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_004",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      28009,
      19179,
      1878,
      997,
      39420,
      6881,
      997,
      544,
      22525,
      20800,
      938,
      1519,
      1316,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      43391,
      5747,
      2076,
      779,
      1739,
      24452,
      20,
      19,
      16,
      794,
      535,
      12220,
      2731,
      3209,
      810,
      6993,
      7239,
      20,
      19,
      16,
      781,
      535,
      9195,
      730,
      2096,
      933,
      1209,
      1779,
      2731,
      20,
      19,
      16,
      778,
      535,
      14566,
      1014,
      4763,
      8107,
      2731,
      3209,
      20,
      21
     ],
     "markers": [
      30,
      41,
      52,
      64
     ],
     "logits_raw": [
      -1.9950649738311768,
      -0.7058923244476318,
      4.306713104248047,
      0.4114217162132263
     ],
     "probs_raw": [
      0.0017816874897107482,
      0.006467134226113558,
      0.9719834327697754,
      0.01976766437292099
     ],
     "probs": [
      0.010075385682284832,
      0.025304943323135376,
      0.9084072113037109,
      0.05621250718832016
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.8521947041153908,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 1.1920928955078125e-07,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.9084072113037109,
      "probabilities": {
       "a": 0.010075385682284832,
       "b": 0.025304943323135376,
       "c": 0.9084072113037109,
       "d": 0.05621250718832016
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_005",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      5606,
      30049,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      22152,
      5939,
      1178,
      15161,
      28327,
      9251,
      55053,
      968,
      1144,
      803,
      779,
      2773,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      41305,
      29349,
      947,
      957,
      20467,
      730,
      533,
      7695,
      20,
      19,
      16,
      794,
      535,
      5001,
      3476,
      803,
      58190,
      20,
      19,
      16,
      781,
      535,
      40494,
      24612,
      14275,
      810,
      18357,
      988,
      2630,
      4492,
      768,
      5511,
      61129,
      37185,
      20,
      19,
      16,
      778,
      535,
      5001,
      63011,
      13415,
      33955,
      1146,
      19288,
      20,
      21
     ],
     "markers": [
      37,
      50,
      59,
      76
     ],
     "logits_raw": [
      -0.5547366142272949,
      0.12310024350881577,
      -2.19450044631958,
      -2.8999176025390625
     ],
     "probs_raw": [
      0.3067985475063324,
      0.6042742133140564,
      0.05952684208750725,
      0.029400456696748734
     ],
     "probs": [
      0.3205031454563141,
      0.520139217376709,
      0.09933988004922867,
      0.06001778692007065
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.1996360719203949,
     "near_tie": false,
     "gold": "b",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.520139217376709,
      "probabilities": {
       "a": 0.3205031454563141,
       "b": 0.520139217376709,
       "c": 0.09933988004922867,
       "d": 0.06001778692007065
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_006",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 243,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 243,
     "positions": 243,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      9152,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      730,
      5489,
      6194,
      10835,
      6394,
      4834,
      811,
      779,
      16329,
      1929,
      803,
      902,
      730,
      533,
      33344,
      3129,
      803,
      13467,
      20738,
      521,
      19236,
      521,
      810,
      8550,
      17828,
      523,
      7074,
      1033,
      2926,
      521,
      1689,
      1236,
      1178,
      1320,
      6448,
      59335,
      774,
      521,
      13467,
      57554,
      521,
      810,
      1178,
      1381,
      9730,
      834,
      27252,
      963,
      1336,
      875,
      34585,
      5019,
      523,
      7438,
      779,
      3650,
      730,
      527,
      6272,
      521,
      1689,
      1178,
      1320,
      2204,
      19941,
      803,
      54670,
      19877,
      810,
      3290,
      2122,
      803,
      8311,
      896,
      1052,
      44405,
      791,
      1450,
      988,
      5392,
      523,
      3365,
      24037,
      856,
      730,
      2346,
      40619,
      521,
      810,
      3186,
      4809,
      856,
      730,
      7984,
      524,
      2475,
      8061,
      904,
      580,
      523,
      20737,
      14435,
      4427,
      1295,
      32219,
      523,
      25650,
      5742,
      14435,
      4427,
      768,
      35459,
      15365,
      810,
      1737,
      34270,
      4531,
      523,
      2799,
      3498,
      896,
      1689,
      856,
      1295,
      5039,
      9368,
      797,
      4637,
      896,
      1689,
      1593,
      811,
      6401,
      523,
      2799,
      1178,
      60532,
      2542,
      852,
      2630,
      768,
      2984,
      523,
      3365,
      51879,
      9403,
      856,
      730,
      1632,
      836,
      51275,
      521,
      810,
      18941,
      7097,
      1215,
      769,
      9403,
      856,
      730,
      9300,
      21955,
      30147,
      523,
      20463,
      803,
      779,
      2773,
      856,
      779,
      1529,
      6994,
      4764,
      3801,
      797,
      3157,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      889,
      1174,
      14191,
      814,
      6738,
      20,
      19,
      16,
      794,
      535,
      12656,
      14641,
      51112,
      6738,
      20,
      19,
      16,
      781,
      535,
      918,
      1335,
      2086,
      13001,
      17292,
      8886,
      6738,
      20,
      19,
      16,
      778,
      535,
      2334,
      1974,
      813,
      976,
      6738,
      20,
      21
     ],
     "markers": [
      202,
      212,
      221,
      233
     ],
     "logits_raw": [
      -1.3662583827972412,
      -2.0687501430511475,
      -1.8015267848968506,
      -0.1909313052892685
     ],
     "probs_raw": [
      0.18581685423851013,
      0.09204428642988205,
      0.12024041265249252,
      0.6018983721733093
     ],
     "probs": [
      0.21488778293132782,
      0.13009950518608093,
      0.1574624925851822,
      0.4975501596927643
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.28266237676143646,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.4975501596927643,
      "probabilities": {
       "a": 0.21488778293132782,
       "b": 0.13009950518608093,
       "c": 0.1574624925851822,
       "d": 0.4975501596927643
      }
     }
    },
    "usage": {
     "input_tokens": 243,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_007",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 92,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 92,
     "positions": 92,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      780,
      776,
      5797,
      1878,
      997,
      39420,
      6881,
      997,
      61490,
      591,
      2115,
      856,
      779,
      1996,
      803,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      5287,
      57918,
      7401,
      810,
      7809,
      1720,
      11192,
      2115,
      523,
      20,
      19,
      16,
      794,
      535,
      54375,
      11150,
      20779,
      30667,
      7401,
      810,
      7809,
      2076,
      11192,
      2115,
      523,
      20,
      19,
      16,
      781,
      535,
      5287,
      57918,
      7401,
      810,
      7809,
      2076,
      11192,
      2115,
      523,
      20,
      19,
      16,
      778,
      535,
      54375,
      11150,
      20779,
      30667,
      7401,
      810,
      7809,
      1720,
      11192,
      2115,
      523,
      20,
      21
     ],
     "markers": [
      32,
      46,
      62,
      76
     ],
     "logits_raw": [
      -1.0777621269226074,
      0.14282402396202087,
      -1.3859832286834717,
      -1.6446017026901245
     ],
     "probs_raw": [
      0.17570854723453522,
      0.5955068469047546,
      0.12910236418247223,
      0.09968220442533493
     ],
     "probs": [
      0.20572371780872345,
      0.4919832646846771,
      0.16506841778755188,
      0.13722461462020874
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.2862595468759537,
     "near_tie": false,
     "gold": "b",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.4919832646846771,
      "probabilities": {
       "a": 0.20572371780872345,
       "b": 0.4919832646846771,
       "c": 0.16506841778755188,
       "d": 0.13722461462020874
      }
     }
    },
    "usage": {
     "input_tokens": 92,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_008",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 106,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 106,
     "positions": 106,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      16290,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      6306,
      856,
      7253,
      875,
      12467,
      3658,
      779,
      1883,
      12709,
      797,
      768,
      12825,
      7322,
      803,
      1883,
      797,
      3053,
      2779,
      586,
      13761,
      803,
      768,
      3226,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      779,
      3636,
      49121,
      5401,
      803,
      1883,
      20,
      19,
      16,
      794,
      535,
      779,
      3636,
      44652,
      801,
      803,
      5401,
      803,
      1883,
      20,
      19,
      16,
      781,
      535,
      779,
      1569,
      5480,
      7614,
      803,
      1883,
      20,
      19,
      16,
      778,
      535,
      779,
      3636,
      44652,
      801,
      5401,
      803,
      59697,
      797,
      779,
      3053,
      2779,
      586,
      2009,
      7680,
      20,
      21
     ],
     "markers": [
      52,
      63,
      76,
      87
     ],
     "logits_raw": [
      -2.249392509460449,
      0.9455831050872803,
      -7.214566707611084,
      1.1811388731002808
     ],
     "probs_raw": [
      0.017758958041667938,
      0.4334886968135834,
      0.00012389960465952754,
      0.5486285090446472
     ],
     "probs": [
      0.04459759220480919,
      0.4370166063308716,
      0.0012851412175223231,
      0.5171006321907043
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.08008402585983276,
     "near_tie": false,
     "gold": "b",
     "correct": false,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5171006321907043,
      "probabilities": {
       "a": 0.04459759220480919,
       "b": 0.4370166063308716,
       "c": 0.0012851412175223231,
       "d": 0.5171006321907043
      }
     }
    },
    "usage": {
     "input_tokens": 106,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_009",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 339,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 339,
     "positions": 339,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      1081,
      3129,
      1878,
      997,
      39420,
      6881,
      997,
      2443,
      3952,
      11335,
      811,
      779,
      2773,
      2203,
      13909,
      587,
      18588,
      550,
      48949,
      1014,
      768,
      3223,
      521,
      2084,
      17423,
      1665,
      521,
      896,
      2908,
      779,
      1509,
      12037,
      803,
      4367,
      843,
      7895,
      856,
      2166,
      3522,
      2759,
      8619,
      810,
      3676,
      1779,
      1109,
      1720,
      1101,
      2507,
      15808,
      3279,
      810,
      3226,
      1327,
      803,
      779,
      59362,
      2892,
      810,
      24408,
      1413,
      521,
      896,
      1023,
      803,
      5690,
      1738,
      3467,
      1052,
      779,
      22148,
      811,
      2021,
      5572,
      3789,
      800,
      797,
      1559,
      38501,
      810,
      48079,
      6454,
      906,
      938,
      3191,
      906,
      4763,
      946,
      53363,
      24597,
      803,
      1377,
      523,
      2439,
      1701,
      1701,
      1701,
      859,
      64096,
      2507,
      6905,
      896,
      779,
      1394,
      2396,
      3314,
      881,
      810,
      2166,
      856,
      896,
      1023,
      803,
      5690,
      1400,
      5611,
      810,
      1192,
      62975,
      1607,
      3015,
      1082,
      779,
      26695,
      570,
      843,
      888,
      1023,
      1188,
      13690,
      518,
      2059,
      916,
      779,
      59362,
      1008,
      810,
      779,
      24408,
      797,
      1192,
      803,
      4367,
      810,
      1312,
      7519,
      906,
      4763,
      1044,
      3762,
      18559,
      13909,
      587,
      18588,
      4483,
      27104,
      1288,
      1413,
      1166,
      797,
      1081,
      896,
      12856,
      811,
      10268,
      779,
      10669,
      803,
      1494,
      3374,
      1144,
      23089,
      811,
      874,
      797,
      1973,
      2184,
      1177,
      521,
      1023,
      1499,
      1014,
      1113,
      1033,
      2158,
      1052,
      1607,
      14908,
      3800,
      570,
      779,
      12762,
      800,
      803,
      1518,
      13967,
      1144,
      803,
      27569,
      810,
      2084,
      1514,
      1007,
      2344,
      30171,
      1081,
      521,
      906,
      968,
      779,
      2006,
      36159,
      896,
      1861,
      6283,
      4650,
      1529,
      15676,
      876,
      13909,
      511,
      569,
      587,
      1424,
      35247,
      53673,
      2417,
      3058,
      521,
      889,
      1449,
      33262,
      1798,
      36116,
      9874,
      779,
      31196,
      803,
      4367,
      810,
      779,
      15602,
      2100,
      583,
      14692,
      521,
      730,
      11909,
      527,
      13699,
      4486,
      2773,
      779,
      6157,
      803,
      7793,
      53673,
      2417,
      3058,
      521,
      5690,
      953,
      7962,
      3222,
      811,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      8323,
      779,
      5271,
      810,
      24408,
      12708,
      1419,
      52096,
      523,
      20,
      19,
      16,
      794,
      535,
      45389,
      2611,
      4863,
      803,
      6202,
      797,
      779,
      45034,
      3935,
      523,
      20,
      19,
      16,
      781,
      535,
      6122,
      2993,
      19087,
      3256,
      779,
      13874,
      53236,
      803,
      3859,
      4367,
      523,
      20,
      19,
      16,
      778,
      535,
      7730,
      779,
      14127,
      6547,
      22892,
      523,
      20,
      21
     ],
     "markers": [
      283,
      297,
      312,
      328
     ],
     "logits_raw": [
      -4.4359869956970215,
      -3.294753313064575,
      1.9535304307937622,
      -4.301299571990967
     ],
     "probs_raw": [
      0.0016643258277326822,
      0.005210384260863066,
      0.9912209510803223,
      0.0019042867934331298
     ],
     "probs": [
      0.009964779019355774,
      0.02251727692782879,
      0.9565467834472656,
      0.010971151292324066
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.9340295065194368,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.9565467834472656,
      "probabilities": {
       "a": 0.009964779019355774,
       "b": 0.02251727692782879,
       "c": 0.9565467834472656,
       "d": 0.010971151292324066
      }
     }
    },
    "usage": {
     "input_tokens": 339,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_010",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 84,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 84,
     "positions": 84,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      586,
      3762,
      965,
      12993,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      28763,
      6142,
      768,
      2129,
      598,
      5508,
      2160,
      34918,
      6557,
      797,
      779,
      12681,
      1452,
      31915,
      43084,
      3504,
      535,
      941,
      923,
      62401,
      803,
      19821,
      516,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      12747,
      920,
      582,
      6998,
      3750,
      20,
      19,
      16,
      794,
      535,
      34483,
      31191,
      20,
      19,
      16,
      781,
      535,
      984,
      1380,
      882,
      1491,
      2774,
      20,
      19,
      16,
      778,
      535,
      30360,
      20227,
      20,
      21
     ],
     "markers": [
      50,
      60,
      67,
      77
     ],
     "logits_raw": [
      -1.631797432899475,
      -2.068960428237915,
      -1.538092017173767,
      -1.4344347715377808
     ],
     "probs_raw": [
      0.2523794174194336,
      0.1630033254623413,
      0.2771722078323364,
      0.3074450194835663
     ],
     "probs": [
      0.25301024317741394,
      0.18514660000801086,
      0.2705257534980774,
      0.2913174331188202
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.020791679620742798,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.2913174331188202,
      "probabilities": {
       "a": 0.25301024317741394,
       "b": 0.18514660000801086,
       "c": 0.2705257534980774,
       "d": 0.2913174331188202
      }
     }
    },
    "usage": {
     "input_tokens": 84,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_011",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 125,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 125,
     "positions": 125,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      2293,
      3810,
      5354,
      4575,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      15450,
      1337,
      17924,
      13236,
      12825,
      4167,
      2783,
      20312,
      856,
      843,
      1072,
      518,
      3347,
      540,
      569,
      587,
      550,
      523,
      38679,
      875,
      17924,
      61359,
      3675,
      1209,
      1394,
      7441,
      1399,
      2025,
      875,
      12825,
      61359,
      13909,
      587,
      2667,
      523,
      31498,
      11236,
      4167,
      769,
      41913,
      938,
      946,
      10305,
      13033,
      13909,
      587,
      7986,
      523,
      31498,
      11236,
      4167,
      769,
      41913,
      3976,
      1052,
      768,
      3437,
      1782,
      3955,
      810,
      5321,
      2374,
      905,
      2460,
      21321,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      859,
      1550,
      20,
      19,
      16,
      794,
      535,
      3504,
      1550,
      20,
      19,
      16,
      781,
      535,
      6572,
      1550,
      20,
      19,
      16,
      778,
      535,
      3504,
      810,
      6572,
      1550,
      20,
      21
     ],
     "markers": [
      95,
      102,
      109,
      116
     ],
     "logits_raw": [
      -3.118961811065674,
      -3.313667058944702,
      -3.6159772872924805,
      -1.2328732013702393
     ],
     "probs_raw": [
      0.11080395430326462,
      0.09120020270347595,
      0.06740687042474747,
      0.730588972568512
     ],
     "probs": [
      0.15580348670482635,
      0.13557299971580505,
      0.10924122482538223,
      0.5993822813034058
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.4435787945985794,
     "near_tie": false,
     "gold": "b",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5993822813034058,
      "probabilities": {
       "a": 0.15580348670482635,
       "b": 0.13557299971580505,
       "c": 0.10924122482538223,
       "d": 0.5993822813034058
      }
     }
    },
    "usage": {
     "input_tokens": 125,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_012",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 97,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 97,
     "positions": 97,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      4167,
      578,
      2499,
      1504,
      1878,
      997,
      39420,
      6881,
      997,
      6626,
      31243,
      5068,
      768,
      9920,
      797,
      1450,
      8292,
      521,
      1689,
      4302,
      28204,
      1209,
      7302,
      16830,
      55324,
      810,
      13962,
      1944,
      592,
      4245,
      523,
      1890,
      31243,
      521,
      1944,
      592,
      4245,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      58969,
      2718,
      779,
      3036,
      803,
      6154,
      20,
      19,
      16,
      794,
      535,
      938,
      768,
      7411,
      2546,
      20,
      19,
      16,
      781,
      535,
      938,
      768,
      35281,
      2184,
      916,
      7302,
      16830,
      20,
      19,
      16,
      778,
      535,
      938,
      902,
      20451,
      2184,
      20,
      21
     ],
     "markers": [
      56,
      67,
      76,
      88
     ],
     "logits_raw": [
      1.5743370056152344,
      -2.8246326446533203,
      -0.30558764934539795,
      -1.4633177518844604
     ],
     "probs_raw": [
      0.8245118856430054,
      0.010133247822523117,
      0.12582182884216309,
      0.03953303396701813
     ],
     "probs": [
      0.70499187707901,
      0.030442405492067337,
      0.18406431376934052,
      0.08050142228603363
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.5209275633096695,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.70499187707901,
      "probabilities": {
       "a": 0.70499187707901,
       "b": 0.030442405492067337,
       "c": 0.18406431376934052,
       "d": 0.08050142228603363
      }
     }
    },
    "usage": {
     "input_tokens": 97,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_013",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 170,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 170,
     "positions": 170,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      51584,
      6368,
      1878,
      997,
      39420,
      6881,
      997,
      1286,
      2642,
      900,
      2774,
      810,
      39279,
      581,
      1172,
      4066,
      884,
      3392,
      10598,
      6180,
      10725,
      521,
      3921,
      2177,
      1390,
      7075,
      3618,
      779,
      22515,
      4348,
      988,
      3822,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      2342,
      7075,
      22792,
      779,
      9397,
      988,
      1993,
      779,
      11811,
      561,
      25900,
      521,
      1144,
      856,
      5175,
      811,
      779,
      22515,
      4348,
      523,
      20,
      19,
      16,
      794,
      535,
      2342,
      7075,
      3807,
      779,
      784,
      810,
      9117,
      1002,
      1860,
      896,
      938,
      1593,
      521,
      1101,
      902,
      20639,
      1126,
      4584,
      2111,
      779,
      9397,
      1172,
      3347,
      784,
      810,
      9117,
      4396,
      20,
      19,
      16,
      781,
      535,
      2342,
      7075,
      7226,
      6229,
      779,
      781,
      15479,
      1885,
      1994,
      1908,
      36446,
      521,
      5430,
      22813,
      2844,
      779,
      14008,
      1685,
      902,
      20639,
      1126,
      1172,
      6136,
      810,
      779,
      22515,
      6825,
      20,
      19,
      16,
      778,
      535,
      2342,
      7075,
      16038,
      768,
      7226,
      3685,
      803,
      1394,
      811,
      779,
      1908,
      25704,
      2888,
      811,
      779,
      37708,
      810,
      9707,
      968,
      779,
      1390,
      7075,
      7226,
      2536,
      877,
      521,
      1144,
      20,
      21
     ],
     "markers": [
      49,
      74,
      106,
      138
     ],
     "logits_raw": [
      -0.35539257526397705,
      0.6858941316604614,
      -0.02686423808336258,
      0.8678652048110962
     ],
     "probs_raw": [
      0.11600898206233978,
      0.32863715291023254,
      0.1611274927854538,
      0.39422640204429626
     ],
     "probs": [
      0.14782999455928802,
      0.31103119254112244,
      0.18693166971206665,
      0.35420718789100647
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.04317599534988403,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.35420718789100647,
      "probabilities": {
       "a": 0.14782999455928802,
       "b": 0.31103119254112244,
       "c": 0.18693166971206665,
       "d": 0.35420718789100647
      }
     }
    },
    "usage": {
     "input_tokens": 170,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_014",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 98,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 98,
     "positions": 98,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      8496,
      21671,
      2499,
      1504,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      4744,
      1178,
      1295,
      1862,
      884,
      27565,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      2297,
      7909,
      768,
      730,
      2081,
      532,
      5204,
      14785,
      988,
      768,
      6709,
      523,
      20,
      19,
      16,
      794,
      535,
      941,
      5455,
      803,
      34736,
      4228,
      3809,
      768,
      6671,
      523,
      20,
      19,
      16,
      781,
      535,
      6066,
      6190,
      1825,
      768,
      4072,
      6073,
      896,
      5535,
      1901,
      12374,
      1523,
      16711,
      523,
      20,
      19,
      16,
      778,
      535,
      835,
      8117,
      1675,
      1264,
      13505,
      523,
      20,
      21
     ],
     "markers": [
      38,
      55,
      69,
      87
     ],
     "logits_raw": [
      -1.4280264377593994,
      -4.1390509605407715,
      -3.740838050842285,
      -2.7201414108276367
     ],
     "probs_raw": [
      0.6943767070770264,
      0.04615429788827896,
      0.06873118132352829,
      0.190737783908844
     ],
     "probs": [
      0.5769809484481812,
      0.08319791406393051,
      0.11057352274656296,
      0.22924762964248657
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.3477333188056946,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.5769809484481812,
      "probabilities": {
       "a": 0.5769809484481812,
       "b": 0.08319791406393051,
       "c": 0.11057352274656296,
       "d": 0.22924762964248657
      }
     }
    },
    "usage": {
     "input_tokens": 98,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_015",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 191,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 191,
     "positions": 191,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      3036,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      8117,
      5006,
      953,
      12260,
      2681,
      988,
      768,
      6999,
      523,
      1405,
      1320,
      1381,
      11249,
      8693,
      963,
      779,
      6999,
      810,
      953,
      1943,
      57778,
      1048,
      523,
      15504,
      8167,
      12900,
      13525,
      779,
      5006,
      810,
      38995,
      1889,
      523,
      3145,
      950,
      17436,
      811,
      5642,
      1523,
      803,
      1149,
      4639,
      521,
      779,
      8167,
      3151,
      779,
      5006,
      797,
      779,
      1892,
      12799,
      803,
      768,
      8167,
      1681,
      523,
      941,
      8167,
      3191,
      779,
      5006,
      797,
      779,
      1892,
      12799,
      803,
      779,
      8167,
      1681,
      875,
      8174,
      1235,
      8831,
      521,
      810,
      1757,
      5789,
      1889,
      523,
      2182,
      779,
      5006,
      60153,
      768,
      4673,
      2723,
      779,
      8167,
      875,
      4831,
      46383,
      521,
      950,
      1188,
      1529,
      3849,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      64155,
      521,
      10967,
      779,
      8167,
      2047,
      768,
      6612,
      18626,
      523,
      20,
      19,
      16,
      794,
      535,
      64155,
      521,
      1288,
      950,
      1348,
      12034,
      811,
      779,
      53600,
      1929,
      950,
      953,
      57778,
      1048,
      523,
      20,
      19,
      16,
      781,
      535,
      1014,
      64155,
      521,
      1929,
      8167,
      4584,
      874,
      4908,
      50359,
      875,
      4831,
      46383,
      523,
      20,
      19,
      16,
      778,
      535,
      1014,
      64155,
      521,
      1929,
      779,
      5006,
      14759,
      1295,
      5808,
      523,
      20,
      21
     ],
     "markers": [
      123,
      138,
      158,
      176
     ],
     "logits_raw": [
      0.18301290273666382,
      -1.1472127437591553,
      0.13043098151683807,
      -0.8381863832473755
     ],
     "probs_raw": [
      0.3885974586009979,
      0.10275200009346008,
      0.36869218945503235,
      0.13995833694934845
     ],
     "probs": [
      0.3531140089035034,
      0.13653221726417542,
      0.3400966227054596,
      0.17025713622570038
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.013017386198043823,
     "near_tie": true,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.3531140089035034,
      "probabilities": {
       "a": 0.3531140089035034,
       "b": 0.13653221726417542,
       "c": 0.3400966227054596,
       "d": 0.17025713622570038
      }
     }
    },
    "usage": {
     "input_tokens": 191,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_016",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 93,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 93,
     "positions": 93,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      54717,
      61895,
      1088,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      32630,
      4174,
      856,
      779,
      1529,
      45796,
      916,
      768,
      7243,
      2193,
      893,
      44352,
      803,
      22355,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      27619,
      14289,
      27952,
      6368,
      523,
      20,
      19,
      16,
      794,
      535,
      27619,
      14289,
      856,
      768,
      2780,
      40380,
      2511,
      968,
      1144,
      811,
      3618,
      933,
      4859,
      12383,
      523,
      20,
      19,
      16,
      781,
      535,
      27619,
      14289,
      27952,
      4297,
      6155,
      523,
      20,
      19,
      16,
      778,
      535,
      27619,
      14289,
      1753,
      6812,
      779,
      12383,
      523,
      20,
      21
     ],
     "markers": [
      40,
      50,
      70,
      81
     ],
     "logits_raw": [
      -3.956939458847046,
      -2.1060941219329834,
      -4.5931596755981445,
      -2.959867238998413
     ],
     "probs_raw": [
      0.09429672360420227,
      0.6002172827720642,
      0.04991023987531662,
      0.25557568669319153
     ],
     "probs": [
      0.13468477129936218,
      0.5052562355995178,
      0.08549527823925018,
      0.274563729763031
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.23069250583648682,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.5052562355995178,
      "probabilities": {
       "a": 0.13468477129936218,
       "b": 0.5052562355995178,
       "c": 0.08549527823925018,
       "d": 0.274563729763031
      }
     }
    },
    "usage": {
     "input_tokens": 93,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_017",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      43046,
      1878,
      997,
      39420,
      6881,
      997,
      20581,
      1316,
      938,
      29286,
      1753,
      23606,
      587,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      29564,
      768,
      1847,
      5337,
      811,
      779,
      2280,
      803,
      779,
      2832,
      20,
      19,
      16,
      794,
      535,
      38084,
      779,
      2043,
      810,
      2337,
      1209,
      6886,
      20,
      19,
      16,
      781,
      535,
      49782,
      1149,
      3426,
      2915,
      13105,
      521,
      3761,
      1149,
      4099,
      2119,
      810,
      4859,
      1149,
      8660,
      1394,
      20,
      19,
      16,
      778,
      535,
      38084,
      1209,
      8123,
      810,
      11346,
      20,
      21
     ],
     "markers": [
      30,
      45,
      57,
      77
     ],
     "logits_raw": [
      -1.8940390348434448,
      -2.765936851501465,
      6.657193660736084,
      -0.15114456415176392
     ],
     "probs_raw": [
      0.00019304058514535427,
      8.072132186498493e-05,
      0.9986231327056885,
      0.0011030061868950725
     ],
     "probs": [
      0.002199452603235841,
      0.0011798404157161713,
      0.9889820218086243,
      0.007638671435415745
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.9813433503732085,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.9889820218086243,
      "probabilities": {
       "a": 0.002199452603235841,
       "b": 0.0011798404157161713,
       "c": 0.9889820218086243,
       "d": 0.007638671435415745
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_018",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 77,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 77,
     "positions": 77,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      11335,
      811,
      779,
      7691,
      803,
      3785,
      902,
      10077,
      6872,
      811,
      4859,
      19553,
      54924,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      5993,
      5134,
      1668,
      5654,
      803,
      17916,
      20,
      19,
      16,
      794,
      535,
      47836,
      25556,
      800,
      20,
      19,
      16,
      781,
      535,
      7996,
      1580,
      783,
      16454,
      20,
      19,
      16,
      778,
      535,
      4885,
      4377,
      20,
      21
     ],
     "markers": [
      42,
      53,
      61,
      70
     ],
     "logits_raw": [
      -0.7562735080718994,
      -1.111168622970581,
      1.7879376411437988,
      -3.8880667686462402
     ],
     "probs_raw": [
      0.0690699890255928,
      0.04843512922525406,
      0.8794806599617004,
      0.003014178015291691
     ],
     "probs": [
      0.12439634650945663,
      0.09654004871845245,
      0.7657828330993652,
      0.013280742801725864
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.6413864865899086,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.7657828330993652,
      "probabilities": {
       "a": 0.12439634650945663,
       "b": 0.09654004871845245,
       "c": 0.7657828330993652,
       "d": 0.013280742801725864
      }
     }
    },
    "usage": {
     "input_tokens": 77,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_019",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 145,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 145,
     "positions": 145,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      586,
      8988,
      21469,
      1878,
      997,
      39420,
      6881,
      997,
      3111,
      1144,
      803,
      1518,
      1547,
      21469,
      2177,
      779,
      2120,
      3338,
      843,
      18275,
      5582,
      859,
      524,
      1477,
      524,
      3462,
      518,
      1112,
      3503,
      8870,
      61611,
      7871,
      521,
      4262,
      811,
      17027,
      11241,
      8079,
      797,
      779,
      2641,
      906,
      803,
      730,
      1718,
      525,
      540,
      3108,
      31933,
      730,
      526,
      2171,
      859,
      953,
      19049,
      811,
      8498,
      896,
      1727,
      3801,
      31396,
      1320,
      1014,
      17502,
      1626,
      1210,
      7927,
      523,
      3108,
      31933,
      730,
      527,
      2171,
      859,
      2588,
      730,
      20397,
      810,
      4468,
      896,
      1727,
      9652,
      953,
      14209,
      37168,
      963,
      1450,
      2681,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      923,
      35736,
      521,
      923,
      35736,
      20,
      19,
      16,
      794,
      535,
      923,
      35736,
      521,
      4524,
      7871,
      20,
      19,
      16,
      781,
      535,
      4524,
      7871,
      521,
      923,
      35736,
      20,
      19,
      16,
      778,
      535,
      4524,
      7871,
      521,
      4524,
      7871,
      20,
      21
     ],
     "markers": [
      105,
      115,
      125,
      135
     ],
     "logits_raw": [
      -0.6472692489624023,
      -2.074370861053467,
      -1.1838890314102173,
      -1.2575846910476685
     ],
     "probs_raw": [
      0.4223143458366394,
      0.10135693848133087,
      0.24693623185157776,
      0.22939249873161316
     ],
     "probs": [
      0.3718819320201874,
      0.1341748833656311,
      0.2534708082675934,
      0.24047236144542694
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.118411123752594,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.3718819320201874,
      "probabilities": {
       "a": 0.3718819320201874,
       "b": 0.1341748833656311,
       "c": 0.2534708082675934,
       "d": 0.24047236144542694
      }
     }
    },
    "usage": {
     "input_tokens": 145,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_020",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 100,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 100,
     "positions": 100,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      18588,
      9467,
      811,
      904,
      13011,
      810,
      2342,
      20052,
      1008,
      1090,
      3237,
      1430,
      11167,
      2801,
      521,
      43175,
      18588,
      870,
      3071,
      4171,
      36020,
      18588,
      18588,
      856,
      768,
      1947,
      803,
      779,
      19485,
      1172,
      535,
      18588,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      5937,
      803,
      6450,
      2917,
      523,
      20,
      19,
      16,
      794,
      535,
      16798,
      523,
      20,
      19,
      16,
      781,
      535,
      5051,
      810,
      41519,
      811,
      11282,
      10912,
      523,
      20,
      19,
      16,
      778,
      535,
      15209,
      1337,
      779,
      5564,
      810,
      2536,
      803,
      1336,
      523,
      20,
      21
     ],
     "markers": [
      57,
      67,
      74,
      86
     ],
     "logits_raw": [
      -0.6914830803871155,
      0.4765990376472473,
      1.2798019647598267,
      -2.3174314498901367
     ],
     "probs_raw": [
      0.08626310527324677,
      0.27740657329559326,
      0.6193602681159973,
      0.016970114782452583
     ],
     "probs": [
      0.12978744506835938,
      0.2989580035209656,
      0.5306280255317688,
      0.040626559406518936
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.23167002201080322,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.5306280255317688,
      "probabilities": {
       "a": 0.12978744506835938,
       "b": 0.2989580035209656,
       "c": 0.5306280255317688,
       "d": 0.040626559406518936
      }
     }
    },
    "usage": {
     "input_tokens": 100,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_021",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 75,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 75,
     "positions": 75,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      50988,
      4331,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      16426,
      2258,
      1011,
      874,
      1593,
      875,
      17912,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      14201,
      23551,
      878,
      5077,
      523,
      20,
      19,
      16,
      794,
      535,
      1569,
      6660,
      878,
      5077,
      523,
      20,
      19,
      16,
      781,
      535,
      51192,
      4149,
      2363,
      779,
      878,
      4315,
      523,
      20,
      19,
      16,
      778,
      535,
      1485,
      11478,
      803,
      779,
      878,
      4315,
      523,
      20,
      21
     ],
     "markers": [
      31,
      41,
      51,
      63
     ],
     "logits_raw": [
      -1.173835039138794,
      -0.4603675603866577,
      -1.0826773643493652,
      -0.9287775754928589
     ],
     "probs_raw": [
      0.18469972908496857,
      0.37698256969451904,
      0.20232780277729034,
      0.23598989844322205
     ],
     "probs": [
      0.20311520993709564,
      0.33812984824180603,
      0.21678167581558228,
      0.24197322130203247
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.09615662693977356,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.33812984824180603,
      "probabilities": {
       "a": 0.20311520993709564,
       "b": 0.33812984824180603,
       "c": 0.21678167581558228,
       "d": 0.24197322130203247
      }
     }
    },
    "usage": {
     "input_tokens": 75,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_022",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 148,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 148,
     "positions": 148,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      17263,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      14060,
      2312,
      9494,
      779,
      2773,
      2203,
      1337,
      2204,
      24190,
      803,
      62523,
      523,
      32989,
      856,
      1058,
      2198,
      13264,
      514,
      513,
      1209,
      11932,
      1399,
      37436,
      810,
      1178,
      1058,
      1698,
      12185,
      514,
      513,
      2442,
      7460,
      1399,
      3735,
      867,
      523,
      3735,
      867,
      856,
      1058,
      1698,
      12185,
      514,
      513,
      2442,
      11932,
      1399,
      32989,
      810,
      1178,
      1058,
      2625,
      12185,
      3088,
      600,
      526,
      2945,
      528,
      602,
      13264,
      514,
      513,
      1209,
      7460,
      1399,
      37436,
      523,
      37436,
      7025,
      1763,
      569,
      513,
      526,
      523,
      1056,
      513,
      1094,
      6232,
      803,
      7460,
      523,
      3747,
      856,
      779,
      2080,
      803,
      37843,
      1094,
      6232,
      803,
      7460,
      803,
      3735,
      867,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      730,
      2475,
      20,
      19,
      16,
      794,
      535,
      730,
      6505,
      20,
      19,
      16,
      781,
      535,
      730,
      2198,
      20,
      19,
      16,
      778,
      535,
      730,
      16511,
      20,
      21
     ],
     "markers": [
      120,
      127,
      134,
      141
     ],
     "logits_raw": [
      -1.871951937675476,
      -2.1734390258789062,
      -2.9600579738616943,
      -2.754812240600586
     ],
     "probs_raw": [
      0.4015790820121765,
      0.2970550060272217,
      0.13527347147464752,
      0.1660923957824707
     ],
     "probs": [
      0.35737961530685425,
      0.28813666105270386,
      0.16427196562290192,
      0.19021180272102356
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.06924295425415039,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.35737961530685425,
      "probabilities": {
       "a": 0.35737961530685425,
       "b": 0.28813666105270386,
       "c": 0.16427196562290192,
       "d": 0.19021180272102356
      }
     }
    },
    "usage": {
     "input_tokens": 148,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_023",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 104,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 104,
     "positions": 104,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      586,
      8988,
      38419,
      1878,
      997,
      39420,
      6881,
      997,
      1030,
      59507,
      886,
      1090,
      3160,
      521,
      779,
      1509,
      810,
      43997,
      3801,
      7069,
      768,
      5516,
      803,
      1494,
      5962,
      3775,
      856,
      768,
      4484,
      12487,
      896,
      1494,
      13010,
      856,
      1014,
      10349,
      3077,
      933,
      1883,
      1203,
      4286,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1429,
      13985,
      2425,
      803,
      11244,
      523,
      20,
      19,
      16,
      794,
      535,
      902,
      6807,
      15372,
      1587,
      5158,
      3054,
      523,
      20,
      19,
      16,
      781,
      535,
      1429,
      29501,
      2425,
      803,
      11244,
      523,
      20,
      19,
      16,
      778,
      535,
      779,
      25369,
      803,
      1113,
      4624,
      3311,
      523,
      20,
      21
     ],
     "markers": [
      58,
      69,
      81,
      92
     ],
     "logits_raw": [
      -2.9687066078186035,
      0.7563834190368652,
      -3.7341668605804443,
      -6.6945295333862305
     ],
     "probs_raw": [
      0.023275204002857208,
      0.9653382897377014,
      0.01082576159387827,
      0.0005607756902463734
     ],
     "probs": [
      0.06266120076179504,
      0.8966939449310303,
      0.03626834228634834,
      0.004376488737761974
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.8340327441692352,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.8966939449310303,
      "probabilities": {
       "a": 0.06266120076179504,
       "b": 0.8966939449310303,
       "c": 0.03626834228634834,
       "d": 0.004376488737761974
      }
     }
    },
    "usage": {
     "input_tokens": 104,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_024",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 119,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 119,
     "positions": 119,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      53983,
      3609,
      1878,
      997,
      39420,
      6881,
      997,
      3493,
      856,
      779,
      2536,
      803,
      768,
      2375,
      4734,
      1785,
      1786,
      893,
      28714,
      797,
      6368,
      3609,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      2989,
      11787,
      1081,
      811,
      2846,
      779,
      1336,
      1327,
      803,
      4665,
      3910,
      5720,
      592,
      3191,
      1230,
      803,
      6257,
      2006,
      36159,
      884,
      6368,
      523,
      20,
      19,
      16,
      794,
      535,
      2989,
      14057,
      779,
      4077,
      803,
      2410,
      810,
      1352,
      6079,
      521,
      797,
      5574,
      11336,
      1399,
      6257,
      3606,
      523,
      20,
      19,
      16,
      781,
      535,
      2989,
      8759,
      779,
      4588,
      896,
      10338,
      14773,
      810,
      4109,
      938,
      779,
      5592,
      9715,
      18572,
      6368,
      4008,
      523,
      20,
      19,
      16,
      778,
      535,
      2595,
      803,
      1518,
      523,
      20,
      21
     ],
     "markers": [
      39,
      66,
      88,
      110
     ],
     "logits_raw": [
      1.9829537868499756,
      1.969934344291687,
      -0.5511482954025269,
      3.171776294708252
     ],
     "probs_raw": [
      0.18692950904369354,
      0.18451154232025146,
      0.014829661697149277,
      0.6137293577194214
     ],
     "probs": [
      0.22260868549346924,
      0.22054797410964966,
      0.03642337769269943,
      0.5204199552536011
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.29781126976013184,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5204199552536011,
      "probabilities": {
       "a": 0.22260868549346924,
       "b": 0.22054797410964966,
       "c": 0.03642337769269943,
       "d": 0.5204199552536011
      }
     }
    },
    "usage": {
     "input_tokens": 119,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_025",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 192,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 192,
     "positions": 192,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      3036,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      45301,
      20271,
      968,
      768,
      6394,
      797,
      730,
      1086,
      526,
      60843,
      768,
      13733,
      803,
      2496,
      875,
      768,
      15879,
      803,
      1235,
      23589,
      521,
      43152,
      803,
      1144,
      953,
      28984,
      521,
      43175,
      1095,
      1727,
      6709,
      875,
      2105,
      521,
      1203,
      1288,
      58154,
      856,
      1760,
      4997,
      884,
      779,
      13733,
      803,
      2496,
      521,
      1757,
      811,
      1727,
      30221,
      810,
      1210,
      950,
      14352,
      521,
      810,
      1288,
      875,
      1523,
      4092,
      779,
      2862,
      2712,
      2448,
      60843,
      811,
      1727,
      30221,
      856,
      1014,
      6612,
      521,
      1757,
      859,
      21225,
      779,
      2084,
      811,
      840,
      522,
      9381,
      779,
      13733,
      803,
      2496,
      810,
      2337,
      1892,
      1727,
      5401,
      523,
      43175,
      1030,
      730,
      1086,
      529,
      521,
      779,
      6709,
      5942,
      14643,
      931,
      2076,
      779,
      18302,
      950,
      1320,
      21440,
      1738,
      874,
      33533,
      523,
      12747,
      856,
      18887,
      811,
      779,
      40951,
      803,
      779,
      13838,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      941,
      6709,
      1090,
      950,
      14352,
      523,
      20,
      19,
      16,
      794,
      535,
      941,
      30221,
      523,
      20,
      19,
      16,
      781,
      535,
      941,
      6394,
      523,
      20,
      19,
      16,
      778,
      535,
      889,
      47348,
      15395,
      1685,
      779,
      6709,
      1090,
      950,
      14352,
      810,
      779,
      6394,
      523,
      20,
      21
     ],
     "markers": [
      147,
      158,
      166,
      174
     ],
     "logits_raw": [
      -0.358635812997818,
      -0.6387908458709717,
      0.28748929500579834,
      0.5898016095161438
     ],
     "probs_raw": [
      0.16011600196361542,
      0.12099432200193405,
      0.3055225908756256,
      0.4133670926094055
     ],
     "probs": [
      0.18607696890830994,
      0.15232786536216736,
      0.2952173948287964,
      0.3663778007030487
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.07116040587425232,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.3663778007030487,
      "probabilities": {
       "a": 0.18607696890830994,
       "b": 0.15232786536216736,
       "c": 0.2952173948287964,
       "d": 0.3663778007030487
      }
     }
    },
    "usage": {
     "input_tokens": 192,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_026",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 105,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 105,
     "positions": 105,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      54717,
      61895,
      1088,
      1878,
      997,
      39420,
      6881,
      997,
      50902,
      32600,
      10723,
      843,
      7377,
      560,
      518,
      856,
      2492,
      4965,
      811,
      843,
      777,
      916,
      518,
      2473,
      59491,
      523,
      20463,
      803,
      779,
      2773,
      15450,
      856,
      55495,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      9758,
      938,
      11483,
      916,
      779,
      1452,
      26196,
      797,
      4272,
      12158,
      20,
      19,
      16,
      794,
      535,
      9758,
      938,
      5898,
      21127,
      786,
      893,
      810,
      775,
      1570,
      1021,
      20,
      19,
      16,
      781,
      535,
      9758,
      6378,
      768,
      19467,
      37344,
      523,
      20,
      19,
      16,
      778,
      535,
      9758,
      5308,
      811,
      1640,
      6033,
      2563,
      779,
      3036,
      523,
      20,
      21
     ],
     "markers": [
      50,
      65,
      80,
      91
     ],
     "logits_raw": [
      -8.456859588623047,
      -7.910188674926758,
      -4.601129531860352,
      -2.9162330627441406
     ],
     "probs_raw": [
      0.0032805362716317177,
      0.005667101591825485,
      0.15504834055900574,
      0.8360040187835693
     ],
     "probs": [
      0.014177599921822548,
      0.0209506768733263,
      0.2227291464805603,
      0.7421425580978394
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.519413411617279,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.7421425580978394,
      "probabilities": {
       "a": 0.014177599921822548,
       "b": 0.0209506768733263,
       "c": 0.2227291464805603,
       "d": 0.7421425580978394
      }
     }
    },
    "usage": {
     "input_tokens": 105,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_027",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 94,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 94,
     "positions": 94,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      8496,
      21671,
      2499,
      1504,
      1878,
      997,
      39420,
      6881,
      997,
      561,
      788,
      20167,
      810,
      17683,
      802,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1725,
      797,
      3730,
      10798,
      11128,
      523,
      20,
      19,
      16,
      794,
      535,
      5650,
      9920,
      810,
      5650,
      3730,
      779,
      3685,
      803,
      779,
      2546,
      4997,
      52797,
      1939,
      523,
      20,
      19,
      16,
      781,
      535,
      3730,
      779,
      3685,
      803,
      779,
      2546,
      4997,
      52797,
      1939,
      523,
      20,
      19,
      16,
      778,
      535,
      9920,
      779,
      3685,
      803,
      779,
      2546,
      4997,
      52797,
      1939,
      523,
      20,
      21
     ],
     "markers": [
      34,
      45,
      64,
      79
     ],
     "logits_raw": [
      -1.4790819883346558,
      3.206943988800049,
      -0.3642081022262573,
      -1.1758590936660767
     ],
     "probs_raw": [
      0.008785426616668701,
      0.9525288343429565,
      0.02678837813436985,
      0.011897370219230652
     ],
     "probs": [
      0.03040599264204502,
      0.8644078373908997,
      0.06742636859416962,
      0.03775973990559578
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.79698146879673,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.8644078373908997,
      "probabilities": {
       "a": 0.03040599264204502,
       "b": 0.8644078373908997,
       "c": 0.06742636859416962,
       "d": 0.03775973990559578
      }
     }
    },
    "usage": {
     "input_tokens": 94,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_028",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 319,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 319,
     "positions": 319,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      3036,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      64188,
      856,
      779,
      9829,
      27355,
      803,
      768,
      5792,
      12800,
      968,
      902,
      12235,
      4129,
      2083,
      2844,
      779,
      2105,
      803,
      1210,
      7950,
      521,
      2166,
      40918,
      523,
      941,
      5792,
      6034,
      768,
      32787,
      7376,
      896,
      6527,
      32886,
      587,
      960,
      856,
      4025,
      1108,
      797,
      779,
      3308,
      896,
      3295,
      803,
      779,
      2083,
      3107,
      43175,
      27012,
      5828,
      521,
      810,
      19848,
      803,
      1113,
      1312,
      5817,
      521,
      988,
      40314,
      8462,
      1862,
      791,
      22333,
      1626,
      9774,
      17844,
      810,
      45127,
      3255,
      523,
      43175,
      1098,
      64188,
      1090,
      7950,
      1827,
      1450,
      3295,
      797,
      779,
      4598,
      1047,
      18660,
      803,
      1450,
      4799,
      523,
      941,
      3295,
      12393,
      988,
      30080,
      5457,
      968,
      779,
      21384,
      803,
      768,
      6527,
      1985,
      37249,
      1859,
      10231,
      42605,
      523,
      941,
      64188,
      953,
      20890,
      810,
      23548,
      12608,
      875,
      779,
      12494,
      803,
      1210,
      7950,
      523,
      4162,
      768,
      42212,
      8727,
      521,
      779,
      64188,
      953,
      5930,
      4477,
      803,
      779,
      7786,
      523,
      4162,
      779,
      12235,
      4129,
      17436,
      811,
      4025,
      779,
      64188,
      1090,
      12235,
      4673,
      521,
      779,
      64188,
      60509,
      1033,
      5614,
      4272,
      523,
      941,
      38203,
      8753,
      14340,
      779,
      5792,
      521,
      13621,
      774,
      779,
      11445,
      10674,
      779,
      3295,
      803,
      779,
      2083,
      3107,
      521,
      1563,
      871,
      896,
      779,
      3295,
      953,
      2638,
      779,
      5792,
      23984,
      875,
      6527,
      32886,
      587,
      960,
      16655,
      521,
      810,
      16365,
      20524,
      30525,
      523,
      2805,
      8727,
      521,
      779,
      12235,
      4129,
      1090,
      26917,
      48207,
      811,
      14993,
      41240,
      592,
      988,
      779,
      64188,
      1090,
      12494,
      8727,
      523,
      941,
      64188,
      1090,
      26917,
      7007,
      811,
      779,
      11258,
      803,
      1559,
      4589,
      523,
      941,
      8727,
      16277,
      1753,
      7330,
      779,
      899,
      1016,
      3501,
      4589,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      6416,
      41748,
      906,
      768,
      31137,
      2723,
      2862,
      523,
      20,
      19,
      16,
      794,
      535,
      6416,
      41748,
      906,
      5211,
      33763,
      523,
      20,
      19,
      16,
      781,
      535,
      797,
      51039,
      41748,
      521,
      1929,
      803,
      3749,
      15850,
      1170,
      31355,
      523,
      20,
      19,
      16,
      778,
      535,
      797,
      51039,
      41748,
      521,
      1929,
      803,
      6527,
      2250,
      36115,
      598,
      523,
      20,
      21
     ],
     "markers": [
      263,
      276,
      287,
      303
     ],
     "logits_raw": [
      -1.302972435951233,
      1.3318623304367065,
      -1.1114861965179443,
      -1.2292307615280151
     ],
     "probs_raw": [
      0.058043066412210464,
      0.80917888879776,
      0.07029295712709427,
      0.06248502433300018
     ],
     "probs": [
      0.10237134993076324,
      0.6723430156707764,
      0.11737720668315887,
      0.1079084724187851
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.5549658089876175,
     "near_tie": false,
     "gold": "b",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.6723430156707764,
      "probabilities": {
       "a": 0.10237134993076324,
       "b": 0.6723430156707764,
       "c": 0.11737720668315887,
       "d": 0.1079084724187851
      }
     }
    },
    "usage": {
     "input_tokens": 319,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_029",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 74,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 74,
     "positions": 74,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      16290,
      1878,
      997,
      39420,
      6881,
      997,
      37512,
      2574,
      586,
      3838,
      7809,
      797,
      859,
      5045,
      785,
      17868,
      4027,
      3504,
      523,
      49936,
      582,
      6572,
      523,
      50168,
      38822,
      592,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      859,
      1550,
      20,
      19,
      16,
      794,
      535,
      3504,
      1550,
      20,
      19,
      16,
      781,
      535,
      6572,
      1550,
      20,
      19,
      16,
      778,
      535,
      859,
      810,
      6572,
      20,
      21
     ],
     "markers": [
      45,
      52,
      59,
      66
     ],
     "logits_raw": [
      -0.2669294476509094,
      -0.4548293352127075,
      -1.1632142066955566,
      1.690333366394043
     ],
     "probs_raw": [
      0.10733412951231003,
      0.08894754946231842,
      0.04380122572183609,
      0.759917140007019
     ],
     "probs": [
      0.155056893825531,
      0.1355808526277542,
      0.0817398950457573,
      0.6276223063468933
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.4725654125213623,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.6276223063468933,
      "probabilities": {
       "a": 0.155056893825531,
       "b": 0.1355808526277542,
       "c": 0.0817398950457573,
       "d": 0.6276223063468933
      }
     }
    },
    "usage": {
     "input_tokens": 74,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_030",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 91,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 91,
     "positions": 91,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      52220,
      6028,
      598,
      1878,
      997,
      39420,
      6881,
      997,
      549,
      2995,
      21179,
      29375,
      906,
      17912,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      768,
      22028,
      811,
      33353,
      20845,
      523,
      20,
      19,
      16,
      794,
      535,
      768,
      22028,
      811,
      22710,
      11104,
      523,
      20,
      19,
      16,
      781,
      535,
      12738,
      6421,
      4272,
      933,
      4183,
      6602,
      768,
      5589,
      1888,
      768,
      29647,
      803,
      2396,
      575,
      852,
      523,
      20,
      19,
      16,
      778,
      535,
      12738,
      6421,
      4272,
      933,
      4183,
      14635,
      1337,
      779,
      2238,
      11364,
      875,
      1113,
      523,
      20,
      21
     ],
     "markers": [
      30,
      41,
      52,
      73
     ],
     "logits_raw": [
      -0.01371368020772934,
      -3.478639602661133,
      -2.0193262100219727,
      0.3967164158821106
     ],
     "probs_raw": [
      0.3740668296813965,
      0.011699061840772629,
      0.050341106951236725,
      0.5638930201530457
     ],
     "probs": [
      0.375444233417511,
      0.03159448876976967,
      0.08960627764463425,
      0.5033549666404724
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.12791073322296143,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5033549666404724,
      "probabilities": {
       "a": 0.375444233417511,
       "b": 0.03159448876976967,
       "c": 0.08960627764463425,
       "d": 0.5033549666404724
      }
     }
    },
    "usage": {
     "input_tokens": 91,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_031",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      9467,
      811,
      33620,
      2371,
      521,
      768,
      3444,
      8035,
      1090,
      21925,
      10373,
      14808,
      2363,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      26368,
      3384,
      8199,
      523,
      19628,
      20,
      19,
      16,
      794,
      535,
      9552,
      8199,
      523,
      3671,
      20694,
      20,
      19,
      16,
      781,
      535,
      36218,
      8199,
      523,
      30852,
      810,
      36025,
      20,
      19,
      16,
      778,
      535,
      5294,
      8199,
      523,
      20451,
      960,
      20,
      21
     ],
     "markers": [
      38,
      48,
      58,
      69
     ],
     "logits_raw": [
      -1.1906007528305054,
      2.4247899055480957,
      -2.300215005874634,
      -2.701061248779297
     ],
     "probs_raw": [
      0.025828873738646507,
      0.9599524736404419,
      0.008515420369803905,
      0.005703229922801256
     ],
     "probs": [
      0.06655921787023544,
      0.8806865215301514,
      0.030127935111522675,
      0.022626303136348724
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.8141273036599159,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.8806865215301514,
      "probabilities": {
       "a": 0.06655921787023544,
       "b": 0.8806865215301514,
       "c": 0.030127935111522675,
       "d": 0.022626303136348724
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_032",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 145,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 145,
     "positions": 145,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      3034,
      41336,
      1878,
      997,
      39420,
      6881,
      997,
      3493,
      856,
      779,
      5521,
      1685,
      902,
      13954,
      5562,
      810,
      768,
      7889,
      5562,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1592,
      13954,
      5562,
      856,
      2538,
      968,
      7250,
      521,
      2158,
      768,
      7889,
      5562,
      19000,
      988,
      730,
      1664,
      811,
      730,
      1838,
      1779,
      523,
      20,
      19,
      16,
      794,
      535,
      835,
      7889,
      5562,
      856,
      10491,
      988,
      12255,
      14506,
      14820,
      6793,
      521,
      2158,
      902,
      13954,
      5562,
      856,
      1550,
      3973,
      916,
      4985,
      10621,
      523,
      20,
      19,
      16,
      781,
      535,
      835,
      7889,
      5562,
      11335,
      811,
      768,
      1423,
      933,
      768,
      3795,
      803,
      1779,
      521,
      2158,
      902,
      13954,
      5562,
      11136,
      768,
      17540,
      2802,
      2565,
      523,
      20,
      19,
      16,
      778,
      535,
      1592,
      13954,
      5562,
      11335,
      811,
      768,
      1423,
      933,
      768,
      3795,
      803,
      1779,
      521,
      2158,
      768,
      7889,
      5562,
      11136,
      768,
      17540,
      2802,
      2565,
      523,
      20,
      21
     ],
     "markers": [
      36,
      62,
      89,
      117
     ],
     "logits_raw": [
      -2.510103464126587,
      -1.3151054382324219,
      0.091059111058712,
      0.8483920097351074
     ],
     "probs_raw": [
      0.021492037922143936,
      0.07100004702806473,
      0.2896997630596161,
      0.6178081631660461
     ],
     "probs": [
      0.04813937842845917,
      0.11303896456956863,
      0.30865028500556946,
      0.5301713943481445
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.22152110934257507,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5301713943481445,
      "probabilities": {
       "a": 0.04813937842845917,
       "b": 0.11303896456956863,
       "c": 0.30865028500556946,
       "d": 0.5301713943481445
      }
     }
    },
    "usage": {
     "input_tokens": 145,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_033",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 308,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 308,
     "positions": 308,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18807,
      780,
      3129,
      1878,
      997,
      39420,
      6881,
      997,
      2443,
      3952,
      11335,
      811,
      779,
      2773,
      2203,
      13909,
      587,
      18588,
      2299,
      1090,
      2743,
      510,
      1311,
      1090,
      18805,
      963,
      779,
      7952,
      9276,
      1090,
      6756,
      3823,
      511,
      768,
      27549,
      2448,
      54907,
      963,
      1123,
      906,
      950,
      9657,
      523,
      859,
      8107,
      779,
      44726,
      889,
      49994,
      797,
      1235,
      43053,
      523,
      7356,
      859,
      2062,
      779,
      6282,
      963,
      768,
      38455,
      523,
      7356,
      43175,
      1098,
      5707,
      6252,
      803,
      56312,
      2274,
      3742,
      1846,
      14231,
      521,
      4053,
      730,
      527,
      1607,
      2444,
      529,
      3787,
      18588,
      7356,
      1311,
      953,
      902,
      34760,
      811,
      768,
      3847,
      810,
      768,
      4311,
      5271,
      2217,
      523,
      7356,
      3408,
      510,
      7356,
      20385,
      22720,
      1203,
      48109,
      3085,
      1048,
      521,
      859,
      4549,
      1892,
      811,
      1607,
      3462,
      15600,
      1090,
      6756,
      570,
      810,
      21791,
      1419,
      779,
      6756,
      803,
      29281,
      52460,
      1088,
      1206,
      521,
      1494,
      10292,
      523,
      43175,
      3493,
      856,
      5242,
      8600,
      540,
      18588,
      950,
      5751,
      1123,
      523,
      43175,
      2299,
      1090,
      22062,
      521,
      19398,
      510,
      18588,
      43175,
      2299,
      1188,
      874,
      1113,
      2084,
      1757,
      568,
      3823,
      511,
      52460,
      1088,
      1206,
      25723,
      3787,
      569,
      587,
      559,
      62261,
      889,
      983,
      7569,
      2771,
      521,
      4162,
      730,
      1838,
      21045,
      521,
      781,
      523,
      730,
      1848,
      530,
      13699,
      6314,
      779,
      12543,
      521,
      1235,
      1400,
      10953,
      896,
      889,
      983,
      7569,
      2771,
      33871,
      896,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      19178,
      2274,
      875,
      2104,
      797,
      730,
      2444,
      529,
      953,
      2581,
      23073,
      916,
      2507,
      14801,
      19555,
      1586,
      797,
      5242,
      20,
      19,
      16,
      794,
      535,
      19178,
      2274,
      875,
      2104,
      797,
      730,
      2444,
      529,
      953,
      2581,
      23073,
      916,
      2507,
      14801,
      19555,
      1586,
      1113,
      3648,
      3200,
      20,
      19,
      16,
      781,
      535,
      19178,
      2274,
      875,
      2104,
      797,
      730,
      2444,
      529,
      953,
      2581,
      23073,
      916,
      2507,
      37875,
      1113,
      3648,
      3200,
      20,
      19,
      16,
      778,
      535,
      19178,
      2274,
      875,
      2104,
      797,
      730,
      2444,
      529,
      953,
      2581,
      23073,
      916,
      2507,
      37875,
      797,
      5242,
      20,
      21
     ],
     "markers": [
      218,
      241,
      265,
      287
     ],
     "logits_raw": [
      0.10337462276220322,
      -2.55706524848938,
      -0.875851035118103,
      -0.2083251178264618
     ],
     "probs_raw": [
      0.45919573307037354,
      0.03210579603910446,
      0.17247475683689117,
      0.3362237215042114
     ],
     "probs": [
      0.408709853887558,
      0.06110251322388649,
      0.2030612677335739,
      0.3271264135837555
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.08158344030380249,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.408709853887558,
      "probabilities": {
       "a": 0.408709853887558,
       "b": 0.06110251322388649,
       "c": 0.2030612677335739,
       "d": 0.3271264135837555
      }
     }
    },
    "usage": {
     "input_tokens": 308,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_034",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 87,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 87,
     "positions": 87,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      2514,
      803,
      7524,
      797,
      768,
      5392,
      2033,
      938,
      811,
      874,
      48586,
      803,
      5392,
      875,
      2759,
      3378,
      523,
      4162,
      1149,
      2341,
      35210,
      36744,
      4915,
      521,
      7524,
      1400,
      6305,
      15603,
      1144,
      803,
      779,
      2773,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      63183,
      20,
      19,
      16,
      794,
      535,
      1097,
      1380,
      20,
      19,
      16,
      781,
      535,
      39393,
      803,
      24710,
      20,
      19,
      16,
      778,
      535,
      7329,
      1617,
      10021,
      20,
      21
     ],
     "markers": [
      58,
      64,
      71,
      79
     ],
     "logits_raw": [
      -2.793691396713257,
      -2.0432310104370117,
      -1.2673485279083252,
      2.3498144149780273
     ],
     "probs_raw": [
      0.005585509818047285,
      0.011829973198473454,
      0.02570071816444397,
      0.9568836688995361
     ],
     "probs": [
      0.022172139957547188,
      0.037898775190114975,
      0.06596741825342178,
      0.8739617466926575
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.8079943284392357,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.8739617466926575,
      "probabilities": {
       "a": 0.022172139957547188,
       "b": 0.037898775190114975,
       "c": 0.06596741825342178,
       "d": 0.8739617466926575
      }
     }
    },
    "usage": {
     "input_tokens": 87,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_035",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 120,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 120,
     "positions": 120,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      578,
      2499,
      813,
      8731,
      1878,
      997,
      39420,
      6881,
      997,
      3383,
      1413,
      1166,
      768,
      43186,
      1862,
      797,
      4741,
      521,
      1620,
      1499,
      874,
      779,
      7128,
      803,
      779,
      6196,
      3616,
      13818,
      875,
      906,
      2801,
      896,
      34162,
      875,
      896,
      43186,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1311,
      1499,
      7860,
      1209,
      6214,
      875,
      6384,
      43368,
      1399,
      875,
      4948,
      6531,
      803,
      779,
      2043,
      17165,
      20,
      19,
      16,
      794,
      535,
      1311,
      1499,
      874,
      13820,
      14977,
      1337,
      9617,
      20,
      19,
      16,
      781,
      535,
      1311,
      1499,
      7860,
      2442,
      6214,
      875,
      6384,
      43368,
      1399,
      875,
      4948,
      6531,
      803,
      779,
      2043,
      17165,
      20,
      19,
      16,
      778,
      535,
      1311,
      1499,
      874,
      9617,
      875,
      1113,
      4948,
      43368,
      20,
      21
     ],
     "markers": [
      53,
      74,
      86,
      107
     ],
     "logits_raw": [
      1.3277885913848877,
      -0.30665910243988037,
      -1.4691928625106812,
      -2.5176525115966797
     ],
     "probs_raw": [
      0.7828211188316345,
      0.1526971459388733,
      0.04774731770157814,
      0.016734344884753227
     ],
     "probs": [
      0.661874532699585,
      0.20592838525772095,
      0.08975527435541153,
      0.04244174808263779
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.455946147441864,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.661874532699585,
      "probabilities": {
       "a": 0.661874532699585,
       "b": 0.20592838525772095,
       "c": 0.08975527435541153,
       "d": 0.04244174808263779
      }
     }
    },
    "usage": {
     "input_tokens": 120,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_036",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 150,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 150,
     "positions": 150,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      21167,
      1878,
      997,
      39420,
      6881,
      997,
      553,
      26717,
      13807,
      730,
      1086,
      525,
      19190,
      803,
      24008,
      1969,
      37891,
      523,
      2624,
      9051,
      896,
      1166,
      15399,
      797,
      1423,
      730,
      526,
      963,
      1058,
      1155,
      523,
      1838,
      1094,
      5611,
      523,
      1030,
      1423,
      730,
      529,
      10765,
      772,
      5068,
      768,
      730,
      530,
      514,
      2222,
      522,
      40864,
      1108,
      7533,
      959,
      803,
      24008,
      1969,
      2624,
      9051,
      523,
      1030,
      1423,
      730,
      530,
      779,
      9051,
      13305,
      730,
      527,
      20432,
      522,
      526,
      523,
      1030,
      779,
      2835,
      1423,
      10765,
      772,
      4997,
      730,
      7765,
      19190,
      523,
      3747,
      856,
      10765,
      772,
      1090,
      6524,
      797,
      779,
      730,
      7765,
      19190,
      803,
      9051,
      4997,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1058,
      529,
      521,
      1873,
      20,
      19,
      16,
      794,
      535,
      1058,
      533,
      521,
      1873,
      20,
      19,
      16,
      781,
      535,
      1058,
      533,
      521,
      6019,
      20,
      19,
      16,
      778,
      535,
      730,
      10173,
      1056,
      20,
      21
     ],
     "markers": [
      115,
      124,
      133,
      142
     ],
     "logits_raw": [
      -2.7359061241149902,
      -1.6822254657745361,
      -1.4564356803894043,
      -5.317482948303223
     ],
     "probs_raw": [
      0.13265100121498108,
      0.38046807050704956,
      0.47684529423713684,
      0.010035661049187183
     ],
     "probs": [
      0.17315810918807983,
      0.36756089329719543,
      0.4318934977054596,
      0.027387524023652077
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.06433260440826416,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 1.862645149230957e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.4318934977054596,
      "probabilities": {
       "a": 0.17315810918807983,
       "b": 0.36756089329719543,
       "c": 0.4318934977054596,
       "d": 0.027387524023652077
      }
     }
    },
    "usage": {
     "input_tokens": 150,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_037",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 76,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 76,
     "positions": 76,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      5606,
      30049,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      856,
      779,
      1529,
      3429,
      15488,
      803,
      779,
      29749,
      8046,
      1678,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      44437,
      6754,
      20,
      19,
      16,
      794,
      535,
      837,
      2998,
      20,
      19,
      16,
      781,
      535,
      28718,
      837,
      2998,
      20,
      19,
      16,
      778,
      535,
      1976,
      9529,
      11755,
      803,
      1181,
      55092,
      19453,
      810,
      8305,
      837,
      2998,
      20,
      21
     ],
     "markers": [
      38,
      45,
      52,
      60
     ],
     "logits_raw": [
      -0.564216136932373,
      -0.7476570010185242,
      -0.27579110860824585,
      -0.22039031982421875
     ],
     "probs_raw": [
      0.21848095953464508,
      0.18186379969120026,
      0.2915244400501251,
      0.3081308901309967
     ],
     "probs": [
      0.22808265686035156,
      0.20007041096687317,
      0.28026649355888367,
      0.2915804088115692
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.011313915252685547,
     "near_tie": true,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.2915804088115692,
      "probabilities": {
       "a": 0.22808265686035156,
       "b": 0.20007041096687317,
       "c": 0.28026649355888367,
       "d": 0.2915804088115692
      }
     }
    },
    "usage": {
     "input_tokens": 76,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_038",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 143,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 143,
     "positions": 143,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      578,
      2499,
      813,
      8731,
      1878,
      997,
      39420,
      6881,
      997,
      3493,
      856,
      779,
      2120,
      5521,
      1685,
      779,
      16865,
      2718,
      59024,
      843,
      7094,
      518,
      810,
      34258,
      3995,
      770,
      1711,
      843,
      15612,
      518,
      10524,
      811,
      6232,
      7152,
      6976,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      835,
      7094,
      856,
      768,
      3437,
      10805,
      4355,
      811,
      6232,
      7152,
      6976,
      2158,
      27967,
      856,
      768,
      3769,
      4355,
      20,
      19,
      16,
      794,
      535,
      27967,
      6630,
      15637,
      779,
      40129,
      3996,
      810,
      8530,
      43237,
      1101,
      896,
      1413,
      856,
      6195,
      25923,
      1490,
      779,
      3996,
      16454,
      803,
      779,
      27967,
      1927,
      20,
      19,
      16,
      781,
      535,
      941,
      27967,
      1927,
      42773,
      902,
      22156,
      23819,
      875,
      62228,
      777,
      6566,
      11201,
      13294,
      797,
      779,
      1927,
      20436,
      20,
      19,
      16,
      778,
      535,
      27967,
      6630,
      1052,
      2184,
      2410,
      797,
      2211,
      7515,
      11441,
      40129,
      6630,
      1112,
      1014,
      523,
      20,
      21
     ],
     "markers": [
      52,
      74,
      102,
      124
     ],
     "logits_raw": [
      -0.7698924541473389,
      0.4720029830932617,
      -0.3455861806869507,
      -0.4429304003715515
     ],
     "probs_raw": [
      0.13554823398590088,
      0.46929097175598145,
      0.20718948543071747,
      0.1879713535308838
     ],
     "probs": [
      0.16541790962219238,
      0.401660680770874,
      0.22398363053798676,
      0.20893773436546326
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.17767705023288727,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.401660680770874,
      "probabilities": {
       "a": 0.16541790962219238,
       "b": 0.401660680770874,
       "c": 0.22398363053798676,
       "d": 0.20893773436546326
      }
     }
    },
    "usage": {
     "input_tokens": 143,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_039",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      4167,
      578,
      2499,
      1504,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      3426,
      18768,
      988,
      775,
      1008,
      1666,
      856,
      15372,
      1884,
      1403,
      1186,
      938,
      15399,
      3032,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      22482,
      18768,
      856,
      9617,
      20,
      19,
      16,
      794,
      535,
      22482,
      7691,
      30322,
      22482,
      2780,
      20,
      19,
      16,
      781,
      535,
      14060,
      41938,
      856,
      9617,
      20,
      19,
      16,
      778,
      535,
      16791,
      975,
      9806,
      856,
      10836,
      20,
      21
     ],
     "markers": [
      43,
      52,
      62,
      71
     ],
     "logits_raw": [
      1.7035473585128784,
      -1.886009931564331,
      -2.6342694759368896,
      -5.541640281677246
     ],
     "probs_raw": [
      0.9602558612823486,
      0.026513198390603065,
      0.01254576351493597,
      0.0006852386868558824
     ],
     "probs": [
      0.8867239356040955,
      0.06826366484165192,
      0.039999522268772125,
      0.005012943875044584
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.8184602707624435,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.8867239356040955,
      "probabilities": {
       "a": 0.8867239356040955,
       "b": 0.06826366484165192,
       "c": 0.039999522268772125,
       "d": 0.005012943875044584
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_040",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 219,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 219,
     "positions": 219,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      4083,
      6984,
      768,
      1868,
      988,
      15202,
      523,
      13202,
      1316,
      856,
      1943,
      30133,
      1929,
      1450,
      730,
      1536,
      6194,
      10835,
      9545,
      31520,
      779,
      20481,
      803,
      1450,
      2238,
      6709,
      2204,
      3378,
      6027,
      810,
      856,
      1943,
      1709,
      853,
      57994,
      523,
      941,
      10761,
      27973,
      2094,
      51707,
      521,
      4746,
      2094,
      6791,
      521,
      810,
      1011,
      2094,
      5910,
      40698,
      523,
      2297,
      1052,
      6235,
      3758,
      3822,
      916,
      21570,
      810,
      797,
      7376,
      10373,
      11125,
      4135,
      523,
      2876,
      521,
      1413,
      856,
      1295,
      1235,
      3581,
      797,
      779,
      3681,
      1316,
      856,
      1209,
      9468,
      1399,
      1010,
      938,
      523,
      1783,
      902,
      19316,
      42724,
      521,
      1010,
      1188,
      17912,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      11667,
      811,
      2130,
      15202,
      523,
      13202,
      1172,
      9545,
      797,
      6738,
      2908,
      1010,
      6695,
      1320,
      1429,
      3758,
      7376,
      10373,
      11125,
      8938,
      523,
      20,
      19,
      16,
      794,
      535,
      1836,
      15202,
      523,
      13202,
      1337,
      1180,
      5388,
      803,
      3758,
      810,
      2944,
      1450,
      11245,
      1288,
      1689,
      11467,
      1010,
      811,
      3176,
      6738,
      811,
      1450,
      9545,
      523,
      20,
      19,
      16,
      781,
      535,
      1836,
      15202,
      523,
      13202,
      896,
      1010,
      4584,
      2130,
      1450,
      9545,
      1929,
      803,
      1180,
      5388,
      803,
      3758,
      523,
      20,
      19,
      16,
      778,
      535,
      2130,
      15202,
      523,
      13202,
      1172,
      9545,
      797,
      6738,
      1550,
      3032,
      779,
      10373,
      1178,
      11446,
      933,
      3032,
      1010,
      28062,
      8530,
      4135,
      523,
      20,
      21
     ],
     "markers": [
      116,
      142,
      171,
      193
     ],
     "logits_raw": [
      -2.3792548179626465,
      0.8844364881515503,
      -0.7305554747581482,
      -2.2510766983032227
     ],
     "probs_raw": [
      0.029866043478250504,
      0.7808738350868225,
      0.1553097814321518,
      0.03395038843154907
     ],
     "probs": [
      0.06395912170410156,
      0.6582744121551514,
      0.2076745629310608,
      0.0700918585062027
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.4505998492240906,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.6582744121551514,
      "probabilities": {
       "a": 0.06395912170410156,
       "b": 0.6582744121551514,
       "c": 0.2076745629310608,
       "d": 0.0700918585062027
      }
     }
    },
    "usage": {
     "input_tokens": 219,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_041",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 226,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 226,
     "positions": 226,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      3036,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      16779,
      953,
      11915,
      884,
      768,
      11513,
      12991,
      13668,
      811,
      1825,
      768,
      4153,
      7909,
      523,
      941,
      4153,
      55154,
      953,
      3469,
      5019,
      521,
      1101,
      779,
      16779,
      6806,
      811,
      7498,
      5329,
      797,
      779,
      34517,
      6707,
      7921,
      1929,
      950,
      4602,
      768,
      4605,
      9837,
      523,
      10238,
      53927,
      768,
      1681,
      33138,
      779,
      11513,
      12991,
      810,
      768,
      10231,
      7989,
      1230,
      523,
      941,
      16779,
      1928,
      3700,
      791,
      1391,
      810,
      10357,
      811,
      779,
      54322,
      523,
      2904,
      50322,
      1720,
      1994,
      10231,
      810,
      2076,
      950,
      5942,
      521,
      779,
      16779,
      940,
      21494,
      521,
      43175,
      1098,
      47715,
      2172,
      936,
      510,
      18588,
      941,
      47715,
      953,
      1459,
      13735,
      810,
      856,
      2166,
      884,
      8727,
      875,
      779,
      16779,
      1090,
      12494,
      523,
      18276,
      884,
      1518,
      11445,
      6955,
      521,
      779,
      6416,
      1271,
      3912,
      803,
      779,
      16779,
      1090,
      8294,
      1753,
      874,
      7877,
      968,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      779,
      28292,
      521,
      1720,
      768,
      27519,
      22430,
      968,
      779,
      16277,
      523,
      20,
      19,
      16,
      794,
      535,
      779,
      28292,
      521,
      2630,
      768,
      27519,
      22430,
      968,
      779,
      16277,
      523,
      20,
      19,
      16,
      781,
      535,
      779,
      16277,
      521,
      916,
      11497,
      811,
      779,
      28292,
      896,
      936,
      1400,
      2260,
      779,
      21712,
      31137,
      1288,
      936,
      25195,
      896,
      779,
      16779,
      8291,
      950,
      953,
      21712,
      523,
      20,
      19,
      16,
      778,
      535,
      779,
      16277,
      521,
      2630,
      12224,
      988,
      779,
      28292,
      523,
      20,
      21
     ],
     "markers": [
      149,
      165,
      181,
      212
     ],
     "logits_raw": [
      -0.2598228454589844,
      -0.670441746711731,
      0.7866662740707397,
      -2.2362213134765625
     ],
     "probs_raw": [
      0.21507954597473145,
      0.14264927804470062,
      0.6124681830406189,
      0.02980300597846508
     ],
     "probs": [
      0.24382537603378296,
      0.18184082210063934,
      0.5149134993553162,
      0.059420328587293625
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.2710881233215332,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.5149134993553162,
      "probabilities": {
       "a": 0.24382537603378296,
       "b": 0.18184082210063934,
       "c": 0.5149134993553162,
       "d": 0.059420328587293625
      }
     }
    },
    "usage": {
     "input_tokens": 226,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_042",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 73,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 73,
     "positions": 73,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      3600,
      1021,
      3981,
      36158,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      3981,
      3384,
      803,
      36025,
      968,
      10039,
      856,
      768,
      3680,
      2432,
      803,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      12499,
      3384,
      803,
      10807,
      20,
      19,
      16,
      794,
      535,
      2097,
      949,
      3294,
      16798,
      20,
      19,
      16,
      781,
      535,
      10439,
      5222,
      779,
      3952,
      20,
      19,
      16,
      778,
      535,
      13104,
      1437,
      23223,
      20,
      21
     ],
     "markers": [
      38,
      47,
      56,
      65
     ],
     "logits_raw": [
      1.2054716348648071,
      -2.7291550636291504,
      -2.9371767044067383,
      -3.3536458015441895
     ],
     "probs_raw": [
      0.9561097621917725,
      0.018694812431931496,
      0.01518373005092144,
      0.010011709295213223
     ],
     "probs": [
      0.8691593408584595,
      0.052293624728918076,
      0.04507270082831383,
      0.03347426652908325
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.8168657161295414,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 3.725290298461914e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.8691593408584595,
      "probabilities": {
       "a": 0.8691593408584595,
       "b": 0.052293624728918076,
       "c": 0.04507270082831383,
       "d": 0.03347426652908325
      }
     }
    },
    "usage": {
     "input_tokens": 73,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_043",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 72,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 72,
     "positions": 72,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30735,
      14977,
      10028,
      1878,
      997,
      39420,
      6881,
      997,
      31081,
      15909,
      797,
      784,
      14681,
      2841,
      803,
      768,
      13989,
      10321,
      31067,
      938,
      2588,
      906,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      4021,
      20744,
      32637,
      933,
      7604,
      32637,
      20,
      19,
      16,
      794,
      535,
      20744,
      32637,
      20,
      19,
      16,
      781,
      535,
      7604,
      32637,
      20,
      19,
      16,
      778,
      535,
      1623,
      1143,
      32637,
      20,
      21
     ],
     "markers": [
      39,
      50,
      57,
      64
     ],
     "logits_raw": [
      -1.8836767673492432,
      -2.334036350250244,
      -3.524329423904419,
      -3.608792781829834
     ],
     "probs_raw": [
      0.497659832239151,
      0.31720784306526184,
      0.0964730903506279,
      0.08865928649902344
     ],
     "probs": [
      0.4298710525035858,
      0.31161749362945557,
      0.13315387070178986,
      0.12535753846168518
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.11825355887413025,
     "near_tie": false,
     "gold": "b",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.4298710525035858,
      "probabilities": {
       "a": 0.4298710525035858,
       "b": 0.31161749362945557,
       "c": 0.13315387070178986,
       "d": 0.12535753846168518
      }
     }
    },
    "usage": {
     "input_tokens": 72,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_044",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 89,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 89,
     "positions": 89,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      3134,
      810,
      13032,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      20069,
      3345,
      1678,
      803,
      9170,
      800,
      5614,
      3714,
      10380,
      953,
      10328,
      968,
      779,
      39629,
      1678,
      906,
      768,
      1725,
      803,
      779,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      53311,
      59812,
      20,
      19,
      16,
      794,
      535,
      14366,
      10335,
      23335,
      4120,
      803,
      730,
      1848,
      526,
      20,
      19,
      16,
      781,
      535,
      2947,
      21889,
      40468,
      4120,
      20,
      19,
      16,
      778,
      535,
      54130,
      13501,
      4120,
      803,
      730,
      3699,
      528,
      20,
      21
     ],
     "markers": [
      48,
      55,
      68,
      77
     ],
     "logits_raw": [
      -2.3090124130249023,
      -2.2886719703674316,
      -0.1633472740650177,
      1.2341519594192505
     ],
     "probs_raw": [
      0.022151127457618713,
      0.02260630577802658,
      0.1893417239189148,
      0.7659007906913757
     ],
     "probs": [
      0.052051614969968796,
      0.05281345546245575,
      0.24104060232639313,
      0.654094398021698
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.41305379569530487,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.654094398021698,
      "probabilities": {
       "a": 0.052051614969968796,
       "b": 0.05281345546245575,
       "c": 0.24104060232639313,
       "d": 0.654094398021698
      }
     }
    },
    "usage": {
     "input_tokens": 89,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_045",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 106,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 106,
     "positions": 106,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      578,
      2499,
      813,
      8731,
      1878,
      997,
      39420,
      6881,
      997,
      25551,
      779,
      2773,
      2801,
      875,
      1058,
      598,
      2884,
      513,
      23606,
      587,
      13699,
      513,
      598,
      2884,
      1051,
      13264,
      4634,
      2011,
      13264,
      5762,
      766,
      2011,
      832,
      2884,
      4927,
      587,
      13699,
      28828,
      1235,
      803,
      779,
      2773,
      1529,
      20298,
      11854,
      779,
      1996,
      875,
      1058,
      598,
      2884,
      513,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      835,
      6232,
      7152,
      1996,
      20,
      19,
      16,
      794,
      535,
      835,
      37970,
      1996,
      20,
      19,
      16,
      781,
      535,
      835,
      15249,
      3547,
      13974,
      1996,
      20,
      19,
      16,
      778,
      535,
      835,
      7226,
      5043,
      916,
      26436,
      20,
      21
     ],
     "markers": [
      69,
      78,
      86,
      96
     ],
     "logits_raw": [
      1.1159785985946655,
      -0.7625792026519775,
      -0.41189315915107727,
      -1.6540266275405884
     ],
     "probs_raw": [
      0.6980953812599182,
      0.10667619109153748,
      0.15148460865020752,
      0.04374382644891739
     ],
     "probs": [
      0.5762596130371094,
      0.15060095489025116,
      0.19347375631332397,
      0.07966570556163788
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.3827858567237854,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.5762596130371094,
      "probabilities": {
       "a": 0.5762596130371094,
       "b": 0.15060095489025116,
       "c": 0.19347375631332397,
       "d": 0.07966570556163788
      }
     }
    },
    "usage": {
     "input_tokens": 106,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_046",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      16290,
      1878,
      997,
      39420,
      6881,
      997,
      2443,
      1996,
      8806,
      811,
      779,
      9257,
      3893,
      803,
      1547,
      14471,
      1005,
      595,
      931,
      521,
      1547,
      26171,
      521,
      810,
      1547,
      55984,
      549,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      47256,
      2293,
      2916,
      20,
      19,
      16,
      794,
      535,
      8670,
      2574,
      586,
      3838,
      20,
      19,
      16,
      781,
      535,
      12656,
      957,
      852,
      20,
      19,
      16,
      778,
      535,
      41778,
      8192,
      20,
      21
     ],
     "markers": [
      46,
      54,
      63,
      71
     ],
     "logits_raw": [
      -0.21103397011756897,
      -4.946765422821045,
      -1.4108130931854248,
      -0.5645290017127991
     ],
     "probs_raw": [
      0.4969521462917328,
      0.004361264873296022,
      0.14971217513084412,
      0.34897446632385254
     ],
     "probs": [
      0.4473859369754791,
      0.015188070014119148,
      0.18987658619880676,
      0.3475493788719177
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.0998365581035614,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.4473859369754791,
      "probabilities": {
       "a": 0.4473859369754791,
       "b": 0.015188070014119148,
       "c": 0.18987658619880676,
       "d": 0.3475493788719177
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_047",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 79,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 79,
     "positions": 79,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      542,
      6394,
      5867,
      1512,
      4100,
      44888,
      811,
      1450,
      2153,
      5822,
      1689,
      13564,
      20972,
      4381,
      4100,
      8255,
      523,
      941,
      13789,
      20314,
      968,
      1033,
      6394,
      9784,
      768,
      15488,
      803,
      1450,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1267,
      29871,
      20,
      19,
      16,
      794,
      535,
      16234,
      7695,
      20,
      19,
      16,
      781,
      535,
      49249,
      20,
      19,
      16,
      778,
      535,
      38939,
      20,
      21
     ],
     "markers": [
      53,
      60,
      67,
      73
     ],
     "logits_raw": [
      -3.3293027877807617,
      2.4848766326904297,
      -5.358247756958008,
      1.0101518630981445
     ],
     "probs_raw": [
      0.0024224012158811092,
      0.8115440607070923,
      0.000318483158480376,
      0.18571512401103973
     ],
     "probs": [
      0.011484759859740734,
      0.7309231758117676,
      0.002695730421692133,
      0.25489628314971924
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.47602689266204834,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.7309231758117676,
      "probabilities": {
       "a": 0.011484759859740734,
       "b": 0.7309231758117676,
       "c": 0.002695730421692133,
       "d": 0.25489628314971924
      }
     }
    },
    "usage": {
     "input_tokens": 79,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_048",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 136,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 136,
     "positions": 136,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      586,
      8988,
      21469,
      1878,
      997,
      39420,
      6881,
      997,
      3111,
      1144,
      803,
      1518,
      1547,
      21469,
      2177,
      779,
      2120,
      3338,
      843,
      18275,
      5582,
      859,
      524,
      1477,
      524,
      3462,
      518,
      1112,
      3503,
      8870,
      61611,
      7871,
      521,
      4262,
      811,
      17027,
      11241,
      8079,
      797,
      779,
      2641,
      906,
      803,
      730,
      1718,
      525,
      540,
      3108,
      31933,
      730,
      526,
      2171,
      859,
      3595,
      5665,
      1419,
      768,
      9953,
      1101,
      859,
      1738,
      1927,
      936,
      875,
      6471,
      523,
      3108,
      31933,
      730,
      527,
      2171,
      859,
      17453,
      779,
      30704,
      1680,
      1429,
      6248,
      29293,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      923,
      35736,
      521,
      923,
      35736,
      20,
      19,
      16,
      794,
      535,
      923,
      35736,
      521,
      4524,
      7871,
      20,
      19,
      16,
      781,
      535,
      4524,
      7871,
      521,
      923,
      35736,
      20,
      19,
      16,
      778,
      535,
      4524,
      7871,
      521,
      4524,
      7871,
      20,
      21
     ],
     "markers": [
      96,
      106,
      116,
      126
     ],
     "logits_raw": [
      1.6919503211975098,
      -0.03276406228542328,
      -0.9112541675567627,
      -1.3428339958190918
     ],
     "probs_raw": [
      0.7690266966819763,
      0.13705897331237793,
      0.0569356232881546,
      0.036978673189878464
     ],
     "probs": [
      0.6402612328529358,
      0.18676437437534332,
      0.09971431642770767,
      0.07326005399227142
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.45349685847759247,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.6402612328529358,
      "probabilities": {
       "a": 0.6402612328529358,
       "b": 0.18676437437534332,
       "c": 0.09971431642770767,
       "d": 0.07326005399227142
      }
     }
    },
    "usage": {
     "input_tokens": 136,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_049",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 96,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 96,
     "positions": 96,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      5146,
      3717,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      8294,
      856,
      3347,
      1337,
      3560,
      2514,
      2098,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      941,
      7524,
      1753,
      874,
      7795,
      963,
      7226,
      523,
      20,
      19,
      16,
      794,
      535,
      1311,
      856,
      902,
      2623,
      803,
      768,
      9885,
      2098,
      6690,
      875,
      27877,
      523,
      20,
      19,
      16,
      781,
      535,
      941,
      2203,
      1011,
      874,
      33518,
      811,
      8487,
      768,
      5814,
      3443,
      523,
      20,
      19,
      16,
      778,
      535,
      941,
      7747,
      10361,
      29581,
      1818,
      896,
      1011,
      874,
      1593,
      811,
      1705,
      21622,
      523,
      20,
      21
     ],
     "markers": [
      32,
      45,
      62,
      78
     ],
     "logits_raw": [
      -6.835418224334717,
      -0.9563892483711243,
      -3.4500579833984375,
      -0.31408247351646423
     ],
     "probs_raw": [
      0.0009367892635054886,
      0.3348664939403534,
      0.027662096545100212,
      0.636534571647644
     ],
     "probs": [
      0.005424226634204388,
      0.3615814447402954,
      0.0608958899974823,
      0.5720984935760498
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.2105170488357544,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.5720984935760498,
      "probabilities": {
       "a": 0.005424226634204388,
       "b": 0.3615814447402954,
       "c": 0.0608958899974823,
       "d": 0.5720984935760498
      }
     }
    },
    "usage": {
     "input_tokens": 96,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_050",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 78,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 78,
     "positions": 78,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      578,
      12453,
      17263,
      1878,
      997,
      39420,
      6881,
      997,
      64005,
      779,
      10805,
      15702,
      1417,
      523,
      5843,
      1051,
      15702,
      527,
      523,
      528,
      598,
      523,
      11205,
      1180,
      5642,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      730,
      532,
      523,
      529,
      20,
      19,
      16,
      794,
      535,
      15702,
      532,
      523,
      529,
      20,
      19,
      16,
      781,
      535,
      15702,
      913,
      523,
      2925,
      20,
      19,
      16,
      778,
      535,
      730,
      1536,
      523,
      8333,
      20,
      21
     ],
     "markers": [
      42,
      51,
      60,
      69
     ],
     "logits_raw": [
      -0.026801295578479767,
      -0.5347694158554077,
      -2.2902116775512695,
      -0.3073588013648987
     ],
     "probs_raw": [
      0.4063265323638916,
      0.24449355900287628,
      0.04225601628422737,
      0.30692383646965027
     ],
     "probs": [
      0.36864981055259705,
      0.25646352767944336,
      0.07318627089262009,
      0.3017003834247589
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.06694942712783813,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.36864981055259705,
      "probabilities": {
       "a": 0.36864981055259705,
       "b": 0.25646352767944336,
       "c": 0.07318627089262009,
       "d": 0.3017003834247589
      }
     }
    },
    "usage": {
     "input_tokens": 78,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_051",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 145,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 145,
     "positions": 145,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      3034,
      41336,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      8757,
      797,
      6131,
      803,
      3528,
      769,
      2796,
      3525,
      810,
      1993,
      9903,
      6486,
      968,
      963,
      3653,
      730,
      528,
      523,
      528,
      3847,
      1779,
      6027,
      856,
      7951,
      968,
      17912,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      4169,
      16120,
      884,
      6134,
      13665,
      896,
      1166,
      2047,
      968,
      1203,
      824,
      2644,
      916,
      9556,
      803,
      9903,
      523,
      20,
      19,
      16,
      794,
      535,
      779,
      36448,
      803,
      4163,
      920,
      42437,
      11490,
      3545,
      33450,
      868,
      6108,
      810,
      18583,
      969,
      916,
      3609,
      803,
      4624,
      2222,
      54127,
      57203,
      896,
      13473,
      6690,
      1517,
      523,
      20,
      19,
      16,
      781,
      535,
      4048,
      9903,
      39052,
      521,
      902,
      20247,
      18438,
      521,
      810,
      1434,
      2385,
      1833,
      963,
      779,
      903,
      1071,
      584,
      14415,
      730,
      528,
      3475,
      797,
      3227,
      16485,
      2468,
      521,
      24973,
      20,
      19,
      16,
      778,
      535,
      1113,
      803,
      779,
      3537,
      523,
      20,
      21
     ],
     "markers": [
      50,
      72,
      103,
      135
     ],
     "logits_raw": [
      -2.4747769832611084,
      1.4043216705322266,
      -0.4620096981525421,
      2.141174077987671
     ],
     "probs_raw": [
      0.006331176962703466,
      0.30630606412887573,
      0.047382522374391556,
      0.6399801969528198
     ],
     "probs": [
      0.020735442638397217,
      0.3312358558177948,
      0.08732527494430542,
      0.560703456401825
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.22946760058403015,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 7.450580596923828e-09,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.560703456401825,
      "probabilities": {
       "a": 0.020735442638397217,
       "b": 0.3312358558177948,
       "c": 0.08732527494430542,
       "d": 0.560703456401825
      }
     }
    },
    "usage": {
     "input_tokens": 145,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_052",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 143,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 143,
     "positions": 143,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18166,
      1878,
      997,
      39420,
      6881,
      997,
      1098,
      9257,
      14951,
      793,
      10805,
      5559,
      1403,
      7864,
      803,
      33128,
      529,
      21369,
      810,
      4059,
      20099,
      528,
      938,
      10372,
      3658,
      856,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      4059,
      46777,
      63868,
      518,
      2011,
      2642,
      25960,
      63868,
      518,
      17493,
      4059,
      21369,
      5381,
      518,
      20,
      19,
      16,
      794,
      535,
      33128,
      529,
      46777,
      63868,
      518,
      2011,
      4059,
      46777,
      63868,
      518,
      17493,
      4059,
      22386,
      549,
      529,
      518,
      528,
      520,
      527,
      13792,
      590,
      518,
      20,
      19,
      16,
      781,
      535,
      2642,
      25960,
      63868,
      518,
      2011,
      15081,
      528,
      25960,
      63868,
      518,
      17493,
      15081,
      528,
      21369,
      13792,
      590,
      518,
      20,
      19,
      16,
      778,
      535,
      33128,
      529,
      21369,
      13792,
      590,
      518,
      2011,
      15081,
      528,
      25960,
      63868,
      518,
      17493,
      33128,
      529,
      20099,
      528,
      13792,
      590,
      518,
      2011,
      2642,
      25960,
      63868,
      518,
      20,
      21
     ],
     "markers": [
      45,
      64,
      91,
      113
     ],
     "logits_raw": [
      0.5273202657699585,
      -0.3178127408027649,
      -2.177464246749878,
      0.6166142225265503
     ],
     "probs_raw": [
      0.3861321210861206,
      0.16584382951259613,
      0.02582634799182415,
      0.4221976399421692
     ],
     "probs": [
      0.3626495897769928,
      0.19828923046588898,
      0.05252596735954285,
      0.38653525710105896
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.023885667324066162,
     "near_tie": false,
     "gold": "a",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.38653525710105896,
      "probabilities": {
       "a": 0.3626495897769928,
       "b": 0.19828923046588898,
       "c": 0.05252596735954285,
       "d": 0.38653525710105896
      }
     }
    },
    "usage": {
     "input_tokens": 143,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_053",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 343,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 343,
     "positions": 343,
     "bucket": 512,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      3036,
      1878,
      997,
      39420,
      6881,
      997,
      2797,
      12153,
      16217,
      768,
      7021,
      18479,
      3436,
      523,
      1405,
      8091,
      950,
      7050,
      811,
      42766,
      810,
      9670,
      1210,
      3436,
      875,
      12614,
      523,
      835,
      48463,
      5944,
      779,
      21008,
      810,
      15596,
      779,
      18479,
      3436,
      523,
      941,
      48463,
      5689,
      779,
      12153,
      896,
      1689,
      953,
      1943,
      9368,
      797,
      18649,
      779,
      3436,
      521,
      1203,
      896,
      1689,
      4602,
      811,
      1673,
      768,
      16763,
      988,
      779,
      7493,
      811,
      17830,
      779,
      11931,
      523,
      941,
      9888,
      43360,
      768,
      6827,
      521,
      10423,
      968,
      2059,
      9888,
      521,
      1144,
      9519,
      896,
      779,
      48463,
      11860,
      811,
      11931,
      779,
      18479,
      3436,
      988,
      779,
      12153,
      875,
      1058,
      7896,
      521,
      1873,
      523,
      941,
      4985,
      10449,
      11874,
      768,
      19505,
      41622,
      779,
      12614,
      1499,
      1014,
      2337,
      1862,
      43175,
      885,
      2620,
      779,
      48463,
      856,
      7021,
      797,
      1450,
      23956,
      811,
      4387,
      768,
      16763,
      988,
      768,
      7493,
      811,
      17830,
      779,
      11931,
      803,
      779,
      3436,
      523,
      43175,
      50380,
      521,
      779,
      48463,
      2047,
      1295,
      4516,
      811,
      5308,
      811,
      4387,
      768,
      16763,
      988,
      768,
      7493,
      1101,
      896,
      779,
      12614,
      1738,
      874,
      2524,
      1884,
      523,
      4162,
      2993,
      4493,
      521,
      779,
      12153,
      25752,
      14503,
      2723,
      779,
      48463,
      875,
      43510,
      803,
      6827,
      523,
      20463,
      803,
      779,
      2773,
      1188,
      874,
      779,
      12153,
      1090,
      31712,
      8757,
      797,
      2258,
      803,
      1210,
      4272,
      2723,
      779,
      48463,
      875,
      43510,
      803,
      6827,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      941,
      27471,
      803,
      768,
      16763,
      988,
      768,
      7493,
      953,
      1014,
      2359,
      791,
      906,
      768,
      997,
      25556,
      511,
      811,
      779,
      48463,
      1090,
      15774,
      811,
      7909,
      779,
      3436,
      523,
      20,
      19,
      16,
      794,
      535,
      6463,
      27471,
      768,
      16763,
      988,
      768,
      7493,
      953,
      768,
      4576,
      811,
      779,
      48463,
      1090,
      15774,
      811,
      7909,
      779,
      3436,
      521,
      779,
      4576,
      1753,
      874,
      4016,
      6086,
      1929,
      20,
      19,
      16,
      781,
      535,
      6463,
      27471,
      768,
      16763,
      988,
      768,
      7493,
      953,
      768,
      4576,
      811,
      779,
      48463,
      1090,
      15774,
      811,
      7909,
      779,
      3436,
      521,
      936,
      1753,
      874,
      54604,
      3449,
      988,
      779,
      20,
      19,
      16,
      778,
      535,
      941,
      48463,
      2276,
      4660,
      902,
      32056,
      16114,
      811,
      1825,
      768,
      16336,
      4516,
      811,
      4387,
      768,
      16763,
      988,
      768,
      7493,
      523,
      20,
      21
     ],
     "markers": [
      222,
      254,
      286,
      318
     ],
     "logits_raw": [
      0.5962756872177124,
      -0.33312368392944336,
      0.5012220740318298,
      -0.5879086852073669
     ],
     "probs_raw": [
      0.38312551379203796,
      0.15125441551208496,
      0.3483853042125702,
      0.11723475158214569
     ],
     "probs": [
      0.34742024540901184,
      0.17886479198932648,
      0.3246133625507355,
      0.14910155534744263
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.022806882858276367,
     "near_tie": false,
     "gold": "d",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.34742024540901184,
      "probabilities": {
       "a": 0.34742024540901184,
       "b": 0.17886479198932648,
       "c": 0.3246133625507355,
       "d": 0.14910155534744263
      }
     }
    },
    "usage": {
     "input_tokens": 343,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_054",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 109,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 109,
     "positions": 109,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      2293,
      3810,
      18166,
      1878,
      997,
      39420,
      6881,
      997,
      59061,
      931,
      779,
      12277,
      26698,
      21232,
      803,
      730,
      4613,
      35117,
      523,
      1607,
      550,
      1051,
      730,
      532,
      41562,
      736,
      527,
      570,
      511,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      730,
      527,
      523,
      10782,
      3053,
      730,
      1155,
      34193,
      2933,
      962,
      775,
      20,
      19,
      16,
      794,
      535,
      730,
      528,
      523,
      25785,
      3053,
      730,
      1155,
      34193,
      2933,
      962,
      775,
      20,
      19,
      16,
      781,
      535,
      730,
      529,
      523,
      4420,
      3053,
      730,
      1155,
      34193,
      2933,
      962,
      775,
      20,
      19,
      16,
      778,
      535,
      730,
      530,
      523,
      33052,
      3053,
      730,
      1155,
      34193,
      2933,
      962,
      775,
      20,
      21
     ],
     "markers": [
      45,
      61,
      77,
      93
     ],
     "logits_raw": [
      -3.5054848194122314,
      -2.425651788711548,
      -0.6409085988998413,
      -3.510509490966797
     ],
     "probs_raw": [
      0.044482409954071045,
      0.13096457719802856,
      0.7802935242652893,
      0.04425947368144989
     ],
     "probs": [
      0.08404666930437088,
      0.18176917731761932,
      0.6504386067390442,
      0.08374553918838501
     ],
     "argmax_index": 2,
     "argmax": "c",
     "top2_margin": 0.46866942942142487,
     "near_tie": false,
     "gold": "c",
     "correct": true,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "c",
      "confidence": 0.6504386067390442,
      "probabilities": {
       "a": 0.08404666930437088,
       "b": 0.18176917731761932,
       "c": 0.6504386067390442,
       "d": 0.08374553918838501
      }
     }
    },
    "usage": {
     "input_tokens": 109,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_055",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 169,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 169,
     "positions": 169,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      30327,
      16964,
      21167,
      1878,
      997,
      39420,
      6881,
      997,
      43211,
      853,
      3862,
      523,
      5582,
      768,
      26856,
      30174,
      1678,
      523,
      941,
      2773,
      938,
      30174,
      23301,
      875,
      779,
      5805,
      803,
      5250,
      535,
      730,
      526,
      524,
      526,
      47044,
      30174,
      730,
      1086,
      1056,
      8202,
      963,
      1058,
      1532,
      730,
      526,
      524,
      898,
      873,
      43418,
      730,
      5091,
      1056,
      8202,
      963,
      1058,
      1386,
      730,
      526,
      524,
      1968,
      873,
      43418,
      730,
      6019,
      1056,
      8202,
      963,
      1058,
      1417,
      730,
      526,
      524,
      2999,
      35791,
      963,
      1058,
      898,
      1094,
      6232,
      730,
      4291,
      1056,
      8202,
      47751,
      5582,
      779,
      903,
      8526,
      556,
      2511,
      811,
      6187,
      779,
      2536,
      803,
      1352,
      30174,
      523,
      3747,
      3685,
      1753,
      47751,
      2312,
      906,
      2780,
      803,
      11391,
      4997,
      884,
      1352,
      8292,
      8294,
      875,
      779,
      5805,
      803,
      5250,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      1058,
      22557,
      521,
      1873,
      20,
      19,
      16,
      794,
      535,
      1058,
      14671,
      521,
      1873,
      20,
      19,
      16,
      781,
      535,
      1058,
      23975,
      521,
      1873,
      20,
      19,
      16,
      778,
      535,
      730,
      2096,
      1873,
      525,
      20,
      21
     ],
     "markers": [
      133,
      142,
      151,
      160
     ],
     "logits_raw": [
      -3.6334569454193115,
      -0.35530930757522583,
      -3.424314260482788,
      -5.927770137786865
     ],
     "probs_raw": [
      0.034649983048439026,
      0.9191460013389587,
      0.04271029308438301,
      0.003493780270218849
     ],
     "probs": [
      0.07840510457754135,
      0.8153303861618042,
      0.09103896468877792,
      0.015225550159811974
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.7242914214730263,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 0.0,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.8153303861618042,
      "probabilities": {
       "a": 0.07840510457754135,
       "b": 0.8153303861618042,
       "c": 0.09103896468877792,
       "d": 0.015225550159811974
      }
     }
    },
    "usage": {
     "input_tokens": 169,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_056",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 81,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 81,
     "positions": 81,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      31116,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      856,
      14353,
      768,
      5992,
      803,
      902,
      13793,
      9147,
      9985,
      843,
      10153,
      567,
      518,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      14710,
      3890,
      35495,
      884,
      30118,
      20,
      19,
      16,
      794,
      535,
      27965,
      5168,
      28443,
      15774,
      11867,
      20,
      19,
      16,
      781,
      535,
      2315,
      34987,
      988,
      30101,
      20698,
      20,
      19,
      16,
      778,
      535,
      1087,
      2930,
      11244,
      14272,
      20,
      21
     ],
     "markers": [
      42,
      52,
      62,
      72
     ],
     "logits_raw": [
      -7.277989864349365,
      -2.838202953338623,
      -4.520140171051025,
      -0.9320415258407593
     ],
     "probs_raw": [
      0.0014887587167322636,
      0.12618252635002136,
      0.02347162738442421,
      0.8488571047782898
     ],
     "probs": [
      0.007995810359716415,
      0.1906471699476242,
      0.057337407022714615,
      0.744019627571106
     ],
     "argmax_index": 3,
     "argmax": "d",
     "top2_margin": 0.5533724576234818,
     "near_tie": false,
     "gold": "d",
     "correct": true,
     "host_softmax_max_abs_dp": 5.960464477539063e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "d",
      "confidence": 0.744019627571106,
      "probabilities": {
       "a": 0.007995810359716415,
       "b": 0.1906471699476242,
       "c": 0.057337407022714615,
       "d": 0.744019627571106
      }
     }
    },
    "usage": {
     "input_tokens": 81,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_057",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 73,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 73,
     "positions": 73,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      28009,
      19179,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      856,
      14353,
      1235,
      803,
      779,
      1209,
      9146,
      5668,
      4637,
      968,
      2025,
      1316,
      938,
      19620,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      42154,
      2644,
      20,
      19,
      16,
      794,
      535,
      26596,
      20,
      19,
      16,
      781,
      535,
      5148,
      54599,
      779,
      2681,
      20,
      19,
      16,
      778,
      535,
      5754,
      4289,
      979,
      2362,
      20,
      21
     ],
     "markers": [
      42,
      49,
      55,
      64
     ],
     "logits_raw": [
      -8.583589553833008,
      -5.527565956115723,
      -6.5222697257995605,
      -5.621074676513672
     ],
     "probs_raw": [
      0.020224157720804214,
      0.4296201169490814,
      0.15888769924640656,
      0.39126795530319214
     ],
     "probs": [
      0.044379282742738724,
      0.39378514885902405,
      0.19349488615989685,
      0.36834073066711426
     ],
     "argmax_index": 1,
     "argmax": "b",
     "top2_margin": 0.02544441819190979,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 2.9802322387695312e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "b",
      "confidence": 0.39378514885902405,
      "probabilities": {
       "a": 0.044379282742738724,
       "b": 0.39378514885902405,
       "c": 0.19349488615989685,
       "d": 0.36834073066711426
      }
     }
    },
    "usage": {
     "input_tokens": 73,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_058",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 147,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 147,
     "positions": 147,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      586,
      8988,
      21469,
      1878,
      997,
      39420,
      6881,
      997,
      3111,
      1144,
      803,
      1518,
      1547,
      21469,
      2177,
      779,
      2120,
      3338,
      843,
      18275,
      5582,
      859,
      524,
      1477,
      524,
      3462,
      518,
      1112,
      3503,
      8870,
      61611,
      7871,
      521,
      4262,
      811,
      17027,
      11241,
      8079,
      797,
      779,
      2641,
      906,
      803,
      730,
      1718,
      525,
      540,
      3108,
      31933,
      730,
      526,
      2171,
      859,
      8940,
      1727,
      6709,
      1673,
      1230,
      803,
      779,
      3622,
      1720,
      950,
      5807,
      1210,
      58110,
      523,
      3108,
      31933,
      730,
      527,
      2171,
      859,
      4925,
      1886,
      45201,
      2601,
      11468,
      797,
      779,
      3681,
      12420,
      521,
      2158,
      7250,
      953,
      963,
      779,
      12420,
      3662,
      2927,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      923,
      35736,
      521,
      923,
      35736,
      20,
      19,
      16,
      794,
      535,
      923,
      35736,
      521,
      4524,
      7871,
      20,
      19,
      16,
      781,
      535,
      4524,
      7871,
      521,
      923,
      35736,
      20,
      19,
      16,
      778,
      535,
      4524,
      7871,
      521,
      4524,
      7871,
      20,
      21
     ],
     "markers": [
      107,
      117,
      127,
      137
     ],
     "logits_raw": [
      0.6942271590232849,
      -2.231766939163208,
      -0.7739002704620361,
      -1.7253468036651611
     ],
     "probs_raw": [
      0.7283706665039062,
      0.039048947393894196,
      0.16778486967086792,
      0.06479549407958984
     ],
     "probs": [
      0.605466902256012,
      0.07487740367650986,
      0.21214313805103302,
      0.10751249641180038
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.39332376420497894,
     "near_tie": false,
     "gold": "c",
     "correct": false,
     "host_softmax_max_abs_dp": 1.4901161193847656e-08,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.605466902256012,
      "probabilities": {
       "a": 0.605466902256012,
       "b": 0.07487740367650986,
       "c": 0.21214313805103302,
       "d": 0.10751249641180038
      }
     }
    },
    "usage": {
     "input_tokens": 147,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  },
  {
   "id": "tv4_059",
   "mode": "text",
   "source": "transfer_v4_dev_head",
   "native": true,
   "prefix": 0,
   "usage_input_tokens": 115,
   "questions": [
    {
     "qid": "answer",
     "type": "choice",
     "K": 4,
     "calibrate": true,
     "temperature_key": "choice:3-5",
     "T": 1.3998981714248657,
     "prefix": 0,
     "n_ids": 115,
     "positions": 115,
     "bucket": 256,
     "ids": [
      1,
      17,
      38685,
      42505,
      6881,
      997,
      17756,
      2254,
      18849,
      1878,
      997,
      39420,
      6881,
      997,
      28828,
      803,
      779,
      2773,
      856,
      2492,
      3347,
      803,
      6448,
      18328,
      8523,
      602,
      18,
      28828,
      7832,
      16066,
      13041,
      779,
      3952,
      540,
      19,
      16,
      768,
      535,
      7817,
      938,
      1209,
      3849,
      811,
      18328,
      2203,
      896,
      856,
      880,
      27354,
      806,
      916,
      1149,
      5344,
      5812,
      7321,
      523,
      20,
      19,
      16,
      794,
      535,
      7817,
      1188,
      1014,
      18328,
      2203,
      10967,
      936,
      856,
      11611,
      797,
      3822,
      6448,
      523,
      20,
      19,
      16,
      781,
      535,
      7817,
      1052,
      1295,
      4978,
      811,
      18328,
      2203,
      896,
      1178,
      1381,
      1741,
      40927,
      523,
      20,
      19,
      16,
      778,
      535,
      7817,
      1188,
      18328,
      34045,
      2378,
      28450,
      3037,
      1209,
      11675,
      1399,
      1186,
      1188,
      8753,
      803,
      5743,
      523,
      20,
      21
     ],
     "markers": [
      35,
      58,
      76,
      94
     ],
     "logits_raw": [
      3.9577229022979736,
      -3.6439967155456543,
      -2.666415214538574,
      -0.6292939186096191
     ],
     "probs_raw": [
      0.9881318807601929,
      0.0004936622572131455,
      0.0013121648225933313,
      0.01006233412772417
     ],
     "probs": [
      0.951524019241333,
      0.004169808700680733,
      0.008382895030081272,
      0.03592329099774361
     ],
     "argmax_index": 0,
     "argmax": "a",
     "top2_margin": 0.9156007282435894,
     "near_tie": false,
     "gold": "a",
     "correct": true,
     "host_softmax_max_abs_dp": 4.656612873077393e-10,
     "batch": {
      "call": 0,
      "index": 0,
      "B": 1
     }
    }
   ],
   "response": {
    "answers": {
     "answer": {
      "type": "choice",
      "choice": "a",
      "confidence": 0.951524019241333,
      "probabilities": {
       "a": 0.951524019241333,
       "b": 0.004169808700680733,
       "c": 0.008382895030081272,
       "d": 0.03592329099774361
      }
     }
    },
    "usage": {
     "input_tokens": 115,
     "output_tokens": 0
    }
   },
   "oracle_file": "records_ref.json"
  }
 ]
}