| {"record_id": "jevbench:original-policy-01-0", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9984855055809021, "true": 0.001514469855464995}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9984855055809021, "record_seconds": 1.3813080042600632, "batch_seconds": 1.3813080042600632, "batch_records": 1, "timing_scope": "record", "brier": 4.587312105286401e-06, "nll": 0.0015156423905864358} |
| {"record_id": "jevbench:original-policy-01-1", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9987403750419617, "true": 0.0012596570886671543}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9987403750419617, "record_seconds": 0.09993188735097647, "batch_seconds": 0.09993188735097647, "batch_records": 1, "timing_scope": "record", "brier": 3.173390950905741e-06, "nll": 0.0012604189105331898} |
| {"record_id": "jevbench:original-policy-02-0", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00046279566595330834, "true": 0.9995372295379639}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9995372295379639, "record_seconds": 0.09792890585958958, "batch_seconds": 0.09792890585958958, "batch_records": 1, "timing_scope": "record", "brier": 4.2833633528971404e-07, "nll": 0.0004628775641322136} |
| {"record_id": "jevbench:original-policy-02-1", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0008430000743828714, "true": 0.9991569519042969}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9991569519042969, "record_seconds": 0.09706573095172644, "batch_seconds": 0.09706573095172644, "batch_records": 1, "timing_scope": "record", "brier": 1.421379238308873e-06, "nll": 0.0008434036863036454} |
| {"record_id": "jevbench:original-policy-03-0", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0014655775157734752, "true": 0.9985344409942627}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9985344409942627, "record_seconds": 0.09960655402392149, "batch_seconds": 0.09960655402392149, "batch_records": 1, "timing_scope": "record", "brier": 4.295780854590703e-06, "nll": 0.0014666339848190546} |
| {"record_id": "jevbench:original-policy-03-1", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.002510687569156289, "true": 0.997489333152771}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.997489333152771, "record_seconds": 0.09843722358345985, "batch_seconds": 0.09843722358345985, "batch_records": 1, "timing_scope": "record", "brier": 1.2607000826392323e-05, "nll": 0.0025138237979263067} |
| {"record_id": "jevbench:original-policy-04-0", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9954409599304199, "true": 0.004559068940579891}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9954409599304199, "record_seconds": 0.09798093419522047, "batch_seconds": 0.09798093419522047, "batch_records": 1, "timing_scope": "record", "brier": 4.156995419180021e-05, "nll": 0.004569464363157749} |
| {"record_id": "jevbench:original-policy-04-1", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9987432360649109, "true": 0.0012568174861371517}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9987432360649109, "record_seconds": 0.09859003964811563, "batch_seconds": 0.09859003964811563, "batch_records": 1, "timing_scope": "record", "brier": 3.1590457183483522e-06, "nll": 0.0012575542787089944} |
| {"record_id": "jevbench:original-policy-05-0", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9948875308036804, "true": 0.005112487357109785}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9948875308036804, "record_seconds": 0.09626028407365084, "batch_seconds": 0.09626028407365084, "batch_records": 1, "timing_scope": "record", "brier": 5.2274866902735084e-05, "nll": 0.005125582683831453} |
| {"record_id": "jevbench:original-policy-05-1", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9928157329559326, "true": 0.007184210699051619}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9928157329559326, "record_seconds": 0.09555683098733425, "batch_seconds": 0.09555683098733425, "batch_records": 1, "timing_scope": "record", "brier": 0.00010322657180950046, "nll": 0.007210198324173689} |
| {"record_id": "jevbench:original-policy-06-0", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0027746190316975117, "true": 0.9972254037857056}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9972254037857056, "record_seconds": 0.09363865200430155, "batch_seconds": 0.09363865200430155, "batch_records": 1, "timing_scope": "record", "brier": 1.5396894013974816e-05, "nll": 0.0027784525882452726} |
| {"record_id": "jevbench:original-policy-06-1", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.1639527678489685, "true": 0.8360472321510315}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8360472321510315, "record_seconds": 0.0928699653595686, "batch_seconds": 0.0928699653595686, "batch_records": 1, "timing_scope": "record", "brier": 0.05376102030277252, "nll": 0.17907017469406128} |
| {"record_id": "jevbench:original-intent-01-0", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "refund", "probabilities": {"cancel": 0.005770411808043718, "change_address": 0.00036984257167205215, "other": 0.1798495352268219, "refund": 0.8134625554084778, "status": 0.0005476485239341855}, "none_candidate_id": null, "predicted_none": false, "label": "refund", "correct": true, "confidence": 0.8134625554084778, "record_seconds": 0.09948627650737762, "batch_seconds": 0.09948627650737762, "batch_records": 1, "timing_scope": "record", "brier": 0.06717580556869507, "nll": 0.20645537972450256} |
| {"record_id": "jevbench:original-intent-01-1", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "refund", "probabilities": {"cancel": 0.0029496275819838047, "change_address": 0.0003674043982755393, "other": 0.03961862623691559, "refund": 0.95660400390625, "status": 0.0004603115958161652}, "none_candidate_id": null, "predicted_none": false, "label": "refund", "correct": true, "confidence": 0.95660400390625, "record_seconds": 0.09593535494059324, "batch_seconds": 0.09593535494059324, "batch_records": 1, "timing_scope": "record", "brier": 0.00346189527772367, "nll": 0.044365763664245605} |
| {"record_id": "jevbench:original-intent-02-0", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.0004093643219675869, "change_address": 0.012510926462709904, "other": 0.8233280777931213, "refund": 0.0006602034554816782, "status": 0.16309146583080292}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": false, "confidence": 0.8233280777931213, "record_seconds": 0.09577611275017262, "batch_seconds": 0.09577611275017262, "batch_records": 1, "timing_scope": "record", "brier": 1.3784421682357788, "nll": 1.8134441375732422} |
| {"record_id": "jevbench:original-intent-02-1", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.00028339223354123533, "change_address": 0.0005344777018763125, "other": 0.007769440766423941, "refund": 0.00031856782152317464, "status": 0.991094172000885}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.991094172000885, "record_seconds": 0.09616079460829496, "batch_seconds": 0.09616079460829496, "batch_records": 1, "timing_scope": "record", "brier": 0.00014014545013196766, "nll": 0.008945722132921219} |
| {"record_id": "jevbench:original-intent-03-0", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "change_address", "probabilities": {"cancel": 0.00017707474762573838, "change_address": 0.9939514398574829, "other": 0.0046700951643288136, "refund": 0.00021984096383675933, "status": 0.0009816226083785295}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9939514398574829, "record_seconds": 0.09721929859369993, "batch_seconds": 0.09721929859369993, "batch_records": 1, "timing_scope": "record", "brier": 5.9438138123368844e-05, "nll": 0.006066926755011082} |
| {"record_id": "jevbench:original-intent-03-1", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "change_address", "probabilities": {"cancel": 6.652054435107857e-05, "change_address": 0.9986546039581299, "other": 0.0010607001604512334, "refund": 9.432627120986581e-05, "status": 0.00012388020695652813}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9986546039581299, "record_seconds": 0.09582789801061153, "batch_seconds": 0.09582789801061153, "batch_records": 1, "timing_scope": "record", "brier": 2.9638438263646094e-06, "nll": 0.0013463018694892526} |
| {"record_id": "jevbench:original-intent-04-0", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "cancel", "probabilities": {"cancel": 0.9711443781852722, "change_address": 0.0002485477307345718, "other": 0.02657291293144226, "refund": 0.0015813872450962663, "status": 0.0004527765850070864}, "none_candidate_id": null, "predicted_none": false, "label": "cancel", "correct": true, "confidence": 0.9711443781852722, "record_seconds": 0.10118868667632341, "batch_seconds": 0.10118868667632341, "batch_records": 1, "timing_scope": "record", "brier": 0.0015415342058986425, "nll": 0.029280131682753563} |
| {"record_id": "jevbench:original-intent-04-1", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "cancel", "probabilities": {"cancel": 0.9872395396232605, "change_address": 0.00016035439330153167, "other": 0.011162963695824146, "refund": 0.0010489849373698235, "status": 0.00038818514440208673}, "none_candidate_id": null, "predicted_none": false, "label": "cancel", "correct": true, "confidence": 0.9872395396232605, "record_seconds": 0.10010680370032787, "batch_seconds": 0.10010680370032787, "batch_records": 1, "timing_scope": "record", "brier": 0.00028871785616502166, "nll": 0.012842574156820774} |
| {"record_id": "jevbench:original-intent-05-0", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.004979630000889301, "change_address": 0.00033165235072374344, "other": 0.8878247141838074, "refund": 0.10613005608320236, "status": 0.0007339264848269522}, "none_candidate_id": null, "predicted_none": false, "label": "other", "correct": true, "confidence": 0.8878247141838074, "record_seconds": 0.09791677910834551, "batch_seconds": 0.09791677910834551, "batch_records": 1, "timing_scope": "record", "brier": 0.02387232892215252, "nll": 0.11898095160722733} |
| {"record_id": "jevbench:original-intent-05-1", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.004723058082163334, "change_address": 0.0003077905857935548, "other": 0.941359281539917, "refund": 0.053083453327417374, "status": 0.0005264152423478663}, "none_candidate_id": null, "predicted_none": false, "label": "other", "correct": true, "confidence": 0.941359281539917, "record_seconds": 0.09737754799425602, "batch_seconds": 0.09737754799425602, "batch_records": 1, "timing_scope": "record", "brier": 0.006279265973716974, "nll": 0.06043040379881859} |
| {"record_id": "jevbench:original-intent-06-0", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.002207857323810458, "change_address": 0.0008924020803533494, "other": 0.05526787415146828, "refund": 0.0014262654585763812, "status": 0.9402056336402893}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.9402056336402893, "record_seconds": 0.09590517543256283, "batch_seconds": 0.09590517543256283, "batch_records": 1, "timing_scope": "record", "brier": 0.006637608632445335, "nll": 0.06165666878223419} |
| {"record_id": "jevbench:original-intent-06-1", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.000745632394682616, "change_address": 0.0006186864338815212, "other": 0.06662928313016891, "refund": 0.0012737010838463902, "status": 0.9307326674461365}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.9307326674461365, "record_seconds": 0.09833471197634935, "batch_seconds": 0.09833471197634935, "batch_records": 1, "timing_scope": "record", "brier": 0.009239986538887024, "nll": 0.07178318500518799} |
| {"record_id": "jevbench:original-ordinal-01-0", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.9069372415542603, "1": 0.08798912167549133, "2": 0.004743523430079222, "3": 0.0003301222459413111}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.9069372415542603, "record_seconds": 0.09807910490781069, "batch_seconds": 0.09807910490781069, "batch_records": 1, "timing_scope": "record", "brier": 0.016425373032689095, "nll": 0.09768202155828476} |
| {"record_id": "jevbench:original-ordinal-01-1", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.9621517062187195, "1": 0.036195944994688034, "2": 0.0014030198799446225, "3": 0.0002493661595508456}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.9621517062187195, "record_seconds": 0.09467122051864862, "batch_seconds": 0.09467122051864862, "batch_records": 1, "timing_scope": "record", "brier": 0.002744670258834958, "nll": 0.038583140820264816} |
| {"record_id": "jevbench:original-ordinal-02-0", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "1", "probabilities": {"0": 0.005515758413821459, "1": 0.9491519927978516, "2": 0.044829364866018295, "3": 0.0005028908490203321}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": true, "confidence": 0.9491519927978516, "record_seconds": 0.09466544352471828, "batch_seconds": 0.09466544352471828, "batch_records": 1, "timing_scope": "record", "brier": 0.004625868052244186, "nll": 0.052186332643032074} |
| {"record_id": "jevbench:original-ordinal-02-1", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "1", "probabilities": {"0": 0.0029711895622313023, "1": 0.9944208264350891, "2": 0.0020151834469288588, "3": 0.0005928423488512635}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": true, "confidence": 0.9944208264350891, "record_seconds": 0.09447807632386684, "batch_seconds": 0.09447807632386684, "batch_records": 1, "timing_scope": "record", "brier": 4.436756717041135e-05, "nll": 0.005594795104116201} |
| {"record_id": "jevbench:original-ordinal-03-0", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.006719101220369339, "1": 0.013855854980647564, "2": 0.9789708256721497, "3": 0.0004541864909697324}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9789708256721497, "record_seconds": 0.09909789077937603, "batch_seconds": 0.09909789077937603, "batch_records": 1, "timing_scope": "record", "brier": 0.0006795634981244802, "nll": 0.02125343680381775} |
| {"record_id": "jevbench:original-ordinal-03-1", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.0038381339982151985, "1": 0.00711423996835947, "2": 0.9885244965553284, "3": 0.0005231441464275122}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9885244965553284, "record_seconds": 0.09473077300935984, "batch_seconds": 0.09473077300935984, "batch_records": 1, "timing_scope": "record", "brier": 0.00019730454368982464, "nll": 0.01154185552150011} |
| {"record_id": "jevbench:original-ordinal-04-0", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "3", "probabilities": {"0": 0.000596384983509779, "1": 0.001772022806107998, "2": 0.0021136163268238306, "3": 0.9955180287361145}, "none_candidate_id": null, "predicted_none": false, "label": "3", "correct": true, "confidence": 0.9955180287361145, "record_seconds": 0.09386378712952137, "batch_seconds": 0.09386378712952137, "batch_records": 1, "timing_scope": "record", "brier": 2.8051179469912313e-05, "nll": 0.004492045380175114} |
| {"record_id": "jevbench:original-ordinal-04-1", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "3", "probabilities": {"0": 0.0004686632310040295, "1": 0.0014648206997662783, "2": 0.0018578465096652508, "3": 0.996208667755127}, "none_candidate_id": null, "predicted_none": false, "label": "3", "correct": true, "confidence": 0.996208667755127, "record_seconds": 0.09416941273957491, "batch_seconds": 0.09416941273957491, "batch_records": 1, "timing_scope": "record", "brier": 2.0191138901282102e-05, "nll": 0.003798537654802203} |
| {"record_id": "jevbench:original-ordinal-05-0", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.9425404071807861, "1": 0.05053132772445679, "2": 0.0047624120488762856, "3": 0.0021659005433321}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.9425404071807861, "record_seconds": 0.09398620296269655, "batch_seconds": 0.09398620296269655, "batch_records": 1, "timing_scope": "record", "brier": 0.005882391706109047, "nll": 0.0591764897108078} |
| {"record_id": "jevbench:original-ordinal-05-1", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.9743841886520386, "1": 0.011417079716920853, "2": 0.012705789878964424, "3": 0.001492929644882679}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.9743841886520386, "record_seconds": 0.09364635962992907, "batch_seconds": 0.09364635962992907, "batch_records": 1, "timing_scope": "record", "brier": 0.0009501854074187577, "nll": 0.02594960853457451} |
| {"record_id": "jevbench:original-ordinal-06-0", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.0041262307204306126, "1": 0.0051200492307543755, "2": 0.9904776811599731, "3": 0.00027602017507888377}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9904776811599731, "record_seconds": 0.09439940564334393, "batch_seconds": 0.09439940564334393, "batch_records": 1, "timing_scope": "record", "brier": 0.0001339914306299761, "nll": 0.009567946195602417} |
| {"record_id": "jevbench:original-ordinal-06-1", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.0007202672422863543, "1": 0.001221069018356502, "2": 0.9978796243667603, "3": 0.00017910377937369049}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9978796243667603, "record_seconds": 0.10048614908009768, "batch_seconds": 0.10048614908009768, "batch_records": 1, "timing_scope": "record", "brier": 6.537865829159273e-06, "nll": 0.0021226268727332354} |
| {"record_id": "jevbench:original-extraction-01-0", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "pickup", "probabilities": {"courier": 0.07973848283290863, "pickup": 0.4920087158679962, "post": 0.06746464222669601, "unknown": 0.36078816652297974}, "none_candidate_id": null, "predicted_none": false, "label": "pickup", "correct": true, "confidence": 0.4920087158679962, "record_seconds": 0.09471116401255131, "batch_seconds": 0.09471116401255131, "batch_records": 1, "timing_scope": "record", "brier": 0.39913296699523926, "nll": 0.7092588543891907} |
| {"record_id": "jevbench:original-extraction-01-1", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.013369367457926273, "pickup": 0.15290634334087372, "post": 0.07921968400478363, "unknown": 0.7545046210289001}, "none_candidate_id": null, "predicted_none": false, "label": "pickup", "correct": false, "confidence": 0.7545046210289001, "record_seconds": 0.09547523409128189, "batch_seconds": 0.09547523409128189, "batch_records": 1, "timing_scope": "record", "brier": 1.2932994365692139, "nll": 1.8779296875} |
| {"record_id": "jevbench:original-extraction-02-0", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.0008816977497190237, "pickup": 0.0008721789345145226, "post": 0.01271247211843729, "unknown": 0.9855336546897888}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9855336546897888, "record_seconds": 0.09977912530303001, "batch_seconds": 0.09977912530303001, "batch_records": 1, "timing_scope": "record", "brier": 0.0003724201815202832, "nll": 0.014572002924978733} |
| {"record_id": "jevbench:original-extraction-02-1", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.004350361879914999, "pickup": 0.003728343639522791, "post": 0.00994954351335764, "unknown": 0.9819716811180115}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9819716811180115, "record_seconds": 0.09681890811771154, "batch_seconds": 0.09681890811771154, "batch_records": 1, "timing_scope": "record", "brier": 0.00045683991629630327, "nll": 0.01819280907511711} |
| {"record_id": "jevbench:original-extraction-03-0", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9548456072807312, "pickup": 0.01025948766618967, "post": 0.005538697820156813, "unknown": 0.02935619279742241}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9548456072807312, "record_seconds": 0.09568748995661736, "batch_seconds": 0.09568748995661736, "batch_records": 1, "timing_scope": "record", "brier": 0.0030366394203156233, "nll": 0.04620561748743057} |
| {"record_id": "jevbench:original-extraction-03-1", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9955267310142517, "pickup": 0.0007791799143888056, "post": 0.0023811794817447662, "unknown": 0.0013128345599398017}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9955267310142517, "record_seconds": 0.09950458165258169, "batch_seconds": 0.09950458165258169, "batch_records": 1, "timing_scope": "record", "brier": 2.801080518111121e-05, "nll": 0.004483303986489773} |
| {"record_id": "jevbench:original-extraction-04-0", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "post", "probabilities": {"courier": 0.00030044952291063964, "pickup": 0.000136725720949471, "post": 0.9990531802177429, "unknown": 0.0005096640670672059}, "none_candidate_id": null, "predicted_none": false, "label": "post", "correct": true, "confidence": 0.9990531802177429, "record_seconds": 0.09640215337276459, "batch_seconds": 0.09640215337276459, "batch_records": 1, "timing_scope": "record", "brier": 1.2651890983761405e-06, "nll": 0.0009472682722844183} |
| {"record_id": "jevbench:original-extraction-04-1", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "post", "probabilities": {"courier": 0.0002985874598380178, "pickup": 0.00014078832464292645, "post": 0.9990274906158447, "unknown": 0.0005331981228664517}, "none_candidate_id": null, "predicted_none": false, "label": "post", "correct": true, "confidence": 0.9990274906158447, "record_seconds": 0.09477932658046484, "batch_seconds": 0.09477932658046484, "batch_records": 1, "timing_scope": "record", "brier": 1.3390505273491726e-06, "nll": 0.000972982554230839} |
| {"record_id": "jevbench:original-extraction-05-0", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.0007810601382516325, "pickup": 0.005585331469774246, "post": 0.006547011900693178, "unknown": 0.9870865345001221}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9870865345001221, "record_seconds": 0.0945571269840002, "batch_seconds": 0.0945571269840002, "batch_records": 1, "timing_scope": "record", "brier": 0.00024142692564055324, "nll": 0.012997569516301155} |
| {"record_id": "jevbench:original-extraction-05-1", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.0027425282169133425, "pickup": 0.019140515476465225, "post": 0.014788559637963772, "unknown": 0.9633283019065857}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9633283019065857, "record_seconds": 0.09474862087517977, "batch_seconds": 0.09474862087517977, "batch_records": 1, "timing_scope": "record", "brier": 0.0019373956602066755, "nll": 0.037361010909080505} |
| {"record_id": "jevbench:original-extraction-06-0", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9995028972625732, "pickup": 6.814963853685185e-05, "post": 0.00016549650172237307, "unknown": 0.00026339292526245117}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9995028972625732, "record_seconds": 0.0938554760068655, "batch_seconds": 0.0938554760068655, "batch_records": 1, "timing_scope": "record", "brier": 3.485204160824651e-07, "nll": 0.0004972263122908771} |
| {"record_id": "jevbench:original-extraction-06-1", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9978680610656738, "pickup": 0.0002771036233752966, "post": 0.0005262523191049695, "unknown": 0.001328574842773378}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9978680610656738, "record_seconds": 0.09394254814833403, "batch_seconds": 0.09394254814833403, "batch_records": 1, "timing_scope": "record", "brier": 6.664002285106108e-06, "nll": 0.0021342148538678885} |
| {"record_id": "jevbench:original-adequacy-01-0", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.018592849373817444, "true": 0.9814071655273438}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9814071655273438, "record_seconds": 0.09743140917271376, "batch_seconds": 0.09743140917271376, "batch_records": 1, "timing_scope": "record", "brier": 0.0006913875695317984, "nll": 0.01876785419881344} |
| {"record_id": "jevbench:original-adequacy-01-1", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.01801377534866333, "true": 0.9819862842559814}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9819862842559814, "record_seconds": 0.09208573680371046, "batch_seconds": 0.09208573680371046, "batch_records": 1, "timing_scope": "record", "brier": 0.0006489900406450033, "nll": 0.01817793771624565} |
| {"record_id": "jevbench:original-adequacy-02-0", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9983678460121155, "true": 0.0016321500297635794}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9983678460121155, "record_seconds": 0.09287840221077204, "batch_seconds": 0.09287840221077204, "batch_records": 1, "timing_scope": "record", "brier": 5.327839971869253e-06, "nll": 0.0016334874089807272} |
| {"record_id": "jevbench:original-adequacy-02-1", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9979181885719299, "true": 0.0020818777848035097}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9979181885719299, "record_seconds": 0.09165714122354984, "batch_seconds": 0.09165714122354984, "batch_records": 1, "timing_scope": "record", "brier": 8.668153895996511e-06, "nll": 0.0020839814096689224} |
| {"record_id": "jevbench:original-adequacy-03-0", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9811999201774597, "true": 0.018800031393766403}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9811999201774597, "record_seconds": 0.09231115318834782, "batch_seconds": 0.09231115318834782, "batch_records": 1, "timing_scope": "record", "brier": 0.0007068841950967908, "nll": 0.01897904835641384} |
| {"record_id": "jevbench:original-adequacy-03-1", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.15529592335224152, "true": 0.8447041511535645}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.8447041511535645, "record_seconds": 0.09252635203301907, "batch_seconds": 0.09252635203301907, "batch_records": 1, "timing_scope": "record", "brier": 1.4270501136779785, "nll": 1.8624228239059448} |
| {"record_id": "jevbench:original-adequacy-04-0", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.010644986294209957, "true": 0.9893550872802734}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9893550872802734, "record_seconds": 0.09349158126860857, "batch_seconds": 0.09349158126860857, "batch_records": 1, "timing_scope": "record", "brier": 0.00022662989795207977, "nll": 0.010701974853873253} |
| {"record_id": "jevbench:original-adequacy-04-1", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.015104885213077068, "true": 0.9848951697349548}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9848951697349548, "record_seconds": 0.0953884581103921, "batch_seconds": 0.0953884581103921, "batch_records": 1, "timing_scope": "record", "brier": 0.0004563134571071714, "nll": 0.015220070257782936} |
| {"record_id": "jevbench:original-adequacy-05-0", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8284814357757568, "true": 0.17151857912540436}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.8284814357757568, "record_seconds": 0.09460048843175173, "batch_seconds": 0.09460048843175173, "batch_records": 1, "timing_scope": "record", "brier": 0.05883724242448807, "nll": 0.18816085159778595} |
| {"record_id": "jevbench:original-adequacy-05-1", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9791558384895325, "true": 0.02084413357079029}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9791558384895325, "record_seconds": 0.09398901462554932, "batch_seconds": 0.09398901462554932, "batch_records": 1, "timing_scope": "record", "brier": 0.0008689569658599794, "nll": 0.02106446772813797} |
| {"record_id": "jevbench:original-adequacy-06-0", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.003604758996516466, "true": 0.9963952898979187}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9963952898979187, "record_seconds": 0.0941674206405878, "batch_seconds": 0.0941674206405878, "batch_records": 1, "timing_scope": "record", "brier": 2.5988221750594676e-05, "nll": 0.003611222840845585} |
| {"record_id": "jevbench:original-adequacy-06-1", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0775633305311203, "true": 0.9224367141723633}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9224367141723633, "record_seconds": 0.09483137633651495, "batch_seconds": 0.09483137633651495, "batch_records": 1, "timing_scope": "record", "brier": 0.012032133527100086, "nll": 0.08073651045560837} |
| {"record_id": "jevbench:original-routing-01-0", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "math", "probabilities": {"coding": 0.004166068509221077, "coding_agent": 0.0006556396256200969, "document": 0.001939342706464231, "general": 0.01307486928999424, "math": 0.9787923693656921, "tools": 0.001371602644212544}, "none_candidate_id": null, "predicted_none": false, "label": "math", "correct": true, "confidence": 0.9787923693656921, "record_seconds": 0.12460437323898077, "batch_seconds": 0.12460437323898077, "batch_records": 1, "timing_scope": "record", "brier": 0.0006441441364586353, "nll": 0.02143574319779873} |
| {"record_id": "jevbench:original-routing-01-1", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "math", "probabilities": {"coding": 0.006996527314186096, "coding_agent": 0.0008699778118170798, "document": 0.0025984083767980337, "general": 0.01122293621301651, "math": 0.976285457611084, "tools": 0.002026729052886367}, "none_candidate_id": null, "predicted_none": false, "label": "math", "correct": true, "confidence": 0.976285457611084, "record_seconds": 0.11663827858865261, "batch_seconds": 0.11663827858865261, "batch_records": 1, "timing_scope": "record", "brier": 0.0007489015115424991, "nll": 0.024000259116292} |
| {"record_id": "jevbench:original-routing-02-0", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding", "probabilities": {"coding": 0.9013412594795227, "coding_agent": 0.08789817243814468, "document": 0.0008946412126533687, "general": 0.008686990477144718, "math": 0.00015265228284988552, "tools": 0.0010262904688715935}, "none_candidate_id": null, "predicted_none": false, "label": "coding", "correct": true, "confidence": 0.9013412594795227, "record_seconds": 0.12172336783260107, "batch_seconds": 0.12172336783260107, "batch_records": 1, "timing_scope": "record", "brier": 0.01753697730600834, "nll": 0.10387133806943893} |
| {"record_id": "jevbench:original-routing-02-1", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding", "probabilities": {"coding": 0.6910521984100342, "coding_agent": 0.2919962406158447, "document": 0.0016557397320866585, "general": 0.01323168259114027, "math": 0.00012145473738200963, "tools": 0.0019427364459261298}, "none_candidate_id": null, "predicted_none": false, "label": "coding", "correct": true, "confidence": 0.6910521984100342, "record_seconds": 0.1150245051831007, "batch_seconds": 0.1150245051831007, "batch_records": 1, "timing_scope": "record", "brier": 0.18089215457439423, "nll": 0.36953991651535034} |
| {"record_id": "jevbench:original-routing-03-0", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding_agent", "probabilities": {"coding": 0.0003206115507055074, "coding_agent": 0.997316300868988, "document": 0.00045708936522714794, "general": 0.001298577175475657, "math": 5.2078248700127006e-05, "tools": 0.0005552904331125319}, "none_candidate_id": null, "predicted_none": false, "label": "coding_agent", "correct": true, "confidence": 0.997316300868988, "record_seconds": 0.12063005100935698, "batch_seconds": 0.12063005100935698, "batch_records": 1, "timing_scope": "record", "brier": 9.511325515632052e-06, "nll": 0.002687306609004736} |
| {"record_id": "jevbench:original-routing-03-1", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding_agent", "probabilities": {"coding": 0.00013398550800047815, "coding_agent": 0.9981297850608826, "document": 0.00040612471639178693, "general": 0.0009984718635678291, "math": 3.449246287345886e-05, "tools": 0.00029718727455474436}, "none_candidate_id": null, "predicted_none": false, "label": "coding_agent", "correct": true, "confidence": 0.9981297850608826, "record_seconds": 0.12300171982496977, "batch_seconds": 0.12300171982496977, "batch_records": 1, "timing_scope": "record", "brier": 4.767049176734872e-06, "nll": 0.0018719659419730306} |
| {"record_id": "jevbench:original-routing-04-0", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.00835792999714613, "coding_agent": 0.015060987323522568, "document": 0.15530458092689514, "general": 0.6891046166419983, "math": 0.001173296826891601, "tools": 0.1309986263513565}, "none_candidate_id": null, "predicted_none": false, "label": "document", "correct": false, "confidence": 0.6891046166419983, "record_seconds": 0.1167960362508893, "batch_seconds": 0.1167960362508893, "batch_records": 1, "timing_scope": "record", "brier": 1.2058342695236206, "nll": 1.862367033958435} |
| {"record_id": "jevbench:original-routing-04-1", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.14041973650455475, "coding_agent": 0.019933264702558517, "document": 0.16282270848751068, "general": 0.5972882509231567, "math": 0.0009472452802583575, "tools": 0.07858873158693314}, "none_candidate_id": null, "predicted_none": false, "label": "document", "correct": false, "confidence": 0.5972882509231567, "record_seconds": 0.11558501981198788, "batch_seconds": 0.11558501981198788, "batch_records": 1, "timing_scope": "record", "brier": 1.0839111804962158, "nll": 1.8150933980941772} |
| {"record_id": "jevbench:original-routing-05-0", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "tools", "probabilities": {"coding": 0.0016281695570796728, "coding_agent": 0.0011078374227508903, "document": 0.01282931212335825, "general": 0.26497501134872437, "math": 0.0002433932531857863, "tools": 0.7192163467407227}, "none_candidate_id": null, "predicted_none": false, "label": "tools", "correct": true, "confidence": 0.7192163467407227, "record_seconds": 0.11799859441816807, "batch_seconds": 0.11799859441816807, "batch_records": 1, "timing_scope": "record", "brier": 0.14921975135803223, "nll": 0.32959306240081787} |
| {"record_id": "jevbench:original-routing-05-1", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "tools", "probabilities": {"coding": 0.00275472947396338, "coding_agent": 0.0017068490851670504, "document": 0.019382884725928307, "general": 0.4766019880771637, "math": 0.000328624970279634, "tools": 0.4992249011993408}, "none_candidate_id": null, "predicted_none": false, "label": "tools", "correct": true, "confidence": 0.4992249011993408, "record_seconds": 0.11775056552141905, "batch_seconds": 0.11775056552141905, "batch_records": 1, "timing_scope": "record", "brier": 0.4783114492893219, "nll": 0.6946985721588135} |
| {"record_id": "jevbench:original-routing-06-0", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.004436448682099581, "coding_agent": 0.0038950652815401554, "document": 0.0400475338101387, "general": 0.9239072203636169, "math": 0.0005969765479676425, "tools": 0.027116814628243446}, "none_candidate_id": null, "predicted_none": false, "label": "general", "correct": true, "confidence": 0.9239072203636169, "record_seconds": 0.11646105069667101, "batch_seconds": 0.11646105069667101, "batch_records": 1, "timing_scope": "record", "brier": 0.008164447732269764, "nll": 0.07914362102746964} |
| {"record_id": "jevbench:original-routing-06-1", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.00517650181427598, "coding_agent": 0.0031933970749378204, "document": 0.030980687588453293, "general": 0.9242287874221802, "math": 0.0005328765255399048, "tools": 0.03588784858584404}, "none_candidate_id": null, "predicted_none": false, "label": "general", "correct": true, "confidence": 0.9242287874221802, "record_seconds": 0.1150888567790389, "batch_seconds": 0.1150888567790389, "batch_records": 1, "timing_scope": "record", "brier": 0.008026295341551304, "nll": 0.07879563421010971} |
| {"record_id": "jevbench:easy-intent-00", "group_id": "jevbench:easy-intent-00", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "track_order", "probabilities": {"billing_question": 0.00023506981960963458, "cancel_order": 0.0007278815028257668, "change_address": 0.00027819344541057944, "report_damage": 0.00014160694263409823, "track_order": 0.9986171722412109}, "none_candidate_id": null, "predicted_none": false, "label": "track_order", "correct": true, "confidence": 0.9986171722412109, "record_seconds": 0.09916801005601883, "batch_seconds": 0.09916801005601883, "batch_records": 1, "timing_scope": "record", "brier": 2.594726083771093e-06, "nll": 0.00138378469273448} |
| {"record_id": "jevbench:easy-intent-01", "group_id": "jevbench:easy-intent-01", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "cancel_order", "probabilities": {"billing_question": 6.286358257057145e-05, "cancel_order": 0.9997690320014954, "change_address": 3.5275785194244236e-05, "report_damage": 4.954783071298152e-05, "track_order": 8.323402289533988e-05}, "none_candidate_id": null, "predicted_none": false, "label": "cancel_order", "correct": true, "confidence": 0.9997690320014954, "record_seconds": 0.09908374212682247, "batch_seconds": 0.09908374212682247, "batch_records": 1, "timing_scope": "record", "brier": 6.792532047938948e-08, "nll": 0.00023099467216525227} |
| {"record_id": "jevbench:easy-intent-02", "group_id": "jevbench:easy-intent-02", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "change_address", "probabilities": {"billing_question": 7.414181891363114e-05, "cancel_order": 9.513608529232442e-05, "change_address": 0.9994128942489624, "report_damage": 6.356302765198052e-05, "track_order": 0.0003542599151842296}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9994128942489624, "record_seconds": 0.10268156416714191, "batch_seconds": 0.10268156416714191, "batch_records": 1, "timing_scope": "record", "brier": 4.887813815912523e-07, "nll": 0.0005872781621292233} |
| {"record_id": "jevbench:easy-intent-03", "group_id": "jevbench:easy-intent-03", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "report_damage", "probabilities": {"billing_question": 0.00013116018089931458, "cancel_order": 0.0013751218793913722, "change_address": 0.00017607166955713183, "report_damage": 0.9978836178779602, "track_order": 0.00043403790914453566}, "none_candidate_id": null, "predicted_none": false, "label": "report_damage", "correct": true, "confidence": 0.9978836178779602, "record_seconds": 0.0939539186656475, "batch_seconds": 0.0939539186656475, "batch_records": 1, "timing_scope": "record", "brier": 6.606626811844762e-06, "nll": 0.002118624746799469} |
| {"record_id": "jevbench:easy-intent-04", "group_id": "jevbench:easy-intent-04", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "billing_question", "probabilities": {"billing_question": 0.9948673248291016, "cancel_order": 0.00232583936303854, "change_address": 0.0004147646832279861, "report_damage": 0.0003659023786894977, "track_order": 0.002026206813752651}, "none_candidate_id": null, "predicted_none": false, "label": "billing_question", "correct": true, "confidence": 0.9948673248291016, "record_seconds": 0.09532784204930067, "batch_seconds": 0.09532784204930067, "batch_records": 1, "timing_scope": "record", "brier": 3.616531103034504e-05, "nll": 0.00514589250087738} |
| {"record_id": "jevbench:easy-intent-05", "group_id": "jevbench:easy-intent-05", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "track_order", "probabilities": {"billing_question": 0.0004296795523259789, "cancel_order": 0.0008464027778245509, "change_address": 0.000594735553022474, "report_damage": 0.00028298268443904817, "track_order": 0.997846245765686}, "none_candidate_id": null, "predicted_none": false, "label": "track_order", "correct": true, "confidence": 0.997846245765686, "record_seconds": 0.0941960895434022, "batch_seconds": 0.0941960895434022, "batch_records": 1, "timing_scope": "record", "brier": 5.973468887532363e-06, "nll": 0.002156076952815056} |
| {"record_id": "jevbench:easy-intent-06", "group_id": "jevbench:easy-intent-06", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "set_alarm", "probabilities": {"play_music": 0.0001598826638655737, "send_message": 0.0006884847534820437, "set_alarm": 0.9988538026809692, "turn_off_lights": 0.00015220245404634625, "weather": 0.00014566522440873086}, "none_candidate_id": null, "predicted_none": false, "label": "set_alarm", "correct": true, "confidence": 0.9988538026809692, "record_seconds": 0.09438726026564837, "batch_seconds": 0.09438726026564837, "batch_records": 1, "timing_scope": "record", "brier": 1.8577260334495804e-06, "nll": 0.0011468547163531184} |
| {"record_id": "jevbench:easy-intent-07", "group_id": "jevbench:easy-intent-07", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "play_music", "probabilities": {"play_music": 0.9993081092834473, "send_message": 0.000389132765121758, "set_alarm": 9.288371074944735e-05, "turn_off_lights": 0.00010469283006386831, "weather": 0.00010519583884160966}, "none_candidate_id": null, "predicted_none": false, "label": "play_music", "correct": true, "confidence": 0.9993081092834473, "record_seconds": 0.09406627342104912, "batch_seconds": 0.09406627342104912, "batch_records": 1, "timing_scope": "record", "brier": 6.607912155232043e-07, "nll": 0.0006921301828697324} |
| {"record_id": "jevbench:easy-intent-08", "group_id": "jevbench:easy-intent-08", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "weather", "probabilities": {"play_music": 0.0001080881993402727, "send_message": 0.00023522379342466593, "set_alarm": 0.00017267603834625334, "turn_off_lights": 8.504126890329644e-05, "weather": 0.9993990659713745}, "none_candidate_id": null, "predicted_none": false, "label": "weather", "correct": true, "confidence": 0.9993990659713745, "record_seconds": 0.0945455776527524, "batch_seconds": 0.0945455776527524, "batch_records": 1, "timing_scope": "record", "brier": 4.651840015412745e-07, "nll": 0.0006011146469973028} |
| {"record_id": "jevbench:easy-intent-09", "group_id": "jevbench:easy-intent-09", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "send_message", "probabilities": {"play_music": 0.0002439607196720317, "send_message": 0.9984524250030518, "set_alarm": 0.0009135322761721909, "turn_off_lights": 0.00015629350673407316, "weather": 0.00023379968479275703}, "none_candidate_id": null, "predicted_none": false, "label": "send_message", "correct": true, "confidence": 0.9984524250030518, "record_seconds": 0.09439818281680346, "batch_seconds": 0.09439818281680346, "batch_records": 1, "timing_scope": "record", "brier": 3.368136276549194e-06, "nll": 0.0015487737255170941} |
| {"record_id": "jevbench:easy-intent-10", "group_id": "jevbench:easy-intent-10", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "turn_off_lights", "probabilities": {"play_music": 0.00021021149586886168, "send_message": 0.00036341763916425407, "set_alarm": 0.00013705099991057068, "turn_off_lights": 0.9991747736930847, "weather": 0.00011450023885117844}, "none_candidate_id": null, "predicted_none": false, "label": "turn_off_lights", "correct": true, "confidence": 0.9991747736930847, "record_seconds": 0.09410795662552118, "batch_seconds": 0.09410795662552118, "batch_records": 1, "timing_scope": "record", "brier": 8.891530569599126e-07, "nll": 0.0008255669963546097} |
| {"record_id": "jevbench:easy-intent-11", "group_id": "jevbench:easy-intent-11", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "set_alarm", "probabilities": {"play_music": 4.705490209744312e-05, "send_message": 8.129831257974729e-05, "set_alarm": 0.9997976422309875, "turn_off_lights": 3.4992121072718874e-05, "weather": 3.9034930523484945e-05}, "none_candidate_id": null, "predicted_none": false, "label": "set_alarm", "correct": true, "confidence": 0.9997976422309875, "record_seconds": 0.0952396085485816, "batch_seconds": 0.0952396085485816, "batch_records": 1, "timing_scope": "record", "brier": 5.2520420013024705e-08, "nll": 0.00020237824355717748} |
| {"record_id": "jevbench:easy-fact-00", "group_id": "jevbench:easy-fact-00", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00021680227655451745, "true": 0.9997832179069519}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9997832179069519, "record_seconds": 0.09423449821770191, "batch_seconds": 0.09423449821770191, "batch_records": 1, "timing_scope": "record", "brier": 9.399769851370365e-08, "nll": 0.00021680559439118952} |
| {"record_id": "jevbench:easy-fact-01", "group_id": "jevbench:easy-fact-01", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9988712668418884, "true": 0.0011287190718576312}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9988712668418884, "record_seconds": 0.09327857568860054, "batch_seconds": 0.09327857568860054, "batch_records": 1, "timing_scope": "record", "brier": 2.5480453587078955e-06, "nll": 0.0011293706484138966} |
| {"record_id": "jevbench:easy-fact-02", "group_id": "jevbench:easy-fact-02", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00018087711941916496, "true": 0.9998190999031067}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9998190999031067, "record_seconds": 0.09475950431078672, "batch_seconds": 0.09475950431078672, "batch_records": 1, "timing_scope": "record", "brier": 6.54413767620099e-08, "nll": 0.00018091646779794246} |
| {"record_id": "jevbench:easy-fact-03", "group_id": "jevbench:easy-fact-03", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9977151155471802, "true": 0.0022849501110613346}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9977151155471802, "record_seconds": 0.09596831537783146, "batch_seconds": 0.09596831537783146, "batch_records": 1, "timing_scope": "record", "brier": 1.044169403030537e-05, "nll": 0.00228749867528677} |
| {"record_id": "jevbench:easy-fact-04", "group_id": "jevbench:easy-fact-04", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 7.09780360921286e-05, "true": 0.9999290704727173}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9999290704727173, "record_seconds": 0.09865301847457886, "batch_seconds": 0.09865301847457886, "batch_records": 1, "timing_scope": "record", "brier": 1.0068879063851455e-08, "nll": 7.093204476404935e-05} |
| {"record_id": "jevbench:easy-fact-05", "group_id": "jevbench:easy-fact-05", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9988598823547363, "true": 0.0011401850497350097}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9988598823547363, "record_seconds": 0.09796582069247961, "batch_seconds": 0.09796582069247961, "batch_records": 1, "timing_scope": "record", "brier": 2.5998901946877595e-06, "nll": 0.001140768057666719} |
| {"record_id": "jevbench:easy-fact-06", "group_id": "jevbench:easy-fact-06", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00012463696475606412, "true": 0.9998753070831299}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9998753070831299, "record_seconds": 0.09767782315611839, "batch_seconds": 0.09767782315611839, "batch_records": 1, "timing_scope": "record", "brier": 3.108269908125294e-08, "nll": 0.00012470068759284914} |
| {"record_id": "jevbench:easy-fact-07", "group_id": "jevbench:easy-fact-07", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9989597797393799, "true": 0.0010402482002973557}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9989597797393799, "record_seconds": 0.09728781133890152, "batch_seconds": 0.09728781133890152, "batch_records": 1, "timing_scope": "record", "brier": 2.164174475183245e-06, "nll": 0.0010407617082819343} |
| {"record_id": "jevbench:easy-fact-08", "group_id": "jevbench:easy-fact-08", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 5.801310180686414e-05, "true": 0.9999419450759888}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9999419450759888, "record_seconds": 0.09527972713112831, "batch_seconds": 0.09527972713112831, "batch_records": 1, "timing_scope": "record", "brier": 6.735894508835827e-09, "nll": 5.805660839541815e-05} |
| {"record_id": "jevbench:easy-fact-09", "group_id": "jevbench:easy-fact-09", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9988223910331726, "true": 0.0011775670573115349}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9988223910331726, "record_seconds": 0.09574948344379663, "batch_seconds": 0.09574948344379663, "batch_records": 1, "timing_scope": "record", "brier": 2.7734270133805694e-06, "nll": 0.001178302918560803} |
| {"record_id": "jevbench:easy-fact-10", "group_id": "jevbench:easy-fact-10", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00017742645286489278, "true": 0.9998225569725037}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9998225569725037, "record_seconds": 0.09593365248292685, "batch_seconds": 0.09593365248292685, "batch_records": 1, "timing_scope": "record", "brier": 6.296617272028016e-08, "nll": 0.00017745877266861498} |
| {"record_id": "jevbench:easy-fact-11", "group_id": "jevbench:easy-fact-11", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.998688280582428, "true": 0.0013117040507495403}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.998688280582428, "record_seconds": 0.09840312134474516, "batch_seconds": 0.09840312134474516, "batch_records": 1, "timing_scope": "record", "brier": 3.4411755223118234e-06, "nll": 0.0013125804252922535} |
| {"record_id": "jevbench:easy-extraction-00", "group_id": "jevbench:easy-extraction-00", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "paypal", "probabilities": {"bank_transfer": 5.327923645381816e-05, "cash": 1.907883597596083e-05, "credit_card": 9.259332728106529e-05, "paypal": 0.9998350143432617}, "none_candidate_id": null, "predicted_none": false, "label": "paypal", "correct": true, "confidence": 0.9998350143432617, "record_seconds": 0.0939558083191514, "batch_seconds": 0.0939558083191514, "batch_records": 1, "timing_scope": "record", "brier": 3.899646827676406e-08, "nll": 0.00016499926277901977} |
| {"record_id": "jevbench:easy-extraction-01", "group_id": "jevbench:easy-extraction-01", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "cash", "probabilities": {"bank_transfer": 0.00016858824528753757, "cash": 0.9995042085647583, "credit_card": 0.00016805338964331895, "paypal": 0.00015915861877147108}, "none_candidate_id": null, "predicted_none": false, "label": "cash", "correct": true, "confidence": 0.9995042085647583, "record_seconds": 0.09410236869007349, "batch_seconds": 0.09410236869007349, "batch_records": 1, "timing_scope": "record", "brier": 3.2780454262137937e-07, "nll": 0.0004959143698215485} |
| {"record_id": "jevbench:easy-extraction-02", "group_id": "jevbench:easy-extraction-02", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "bank_transfer", "probabilities": {"bank_transfer": 0.999288022518158, "cash": 0.0001222447754116729, "credit_card": 0.0002993351372424513, "paypal": 0.0002903965942095965}, "none_candidate_id": null, "predicted_none": false, "label": "bank_transfer", "correct": true, "confidence": 0.999288022518158, "record_seconds": 0.09435158781707287, "batch_seconds": 0.09435158781707287, "batch_records": 1, "timing_scope": "record", "brier": 6.957874347790494e-07, "nll": 0.0007122310344129801} |
| {"record_id": "jevbench:easy-extraction-03", "group_id": "jevbench:easy-extraction-03", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "high", "probabilities": {"high": 0.9980825185775757, "low": 0.0001321823219768703, "medium": 0.00016388380026910454, "urgent": 0.0016214944189414382}, "none_candidate_id": null, "predicted_none": false, "label": "high", "correct": true, "confidence": 0.9980825185775757, "record_seconds": 0.09581520408391953, "batch_seconds": 0.09581520408391953, "batch_records": 1, "timing_scope": "record", "brier": 6.350309377012309e-06, "nll": 0.001919322181493044} |
| {"record_id": "jevbench:easy-extraction-04", "group_id": "jevbench:easy-extraction-04", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "low", "probabilities": {"high": 0.00014693793491460383, "low": 0.9995120763778687, "medium": 0.0001987516734516248, "urgent": 0.0001423166540917009}, "none_candidate_id": null, "predicted_none": false, "label": "low", "correct": true, "confidence": 0.9995120763778687, "record_seconds": 0.0935540534555912, "batch_seconds": 0.0935540534555912, "batch_records": 1, "timing_scope": "record", "brier": 3.1941647193889366e-07, "nll": 0.00048804268590174615} |
| {"record_id": "jevbench:easy-extraction-05", "group_id": "jevbench:easy-extraction-05", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "urgent", "probabilities": {"high": 0.0013199354289099574, "low": 0.00017871058662422001, "medium": 0.00022721248387824744, "urgent": 0.9982740879058838}, "none_candidate_id": null, "predicted_none": false, "label": "urgent", "correct": true, "confidence": 0.9982740879058838, "record_seconds": 0.09423444233834743, "batch_seconds": 0.09423444233834743, "batch_records": 1, "timing_scope": "record", "brier": 4.804564923688304e-06, "nll": 0.0017274031415581703} |
| {"record_id": "jevbench:easy-extraction-06", "group_id": "jevbench:easy-extraction-06", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "M", "probabilities": {"L": 0.00015634196461178362, "M": 0.9993394017219543, "S": 0.00042417837539687753, "XL": 8.003601396922022e-05}, "none_candidate_id": null, "predicted_none": false, "label": "M", "correct": true, "confidence": 0.9993394017219543, "record_seconds": 0.09289833344519138, "batch_seconds": 0.09289833344519138, "batch_records": 1, "timing_scope": "record", "brier": 6.471659048656875e-07, "nll": 0.0006608165567740798} |
| {"record_id": "jevbench:easy-extraction-07", "group_id": "jevbench:easy-extraction-07", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "XL", "probabilities": {"L": 0.004397204611450434, "M": 0.0006795792141929269, "S": 0.0004585529095493257, "XL": 0.9944646954536438}, "none_candidate_id": null, "predicted_none": false, "label": "XL", "correct": true, "confidence": 0.9944646954536438, "record_seconds": 0.0948436101898551, "batch_seconds": 0.0948436101898551, "batch_records": 1, "timing_scope": "record", "brier": 5.06471042172052e-05, "nll": 0.00555068114772439} |
| {"record_id": "jevbench:easy-extraction-08", "group_id": "jevbench:easy-extraction-08", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "thursday", "probabilities": {"friday": 4.21702534367796e-05, "monday": 7.193576311692595e-05, "thursday": 0.999768078327179, "tuesday": 4.238284600432962e-05, "wednesday": 7.543269748566672e-05}, "none_candidate_id": null, "predicted_none": false, "label": "thursday", "correct": true, "confidence": 0.999768078327179, "record_seconds": 0.0946273636072874, "batch_seconds": 0.0946273636072874, "batch_records": 1, "timing_scope": "record", "brier": 6.822714482268566e-08, "nll": 0.00023194856476038694} |
| {"record_id": "jevbench:easy-extraction-09", "group_id": "jevbench:easy-extraction-09", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "monday", "probabilities": {"friday": 0.00011350607383064926, "monday": 0.9993995428085327, "thursday": 0.00016167860303539783, "tuesday": 0.00014588903286494315, "wednesday": 0.00017931348702404648}, "none_candidate_id": null, "predicted_none": false, "label": "monday", "correct": true, "confidence": 0.9993995428085327, "record_seconds": 0.09382652770727873, "batch_seconds": 0.09382652770727873, "batch_records": 1, "timing_scope": "record", "brier": 4.530093349330855e-07, "nll": 0.0006006375188007951} |
| {"record_id": "jevbench:easy-extraction-10", "group_id": "jevbench:easy-extraction-10", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "friday", "probabilities": {"friday": 0.9993928670883179, "monday": 0.0001601559342816472, "thursday": 0.00013838245649822056, "tuesday": 0.00012622054782696068, "wednesday": 0.00018231416470371187}, "none_candidate_id": null, "predicted_none": false, "label": "friday", "correct": true, "confidence": 0.9993928670883179, "record_seconds": 0.09316989313811064, "batch_seconds": 0.09316989313811064, "batch_records": 1, "timing_scope": "record", "brier": 4.625800329449703e-07, "nll": 0.0006073173135519028} |
| {"record_id": "jevbench:easy-extraction-11", "group_id": "jevbench:easy-extraction-11", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "credit_card", "probabilities": {"bank_transfer": 0.00022688925673719496, "cash": 0.0001882907235994935, "credit_card": 0.999222993850708, "paypal": 0.0003617680922616273}, "none_candidate_id": null, "predicted_none": false, "label": "credit_card", "correct": true, "confidence": 0.999222993850708, "record_seconds": 0.09491673950105906, "batch_seconds": 0.09491673950105906, "batch_records": 1, "timing_scope": "record", "brier": 8.215467914851615e-07, "nll": 0.0007773081888444722} |
| {"record_id": "jevbench:easy-tool_selection-00", "group_id": "jevbench:easy-tool_selection-00", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "get_weather", "probabilities": {"convert_currency": 0.00015503980102948844, "create_calendar_event": 0.00019015486759599298, "get_weather": 0.9992583394050598, "send_email": 0.00020999583648517728, "translate_text": 0.00018646485114004463}, "none_candidate_id": null, "predicted_none": false, "label": "get_weather", "correct": true, "confidence": 0.9992583394050598, "record_seconds": 0.09473198652267456, "batch_seconds": 0.09473198652267456, "batch_records": 1, "timing_scope": "record", "brier": 6.891240786899289e-07, "nll": 0.0007419357425533235} |
| {"record_id": "jevbench:easy-tool_selection-01", "group_id": "jevbench:easy-tool_selection-01", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "send_email", "probabilities": {"convert_currency": 5.39020293217618e-05, "create_calendar_event": 9.792601485969499e-05, "get_weather": 4.073821037309244e-05, "send_email": 0.9997301697731018, "translate_text": 7.723252929281443e-05}, "none_candidate_id": null, "predicted_none": false, "label": "send_email", "correct": true, "confidence": 0.9997301697731018, "record_seconds": 0.09433833323419094, "batch_seconds": 0.09433833323419094, "batch_records": 1, "timing_scope": "record", "brier": 9.292774905134138e-08, "nll": 0.00026986663579009473} |
| {"record_id": "jevbench:easy-tool_selection-02", "group_id": "jevbench:easy-tool_selection-02", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 1.805400097509846e-05, "create_calendar_event": 0.9999014139175415, "get_weather": 1.7390249922755174e-05, "send_email": 4.7676992835476995e-05, "translate_text": 1.544458427815698e-05}, "none_candidate_id": null, "predicted_none": false, "label": "create_calendar_event", "correct": true, "confidence": 0.9999014139175415, "record_seconds": 0.0954233966767788, "batch_seconds": 0.0954233966767788, "batch_records": 1, "timing_scope": "record", "brier": 1.2859214137961317e-08, "nll": 9.859094279818237e-05} |
| {"record_id": "jevbench:easy-tool_selection-03", "group_id": "jevbench:easy-tool_selection-03", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "convert_currency", "probabilities": {"convert_currency": 0.9997459053993225, "create_calendar_event": 4.514974352787249e-05, "get_weather": 5.259008685243316e-05, "send_email": 5.7951303460868075e-05, "translate_text": 9.83913050731644e-05}, "none_candidate_id": null, "predicted_none": false, "label": "convert_currency", "correct": true, "confidence": 0.9997459053993225, "record_seconds": 0.09419271443039179, "batch_seconds": 0.09419271443039179, "batch_records": 1, "timing_scope": "record", "brier": 8.240747462195941e-08, "nll": 0.00025412687682546675} |
| {"record_id": "jevbench:easy-tool_selection-04", "group_id": "jevbench:easy-tool_selection-04", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "translate_text", "probabilities": {"convert_currency": 0.00011012248432962224, "create_calendar_event": 0.00010040927736554295, "get_weather": 0.00014448681031353772, "send_email": 0.00012460531434044242, "translate_text": 0.9995204210281372}, "none_candidate_id": null, "predicted_none": false, "label": "translate_text", "correct": true, "confidence": 0.9995204210281372, "record_seconds": 0.09406161867082119, "batch_seconds": 0.09406161867082119, "batch_records": 1, "timing_scope": "record", "brier": 2.8860790735052433e-07, "nll": 0.00047969401930458844} |
| {"record_id": "jevbench:easy-tool_selection-05", "group_id": "jevbench:easy-tool_selection-05", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 2.0622124793590046e-05, "create_calendar_event": 0.9998458623886108, "get_weather": 2.0049141312483698e-05, "send_email": 9.4371716841124e-05, "translate_text": 1.8985003407578915e-05}, "none_candidate_id": null, "predicted_none": false, "label": "create_calendar_event", "correct": true, "confidence": 0.9998458623886108, "record_seconds": 0.09517170302569866, "batch_seconds": 0.09517170302569866, "batch_records": 1, "timing_scope": "record", "brier": 3.385209623729679e-08, "nll": 0.0001541494857519865} |
| {"record_id": "jevbench:easy-tool_selection-06", "group_id": "jevbench:easy-tool_selection-06", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "search_flights", "probabilities": {"book_restaurant": 2.7880492780241184e-05, "call_taxi": 2.8385762561811134e-05, "get_stock_price": 3.9211783587234095e-05, "search_flights": 0.9998538494110107, "set_reminder": 5.069385588285513e-05}, "none_candidate_id": null, "predicted_none": false, "label": "search_flights", "correct": true, "confidence": 0.9998538494110107, "record_seconds": 0.09592546615749598, "batch_seconds": 0.09592546615749598, "batch_records": 1, "timing_scope": "record", "brier": 2.7050498729863648e-08, "nll": 0.00014616127009503543} |
| {"record_id": "jevbench:easy-tool_selection-07", "group_id": "jevbench:easy-tool_selection-07", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "book_restaurant", "probabilities": {"book_restaurant": 0.9996104836463928, "call_taxi": 9.248710557585582e-05, "get_stock_price": 5.083525684312917e-05, "search_flights": 6.167095125420019e-05, "set_reminder": 0.0001844574580900371}, "none_candidate_id": null, "predicted_none": false, "label": "book_restaurant", "correct": true, "confidence": 0.9996104836463928, "record_seconds": 0.09638551715761423, "batch_seconds": 0.09638551715761423, "batch_records": 1, "timing_scope": "record", "brier": 2.0068894457381248e-07, "nll": 0.00038959222729317844} |
| {"record_id": "jevbench:easy-tool_selection-08", "group_id": "jevbench:easy-tool_selection-08", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "set_reminder", "probabilities": {"book_restaurant": 2.4509430659236386e-05, "call_taxi": 3.89029155485332e-05, "get_stock_price": 3.1189454603008926e-05, "search_flights": 3.641995499492623e-05, "set_reminder": 0.9998688697814941}, "none_candidate_id": null, "predicted_none": false, "label": "set_reminder", "correct": true, "confidence": 0.9998688697814941, "record_seconds": 0.09564278833568096, "batch_seconds": 0.09564278833568096, "batch_records": 1, "timing_scope": "record", "brier": 2.160847856202963e-08, "nll": 0.00013113881868775934} |
| {"record_id": "jevbench:easy-tool_selection-09", "group_id": "jevbench:easy-tool_selection-09", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "get_stock_price", "probabilities": {"book_restaurant": 6.219933129614219e-05, "call_taxi": 5.54386897420045e-05, "get_stock_price": 0.9996196031570435, "search_flights": 9.578645403962582e-05, "set_reminder": 0.00016695244994480163}, "none_candidate_id": null, "predicted_none": false, "label": "get_stock_price", "correct": true, "confidence": 0.9996196031570435, "record_seconds": 0.09425298776477575, "batch_seconds": 0.09425298776477575, "batch_records": 1, "timing_scope": "record", "brier": 1.8869212681238423e-07, "nll": 0.00038046922418288887} |
| {"record_id": "jevbench:easy-tool_selection-10", "group_id": "jevbench:easy-tool_selection-10", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "call_taxi", "probabilities": {"book_restaurant": 7.127677963580936e-05, "call_taxi": 0.9995115995407104, "get_stock_price": 7.2105074650608e-05, "search_flights": 0.0002164089964935556, "set_reminder": 0.0001286302285734564}, "none_candidate_id": null, "predicted_none": false, "label": "call_taxi", "correct": true, "confidence": 0.9995115995407104, "record_seconds": 0.09376687556505203, "batch_seconds": 0.09376687556505203, "batch_records": 1, "timing_scope": "record", "brier": 3.121930944871565e-07, "nll": 0.0004885197849944234} |
| {"record_id": "jevbench:easy-tool_selection-11", "group_id": "jevbench:easy-tool_selection-11", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "set_reminder", "probabilities": {"book_restaurant": 2.74688263743883e-05, "call_taxi": 3.1586165277985856e-05, "get_stock_price": 3.593683504732326e-05, "search_flights": 3.310634565423243e-05, "set_reminder": 0.9998719692230225}, "none_candidate_id": null, "predicted_none": false, "label": "set_reminder", "correct": true, "confidence": 0.9998719692230225, "record_seconds": 0.09555489383637905, "batch_seconds": 0.09555489383637905, "batch_records": 1, "timing_scope": "record", "brier": 2.053158709713898e-08, "nll": 0.00012803896970581263} |
| {"record_id": "jevbench:hard-opus-a-long_policy-01", "group_id": "jevbench:hard-opus-a-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "pay_subject_to_15000_sublimit", "probabilities": {"deny_repeated_seepage": 0.01561769936233759, "deny_vacancy_exclusion": 0.0038507627323269844, "pay_full_estimate_less_deductible": 0.0017644836334511638, "pay_subject_to_10000_sublimit": 0.0008508871542289853, "pay_subject_to_15000_sublimit": 0.9779161810874939}, "none_candidate_id": null, "predicted_none": false, "label": "pay_subject_to_15000_sublimit", "correct": true, "confidence": 0.9779161810874939, "record_seconds": 5.114693277515471, "batch_seconds": 5.114693277515471, "batch_records": 1, "timing_scope": "record", "brier": 0.0007502734079025686, "nll": 0.022331317886710167} |
| {"record_id": "jevbench:hard-opus-a-long_policy-04", "group_id": "jevbench:hard-opus-a-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "department_head", "probabilities": {"budget_holder": 0.0007201944245025516, "cfo": 0.0035600329283624887, "department_head": 0.6524416208267212, "executive_committee": 0.0010331033263355494, "vp_and_finance_director": 0.34224507212638855}, "none_candidate_id": null, "predicted_none": false, "label": "cfo", "correct": false, "confidence": 0.6524416208267212, "record_seconds": 3.366874076426029, "batch_seconds": 3.366874076426029, "batch_records": 1, "timing_scope": "record", "brier": 1.5357060432434082, "nll": 5.637985706329346} |
| {"record_id": "jevbench:hard-opus-a-long_policy-08", "group_id": "jevbench:hard-opus-a-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "parts_covered_labour_charged", "probabilities": {"full_repair_no_charge": 0.015811489894986153, "not_covered": 0.014671850018203259, "parts_covered_labour_charged": 0.966945469379425, "pro_rata_replacement_credit": 0.0025712342467159033}, "none_candidate_id": null, "predicted_none": false, "label": "parts_covered_labour_charged", "correct": true, "confidence": 0.966945469379425, "record_seconds": 2.4906276809051633, "batch_seconds": 2.4906276809051633, "batch_records": 1, "timing_scope": "record", "brier": 0.0015644796658307314, "nll": 0.0336131751537323} |
| {"record_id": "jevbench:hard-opus-a-long_policy-09", "group_id": "jevbench:hard-opus-a-long_policy-09", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "license_required", "probabilities": {"license_exception_lvs": 0.005376575049012899, "license_required": 0.987146258354187, "no_license_required": 0.0065549458377063274, "shipment_prohibited": 0.0009222267544828355}, "none_candidate_id": null, "predicted_none": false, "label": "license_required", "correct": true, "confidence": 0.987146258354187, "record_seconds": 2.617943374440074, "batch_seconds": 2.617943374440074, "batch_records": 1, "timing_scope": "record", "brier": 0.00023794404114596546, "nll": 0.01293706614524126} |
| {"record_id": "jevbench:hard-opus-a-long_policy-11", "group_id": "jevbench:hard-opus-a-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.17851665616035461, "true": 0.821483314037323}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.821483314037323, "record_seconds": 1.229058182798326, "batch_seconds": 1.229058182798326, "batch_records": 1, "timing_scope": "record", "brier": 0.06373640894889832, "nll": 0.1966436505317688} |
| {"record_id": "jevbench:hard-opus-a-long_policy-13", "group_id": "jevbench:hard-opus-a-long_policy-13", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.050604790449142456, "true": 0.9493951797485352}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9493951797485352, "record_seconds": 1.3146543866023421, "batch_seconds": 1.3146543866023421, "batch_records": 1, "timing_scope": "record", "brier": 0.005121693015098572, "nll": 0.05193014815449715} |
| {"record_id": "jevbench:hard-opus-a-long_policy-17", "group_id": "jevbench:hard-opus-a-long_policy-17", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "database_oncall", "probabilities": {"database_oncall": 0.9577419757843018, "no_page_ticket_only": 0.00897764042019844, "platform_oncall": 0.021983755752444267, "security_incident_response": 0.010802432894706726, "vendor_support": 0.0004942500963807106}, "none_candidate_id": null, "predicted_none": false, "label": "database_oncall", "correct": true, "confidence": 0.9577419757843018, "record_seconds": 3.1045598555356264, "batch_seconds": 3.1045598555356264, "batch_records": 1, "timing_scope": "record", "brier": 0.0024665610399097204, "nll": 0.04317687451839447} |
| {"record_id": "jevbench:hard-opus-a-long_policy-19", "group_id": "jevbench:hard-opus-a-long_policy-19", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.2065100520849228, "true": 0.7934899926185608}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.7934899926185608, "record_seconds": 1.267038645222783, "batch_seconds": 1.267038645222783, "batch_records": 1, "timing_scope": "record", "brier": 1.259252667427063, "nll": 1.5774061679840088} |
| {"record_id": "jevbench:hard-opus-a-probability-03", "group_id": "jevbench:hard-opus-a-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "true", "probabilities": {"false": 0.3126601576805115, "true": 0.6873398423194885}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.6873398423194885, "record_seconds": 0.24835043493658304, "batch_seconds": 0.24835043493658304, "batch_records": 1, "timing_scope": "record", "brier": 0.009560634382069111, "nll": 0.6756748557090759} |
| {"record_id": "jevbench:hard-opus-a-probability-04", "group_id": "jevbench:hard-opus-a-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "on_time", "probabilities": {"early": 0.11096638441085815, "late": 0.3353886008262634, "on_time": 0.5536450147628784}, "none_candidate_id": null, "predicted_none": false, "label": "on_time", "correct": true, "confidence": 0.5536450147628784, "record_seconds": 0.8670931244269013, "batch_seconds": 0.8670931244269013, "batch_records": 1, "timing_scope": "record", "brier": 0.00411266228184104, "nll": 0.8654994964599609} |
| {"record_id": "jevbench:hard-opus-a-probability-07", "group_id": "jevbench:hard-opus-a-probability-07", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "escalated", "probabilities": {"escalated": 0.9159612059593201, "reopened": 0.006251833867281675, "resolved_first_contact": 0.07778695225715637}, "none_candidate_id": null, "predicted_none": false, "label": "escalated", "correct": true, "confidence": 0.9159612059593201, "record_seconds": 0.9559706849977374, "batch_seconds": 0.9559706849977374, "batch_records": 1, "timing_scope": "record", "brier": 0.09772489964962006, "nll": 1.1197067499160767} |
| {"record_id": "jevbench:hard-opus-a-probability-08", "group_id": "jevbench:hard-opus-a-probability-08", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8646033406257629, "true": 0.13539664447307587}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.8646033406257629, "record_seconds": 0.23320289701223373, "batch_seconds": 0.23320289701223373, "batch_records": 1, "timing_scope": "record", "brier": 0.047804392874240875, "nll": 0.6831624507904053} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-03", "group_id": "jevbench:hard-opus-a-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "false", "probabilities": {"false": 0.5792412161827087, "true": 0.42075884342193604}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.5792412161827087, "record_seconds": 0.28368412237614393, "batch_seconds": 0.28368412237614393, "batch_records": 1, "timing_scope": "record", "brier": 0.6710407137870789, "nll": 0.8656954169273376} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-06", "group_id": "jevbench:hard-opus-a-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "false", "probabilities": {"false": 0.984322726726532, "true": 0.015677308663725853}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.984322726726532, "record_seconds": 0.2910752249881625, "batch_seconds": 0.2910752249881625, "batch_records": 1, "timing_scope": "record", "brier": 0.0004915549070574343, "nll": 0.015801461413502693} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-07", "group_id": "jevbench:hard-opus-a-temporal_numeric-07", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "sep_26", "probabilities": {"sep_24": 0.07719635963439941, "sep_25": 0.07619203627109528, "sep_26": 0.5458678603172302, "sep_27": 0.172432079911232, "sep_28": 0.1283116489648819}, "none_candidate_id": null, "predicted_none": false, "label": "sep_26", "correct": true, "confidence": 0.5458678603172302, "record_seconds": 1.6062434492632747, "batch_seconds": 1.6062434492632747, "batch_records": 1, "timing_scope": "record", "brier": 0.2641972005367279, "nll": 0.6053783297538757} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-09", "group_id": "jevbench:hard-opus-a-temporal_numeric-09", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "true", "probabilities": {"false": 0.3931961953639984, "true": 0.606803834438324}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.606803834438324, "record_seconds": 0.2918967558071017, "batch_seconds": 0.2918967558071017, "batch_records": 1, "timing_scope": "record", "brier": 0.7364217042922974, "nll": 0.9334465861320496} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-12", "group_id": "jevbench:hard-opus-a-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "score", "prediction": "2", "probabilities": {"0": 0.11042854934930801, "1": 0.05071394518017769, "2": 0.5422180891036987, "3": 0.29663944244384766}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.5422180891036987, "record_seconds": 0.605930894613266, "batch_seconds": 0.605930894613266, "batch_records": 1, "timing_scope": "record", "brier": 1.295333743095398, "nll": 2.9815542697906494} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-02", "group_id": "jevbench:hard-opus-b-ambiguous-02", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "real_incident", "probabilities": {"cannot_determine": 0.012992545031011105, "false_positive": 0.013660970143973827, "real_incident": 0.9733464121818542}, "none_candidate_id": null, "predicted_none": false, "label": "real_incident", "correct": true, "confidence": 0.9733464121818542, "record_seconds": 0.4173285346478224, "batch_seconds": 0.4173285346478224, "batch_records": 1, "timing_scope": "record", "brier": 0.0010658421088010073, "nll": 0.027015235275030136} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-03", "group_id": "jevbench:hard-opus-b-ambiguous-03", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "within_limitation", "probabilities": {"insufficient_information": 0.004111605230718851, "time_barred": 0.07279568910598755, "within_limitation": 0.9230927228927612}, "none_candidate_id": null, "predicted_none": false, "label": "time_barred", "correct": false, "confidence": 0.9230927228927612, "record_seconds": 0.3492133682593703, "batch_seconds": 0.3492133682593703, "batch_records": 1, "timing_scope": "record", "brier": 1.711824893951416, "nll": 2.62009859085083} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-07", "group_id": "jevbench:hard-opus-b-ambiguous-07", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "within_policy", "probabilities": {"insufficient_information": 0.0006502806209027767, "over_limit": 0.00442925002425909, "within_policy": 0.9949204325675964}, "none_candidate_id": null, "predicted_none": false, "label": "within_policy", "correct": true, "confidence": 0.9949204325675964, "record_seconds": 0.39745834469795227, "batch_seconds": 0.39745834469795227, "batch_records": 1, "timing_scope": "record", "brier": 4.584312409861013e-05, "nll": 0.005092512350529432} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-09", "group_id": "jevbench:hard-opus-b-ambiguous-09", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "legitimate", "probabilities": {"compromised": 0.03136300668120384, "inconclusive": 0.003240426303818822, "legitimate": 0.9653965830802917}, "none_candidate_id": null, "predicted_none": false, "label": "legitimate", "correct": true, "confidence": 0.9653965830802917, "record_seconds": 0.4888626569882035, "batch_seconds": 0.4888626569882035, "batch_records": 1, "timing_scope": "record", "brier": 0.0021915349643677473, "nll": 0.03521629422903061} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-10", "group_id": "jevbench:hard-opus-b-ambiguous-10", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "send_to_alexandra_lind", "probabilities": {"ask_user_which_alex": 0.03372562304139137, "send_to_alex_moreno": 0.012097420170903206, "send_to_alexandra_lind": 0.9541770219802856}, "none_candidate_id": null, "predicted_none": false, "label": "ask_user_which_alex", "correct": false, "confidence": 0.9541770219802856, "record_seconds": 0.29952979553490877, "batch_seconds": 0.29952979553490877, "batch_records": 1, "timing_scope": "record", "brier": 1.844286322593689, "nll": 3.3894975185394287} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-11", "group_id": "jevbench:hard-opus-b-ambiguous-11", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.7100902795791626, "grade_correct": 0.2891767621040344, "grade_should_be_higher": 0.0007329834625124931}, "none_candidate_id": null, "predicted_none": false, "label": "cannot_determine", "correct": true, "confidence": 0.7100902795791626, "record_seconds": 0.3403876619413495, "batch_seconds": 0.3403876619413495, "batch_records": 1, "timing_scope": "record", "brier": 0.16767138242721558, "nll": 0.3423631489276886} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-13", "group_id": "jevbench:hard-opus-b-ambiguous-13", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.999543309211731, "charge_data_team": 0.00013959685747977346, "charge_platform_team": 0.00016692091594450176, "split_50_50": 0.00015013894881121814}, "none_candidate_id": null, "predicted_none": false, "label": "cannot_determine", "correct": true, "confidence": 0.999543309211731, "record_seconds": 0.5758512374013662, "batch_seconds": 0.5758512374013662, "batch_records": 1, "timing_scope": "record", "brier": 2.7845806016557617e-07, "nll": 0.0004567950963973999} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-03", "group_id": "jevbench:hard-opus-b-multi_hop-03", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "tier3_cfo", "probabilities": {"tier1_team_lead": 0.2964632511138916, "tier2_department_head": 0.030093345791101456, "tier3_cfo": 0.6192458271980286, "tier4_board": 0.05419757962226868}, "none_candidate_id": null, "predicted_none": false, "label": "tier3_cfo", "correct": true, "confidence": 0.6192458271980286, "record_seconds": 3.069371370598674, "batch_seconds": 3.069371370598674, "batch_records": 1, "timing_scope": "record", "brier": 0.2367071807384491, "nll": 0.4792529344558716} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-04", "group_id": "jevbench:hard-opus-b-multi_hop-04", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "esra", "probabilities": {"ana": 0.001338483765721321, "bjorn": 0.02321927435696125, "chen": 0.001047267927788198, "dmitri": 0.0008064038120210171, "esra": 0.9668622612953186, "no_page_ticket_only": 0.0067263441160321236}, "none_candidate_id": null, "predicted_none": false, "label": "bjorn", "correct": false, "confidence": 0.9668622612953186, "record_seconds": 3.7161856330931187, "batch_seconds": 3.7161856330931187, "batch_records": 1, "timing_scope": "record", "brier": 1.8889718055725098, "nll": 3.762772560119629} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-05", "group_id": "jevbench:hard-opus-b-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "score", "prediction": "3", "probabilities": {"0": 0.0010685621527954936, "1": 0.011873653158545494, "2": 0.1323651671409607, "3": 0.8468869924545288, "4": 0.007805658038705587}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": false, "confidence": 0.8468869924545288, "record_seconds": 3.3864873042330146, "batch_seconds": 3.3864873042330146, "batch_records": 1, "timing_scope": "record", "brier": 1.4702107906341553, "nll": 2.022190809249878} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-07", "group_id": "jevbench:hard-opus-b-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "credit_20_percent", "probabilities": {"credit_10_percent": 0.0034081735648214817, "credit_20_percent": 0.5509559512138367, "credit_25_percent": 0.2599664628505707, "credit_50_percent": 0.09628866612911224, "credit_full_month": 0.009545296430587769, "no_credit": 0.079835444688797}, "none_candidate_id": null, "predicted_none": false, "label": "credit_20_percent", "correct": true, "confidence": 0.5509559512138367, "record_seconds": 4.076669901609421, "batch_seconds": 4.076669901609421, "batch_records": 1, "timing_scope": "record", "brier": 0.28497105836868286, "nll": 0.596100389957428} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-08", "group_id": "jevbench:hard-opus-b-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9456799030303955, "true": 0.05432013049721718}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9456799030303955, "record_seconds": 1.266324097290635, "batch_seconds": 1.266324097290635, "batch_records": 1, "timing_scope": "record", "brier": 0.00590134970843792, "nll": 0.05585113540291786} |
| {"record_id": "jevbench:hard-opus-b-probability-01", "group_id": "jevbench:hard-opus-b-probability-01", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "merchant_wins", "probabilities": {"customer_wins": 0.09934116899967194, "customer_withdraws": 0.0034202125389128923, "merchant_wins": 0.8972386717796326}, "none_candidate_id": null, "predicted_none": false, "label": "merchant_wins", "correct": true, "confidence": 0.8972386717796326, "record_seconds": 1.5241203289479017, "batch_seconds": 1.5241203289479017, "batch_records": 1, "timing_scope": "record", "brier": 0.093152716755867, "nll": 1.215585708618164} |
| {"record_id": "jevbench:hard-opus-b-probability-02", "group_id": "jevbench:hard-opus-b-probability-02", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "bad_push", "probabilities": {"bad_push": 0.7894552946090698, "database_hardware": 0.06595198810100555, "upstream_provider": 0.14459267258644104}, "none_candidate_id": null, "predicted_none": false, "label": "upstream_provider", "correct": false, "confidence": 0.7894552946090698, "record_seconds": 0.4030967643484473, "batch_seconds": 0.4030967643484473, "batch_records": 1, "timing_scope": "record", "brier": 0.6099581718444824, "nll": 1.7120990753173828} |
| {"record_id": "jevbench:hard-opus-b-probability-03", "group_id": "jevbench:hard-opus-b-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "true", "probabilities": {"false": 0.1256759613752365, "true": 0.8743240833282471}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8743240833282471, "record_seconds": 0.6570494286715984, "batch_seconds": 0.6570494286715984, "batch_records": 1, "timing_scope": "record", "brier": 0.06980645656585693, "nll": 0.740474283695221} |
| {"record_id": "jevbench:hard-opus-b-probability-04", "group_id": "jevbench:hard-opus-b-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "attended", "probabilities": {"attended": 0.8688682317733765, "late_cancellation": 0.037154071033000946, "no_show": 0.09397774189710617}, "none_candidate_id": null, "predicted_none": false, "label": "attended", "correct": true, "confidence": 0.8688682317733765, "record_seconds": 0.460791383869946, "batch_seconds": 0.460791383869946, "batch_records": 1, "timing_scope": "record", "brier": 0.011801657266914845, "nll": 0.6967486143112183} |
| {"record_id": "jevbench:hard-opus-b-probability-06", "group_id": "jevbench:hard-opus-b-probability-06", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "overturned", "probabilities": {"modified": 0.0228720735758543, "overturned": 0.7332876324653625, "upheld": 0.2438403069972992}, "none_candidate_id": null, "predicted_none": false, "label": "upheld", "correct": false, "confidence": 0.7332876324653625, "record_seconds": 0.8032987611368299, "batch_seconds": 0.8032987611368299, "batch_records": 1, "timing_scope": "record", "brier": 0.4319712519645691, "nll": 1.2838979959487915} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-01", "group_id": "jevbench:hard-opus-b-tradeoff-01", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "escalate_to_security", "probabilities": {"escalate_to_engineering": 0.0004004373331554234, "escalate_to_security": 0.9991771578788757, "page_oncall": 7.251615897985175e-05, "process_refund": 0.00014224604819901288, "send_kb_article": 0.00020766489615198225}, "none_candidate_id": null, "predicted_none": false, "label": "escalate_to_engineering", "correct": false, "confidence": 0.9991771578788757, "record_seconds": 1.0279851146042347, "batch_seconds": 1.0279851146042347, "batch_records": 1, "timing_scope": "record", "brier": 1.9975543022155762, "nll": 7.822953224182129} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-03", "group_id": "jevbench:hard-opus-b-tradeoff-03", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "order_b", "probabilities": {"order_a": 0.06544938683509827, "order_b": 0.9026052355766296, "order_c": 0.03001556545495987, "order_d": 0.0019297838443890214}, "none_candidate_id": null, "predicted_none": false, "label": "order_b", "correct": true, "confidence": 0.9026052355766296, "record_seconds": 0.7586934557184577, "batch_seconds": 0.7586934557184577, "batch_records": 1, "timing_scope": "record", "brier": 0.014674020931124687, "nll": 0.10246998816728592} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-06", "group_id": "jevbench:hard-opus-b-tradeoff-06", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "prediction": "false", "probabilities": {"false": 0.7706299424171448, "true": 0.2293701022863388}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.7706299424171448, "record_seconds": 0.424707910977304, "batch_seconds": 0.424707910977304, "batch_records": 1, "timing_scope": "record", "brier": 1.1877409219741821, "nll": 1.4724184274673462} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-07", "group_id": "jevbench:hard-opus-b-tradeoff-07", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "p1_fix_7d", "probabilities": {"out_of_scope": 0.00012638972839340568, "p0_fix_24h": 0.07892152667045593, "p1_fix_7d": 0.9201444387435913, "p2_fix_30d": 0.0004417808959260583, "p3_backlog": 0.0003659161739051342}, "none_candidate_id": null, "predicted_none": false, "label": "p0_fix_24h", "correct": false, "confidence": 0.9201444387435913, "record_seconds": 0.8347569918259978, "batch_seconds": 0.8347569918259978, "batch_records": 1, "timing_scope": "record", "brier": 1.695051670074463, "nll": 2.5393011569976807} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-08", "group_id": "jevbench:hard-opus-b-tradeoff-08", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "prediction": "true", "probabilities": {"false": 0.18428389728069305, "true": 0.8157160878181458}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8157160878181458, "record_seconds": 0.32509409822523594, "batch_seconds": 0.32509409822523594, "batch_records": 1, "timing_scope": "record", "brier": 0.06792111694812775, "nll": 0.20368891954421997} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-12", "group_id": "jevbench:hard-opus-b-tradeoff-12", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "refund_now", "probabilities": {"deny_refund": 0.00013711181236431003, "escalate_to_fraud_team": 6.464016769314185e-05, "offer_store_credit": 9.718539513414726e-05, "refund_now": 0.999548614025116, "wait_for_inspection": 0.0001524741674074903}, "none_candidate_id": null, "predicted_none": false, "label": "refund_now", "correct": true, "confidence": 0.999548614025116, "record_seconds": 0.6892442218959332, "batch_seconds": 0.6892442218959332, "batch_records": 1, "timing_scope": "record", "brier": 2.594206591766124e-07, "nll": 0.00045148786739446223} |
| {"record_id": "jevbench:hard-opus-c-long_policy-02", "group_id": "jevbench:hard-opus-c-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "erase_but_retain_transaction_records", "probabilities": {"erase_all": 0.006850627716630697, "erase_but_retain_transaction_records": 0.991436779499054, "erase_marketing_only": 0.0005527877947315574, "reject_unverified_request": 0.0008328972035087645, "restrict_all_pending_legal_hold": 0.00032699116854928434}, "none_candidate_id": null, "predicted_none": false, "label": "erase_but_retain_transaction_records", "correct": true, "confidence": 0.991436779499054, "record_seconds": 4.756138348020613, "batch_seconds": 4.756138348020613, "batch_records": 1, "timing_scope": "record", "brier": 0.00012136606528656557, "nll": 0.00860009528696537} |
| {"record_id": "jevbench:hard-opus-c-long_policy-03", "group_id": "jevbench:hard-opus-c-long_policy-03", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "eur_0", "probabilities": {"eur_0": 0.505271852016449, "eur_150": 0.1950395405292511, "eur_200": 0.04621446877717972, "eur_300": 0.123713918030262, "eur_400": 0.08915350586175919, "eur_600": 0.04060674458742142}, "none_candidate_id": null, "predicted_none": false, "label": "eur_400", "correct": false, "confidence": 0.505271852016449, "record_seconds": 5.5097070736810565, "batch_seconds": 5.5097070736810565, "batch_records": 1, "timing_scope": "record", "brier": 1.1420711278915405, "nll": 2.41739559173584} |
| {"record_id": "jevbench:hard-opus-c-long_policy-04", "group_id": "jevbench:hard-opus-c-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.25976836681365967, "true": 0.7402316927909851}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.7402316927909851, "record_seconds": 1.7971543418243527, "batch_seconds": 1.7971543418243527, "batch_records": 1, "timing_scope": "record", "brier": 0.1349591761827469, "nll": 0.30079203844070435} |
| {"record_id": "jevbench:hard-opus-c-long_policy-05", "group_id": "jevbench:hard-opus-c-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "prediction": "2", "probabilities": {"0": 0.3398963212966919, "1": 0.044598545879125595, "2": 0.4062090814113617, "3": 0.20929603278636932}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.4062090814113617, "record_seconds": 3.4403375266119838, "batch_seconds": 3.4403375266119838, "batch_records": 1, "timing_scope": "record", "brier": 1.23713219165802, "nll": 3.1100540161132812} |
| {"record_id": "jevbench:hard-opus-c-long_policy-08", "group_id": "jevbench:hard-opus-c-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "prediction": "2", "probabilities": {"0": 0.0022419311571866274, "1": 0.20984816551208496, "2": 0.7693363428115845, "3": 0.01857362687587738}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.7693363428115845, "record_seconds": 3.384277799166739, "batch_seconds": 3.384277799166739, "batch_records": 1, "timing_scope": "record", "brier": 1.2165683507919312, "nll": 1.561371088027954} |
| {"record_id": "jevbench:hard-opus-c-long_policy-10", "group_id": "jevbench:hard-opus-c-long_policy-10", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "credit_minus_restocking_fee", "probabilities": {"credit_minus_restocking_fee": 0.8758305907249451, "full_credit": 0.01141315046697855, "reject_return": 0.10059547424316406, "repair_or_replace_only": 0.012160838581621647}, "none_candidate_id": null, "predicted_none": false, "label": "credit_minus_restocking_fee", "correct": true, "confidence": 0.8758305907249451, "record_seconds": 3.9170706793665886, "batch_seconds": 3.9170706793665886, "batch_records": 1, "timing_scope": "record", "brier": 0.02581563964486122, "nll": 0.13258258998394012} |
| {"record_id": "jevbench:hard-opus-c-long_policy-11", "group_id": "jevbench:hard-opus-c-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.15720397233963013, "true": 0.8427959680557251}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.8427959680557251, "record_seconds": 1.880238763988018, "batch_seconds": 1.880238763988018, "batch_records": 1, "timing_scope": "record", "brier": 1.4206101894378662, "nll": 1.8502111434936523} |
| {"record_id": "jevbench:hard-opus-c-probability-03", "group_id": "jevbench:hard-opus-c-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "halvorsen_polymer", "probabilities": {"halvorsen_polymer": 0.8199381828308105, "kuznets_technik": 0.0883457139134407, "ortega_seal_works": 0.09171609580516815}, "none_candidate_id": null, "predicted_none": false, "label": "kuznets_technik", "correct": false, "confidence": 0.8199381828308105, "record_seconds": 0.424417975358665, "batch_seconds": 0.424417975358665, "batch_records": 1, "timing_scope": "record", "brier": 0.665281355381012, "nll": 1.9814023971557617} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-02", "group_id": "jevbench:hard-opus-c-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8575814366340637, "true": 0.14241856336593628}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.8575814366340637, "record_seconds": 0.5062494724988937, "batch_seconds": 0.5062494724988937, "batch_records": 1, "timing_scope": "record", "brier": 1.4708918333053589, "nll": 1.948984980583191} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-03", "group_id": "jevbench:hard-opus-c-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "eur_693_90", "probabilities": {"eur_617_00": 0.18629465997219086, "eur_688_00": 0.18690037727355957, "eur_692_00": 0.19390882551670074, "eur_693_90": 0.32258912920951843, "eur_812_00": 0.11030693352222443}, "none_candidate_id": null, "predicted_none": false, "label": "eur_692_00", "correct": false, "confidence": 0.32258912920951843, "record_seconds": 1.0576925380155444, "batch_seconds": 1.0576925380155444, "batch_records": 1, "timing_scope": "record", "brier": 0.8356518149375916, "nll": 1.6403671503067017} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-04", "group_id": "jevbench:hard-opus-c-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "usd_2303_33", "probabilities": {"usd_2222_50": 0.1450301706790924, "usd_2303_01": 0.28658199310302734, "usd_2303_33": 0.3007740080356598, "usd_2335_00": 0.16834139823913574, "usd_2340_00": 0.09927239269018173}, "none_candidate_id": null, "predicted_none": false, "label": "usd_2335_00", "correct": false, "confidence": 0.3007740080356598, "record_seconds": 0.855783699080348, "batch_seconds": 0.855783699080348, "batch_records": 1, "timing_scope": "record", "brier": 0.8951390385627747, "nll": 1.7817612886428833} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-06", "group_id": "jevbench:hard-opus-c-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "sep_16_taxi", "probabilities": {"none_declined": 0.03167713060975075, "sep_15_office_chair": 0.04486842080950737, "sep_16_taxi": 0.8677324056625366, "sep_17_parking": 0.002085022861137986, "sep_18_book_set": 0.01097453385591507, "sep_21_client_dinner": 0.04266246408224106}, "none_candidate_id": null, "predicted_none": false, "label": "sep_18_book_set", "correct": false, "confidence": 0.8677324056625366, "record_seconds": 1.3350879410281777, "batch_seconds": 1.3350879410281777, "batch_records": 1, "timing_scope": "record", "brier": 1.7359719276428223, "nll": 4.51217794418335} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-08", "group_id": "jevbench:hard-opus-c-temporal_numeric-08", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "eur_513_69", "probabilities": {"eur_497_40": 0.23418420553207397, "eur_498_81": 0.18666738271713257, "eur_506_06": 0.1953170895576477, "eur_513_69": 0.26525095105171204, "eur_532_01": 0.1185804232954979}, "none_candidate_id": null, "predicted_none": false, "label": "eur_498_81", "correct": false, "confidence": 0.26525095105171204, "record_seconds": 1.1450864495709538, "batch_seconds": 1.1450864495709538, "batch_records": 1, "timing_scope": "record", "brier": 0.8389203548431396, "nll": 1.67842698097229} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-12", "group_id": "jevbench:hard-opus-c-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "expired_30_month_cap", "probabilities": {"covered": 0.18604746460914612, "expired_24_month_term": 0.33958253264427185, "expired_30_month_cap": 0.47436997294425964}, "none_candidate_id": null, "predicted_none": false, "label": "expired_30_month_cap", "correct": true, "confidence": 0.47436997294425964, "record_seconds": 0.6342759029939771, "batch_seconds": 0.6342759029939771, "batch_records": 1, "timing_scope": "record", "brier": 0.42621684074401855, "nll": 0.7457677125930786} |
| {"record_id": "jevbench:hard-sol-a-adversarial-01", "group_id": "jevbench:hard-sol-a-adversarial-01", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "deny_missing_approval", "probabilities": {"deny_inactive_user": 0.00016253742796834558, "deny_missing_approval": 0.9987214207649231, "grant_access": 0.0006865279865451157, "grant_owner_exception": 0.0004294425016269088}, "none_candidate_id": null, "predicted_none": false, "label": "deny_missing_approval", "correct": true, "confidence": 0.9987214207649231, "record_seconds": 0.2596174981445074, "batch_seconds": 0.2596174981445074, "batch_records": 1, "timing_scope": "record", "brier": 2.31692501984071e-06, "nll": 0.0012793972855433822} |
| {"record_id": "jevbench:hard-sol-a-adversarial-06", "group_id": "jevbench:hard-sol-a-adversarial-06", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9967184662818909, "true": 0.0032815446611493826}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9967184662818909, "record_seconds": 0.1463751681149006, "batch_seconds": 0.1463751681149006, "batch_records": 1, "timing_scope": "record", "brier": 2.153699824702926e-05, "nll": 0.003286929801106453} |
| {"record_id": "jevbench:hard-sol-a-adversarial-07", "group_id": "jevbench:hard-sol-a-adversarial-07", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "retain_until_2031", "probabilities": {"delete_now": 0.005227851681411266, "needs_signature_review": 0.0003109938988927752, "retain_indefinitely": 0.0018091031815856695, "retain_until_2031": 0.9926521182060242}, "none_candidate_id": null, "predicted_none": false, "label": "retain_until_2031", "correct": true, "confidence": 0.9926521182060242, "record_seconds": 0.26577954832464457, "batch_seconds": 0.26577954832464457, "batch_records": 1, "timing_scope": "record", "brier": 8.469137537758797e-05, "nll": 0.007375010289251804} |
| {"record_id": "jevbench:hard-sol-a-adversarial-08", "group_id": "jevbench:hard-sol-a-adversarial-08", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0022826914209872484, "true": 0.9977173805236816}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9977173805236816, "record_seconds": 0.12997857760637999, "batch_seconds": 0.12997857760637999, "batch_records": 1, "timing_scope": "record", "brier": 1.0421032129670493e-05, "nll": 0.0022852285765111446} |
| {"record_id": "jevbench:hard-sol-a-adversarial-09", "group_id": "jevbench:hard-sol-a-adversarial-09", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "score", "prediction": "2", "probabilities": {"0": 0.0002813204482663423, "1": 0.0034648231230676174, "2": 0.994398295879364, "3": 0.0018555538263171911}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.994398295879364, "record_seconds": 0.23626681044697762, "batch_seconds": 0.23626681044697762, "batch_records": 1, "timing_scope": "record", "brier": 4.690630885306746e-05, "nll": 0.005617452319711447} |
| {"record_id": "jevbench:hard-sol-a-adversarial-11", "group_id": "jevbench:hard-sol-a-adversarial-11", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "standard_research", "probabilities": {"ethics_review": 0.0002245427021989599, "major_grants": 6.731939356541261e-05, "manual_hold": 0.00015265749243553728, "standard_research": 0.9995554089546204}, "none_candidate_id": null, "predicted_none": false, "label": "standard_research", "correct": true, "confidence": 0.9995554089546204, "record_seconds": 0.28723220713436604, "batch_seconds": 0.28723220713436604, "batch_records": 1, "timing_scope": "record", "brier": 2.759168182819849e-07, "nll": 0.0004446899110917002} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-01", "group_id": "jevbench:hard-sol-a-multi_hop-01", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "accept_s2_dusk", "probabilities": {"accept_s2_dusk": 0.9984365105628967, "reject_cutoff": 0.00027538335416466, "reject_tier": 0.00041784741915762424, "reroute_s3": 0.0008701565093360841}, "none_candidate_id": null, "predicted_none": false, "label": "accept_s2_dusk", "correct": true, "confidence": 0.9984365105628967, "record_seconds": 2.292737141251564, "batch_seconds": 2.292737141251564, "batch_records": 1, "timing_scope": "record", "brier": 3.4521037832746515e-06, "nll": 0.0015647129621356726} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-02", "group_id": "jevbench:hard-sol-a-multi_hop-02", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "security_primary", "probabilities": {"executive_bridge": 0.011698809452354908, "heron_q2": 0.0057142251171171665, "insufficient_information": 0.010286418721079826, "security_primary": 0.9723005890846252}, "none_candidate_id": null, "predicted_none": false, "label": "security_primary", "correct": true, "confidence": 0.9723005890846252, "record_seconds": 2.320096810348332, "batch_seconds": 2.320096810348332, "batch_records": 1, "timing_scope": "record", "brier": 0.0010425823274999857, "nll": 0.028090273961424828} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-05", "group_id": "jevbench:hard-sol-a-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "require_finance_board", "probabilities": {"pay_manager_approved": 0.1871548444032669, "reject_category": 0.005377983208745718, "require_director": 0.10465512424707413, "require_finance_board": 0.7028120756149292}, "none_candidate_id": null, "predicted_none": false, "label": "require_director", "correct": false, "confidence": 0.7028120756149292, "record_seconds": 2.2169039817526937, "batch_seconds": 2.2169039817526937, "batch_records": 1, "timing_scope": "record", "brier": 1.3306431770324707, "nll": 2.257084846496582} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-07", "group_id": "jevbench:hard-sol-a-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "admit", "probabilities": {"admit": 0.998700737953186, "deny_prerequisite": 0.0008097364334389567, "deny_score": 0.00041329037048853934, "manual_equivalency": 7.627115701325238e-05}, "none_candidate_id": null, "predicted_none": false, "label": "admit", "correct": true, "confidence": 0.998700737953186, "record_seconds": 2.2152427500113845, "batch_seconds": 2.2152427500113845, "batch_records": 1, "timing_scope": "record", "brier": 2.5203812583640683e-06, "nll": 0.001300106872804463} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-08", "group_id": "jevbench:hard-sol-a-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "local_maintenance", "probabilities": {"central_engineering": 0.016643282026052475, "insufficient_log": 0.0052484082989394665, "local_maintenance": 0.977771520614624, "safety_shutdown": 0.00033672156860120595}, "none_candidate_id": null, "predicted_none": false, "label": "central_engineering", "correct": false, "confidence": 0.977771520614624, "record_seconds": 2.1499399608001113, "batch_seconds": 2.1499399608001113, "batch_records": 1, "timing_scope": "record", "brier": 1.9230552911758423, "nll": 4.095748424530029} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-09", "group_id": "jevbench:hard-sol-a-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "require_steward", "probabilities": {"confirm_l3": 0.05812842771410942, "reject_capacity": 0.016278304159641266, "require_steward": 0.9210250973701477, "switch_l4": 0.00456817913800478}, "none_candidate_id": null, "predicted_none": false, "label": "require_steward", "correct": true, "confidence": 0.9210250973701477, "record_seconds": 2.148559366352856, "batch_seconds": 2.148559366352856, "batch_records": 1, "timing_scope": "record", "brier": 0.009901800192892551, "nll": 0.08226799219846725} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-10", "group_id": "jevbench:hard-sol-a-multi_hop-10", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "reduce_to_30_days", "probabilities": {"approve_45_days": 0.29610997438430786, "deny_sensitive": 0.012407471425831318, "needs_field_review": 0.002771301893517375, "reduce_to_30_days": 0.6887112855911255}, "none_candidate_id": null, "predicted_none": false, "label": "approve_45_days", "correct": false, "confidence": 0.6887112855911255, "record_seconds": 2.256844243966043, "batch_seconds": 2.256844243966043, "batch_records": 1, "timing_scope": "record", "brier": 0.969946026802063, "nll": 1.217024326324463} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-12", "group_id": "jevbench:hard-sol-a-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "needs_authentication", "probabilities": {"covered_recurrence": 0.020463019609451294, "expired": 0.1258193552494049, "needs_authentication": 0.8506078720092773, "wrong_issue_class": 0.003109769895672798}, "none_candidate_id": null, "predicted_none": false, "label": "needs_authentication", "correct": true, "confidence": 0.8506078720092773, "record_seconds": 2.153882971033454, "batch_seconds": 2.153882971033454, "batch_records": 1, "timing_scope": "record", "brier": 0.03857692331075668, "nll": 0.16180403530597687} |
| {"record_id": "jevbench:hard-sol-a-trap-02", "group_id": "jevbench:hard-sol-a-trap-02", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.99785977602005, "true": 0.0021402561105787754}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.99785977602005, "record_seconds": 0.16391743998974562, "batch_seconds": 0.16391743998974562, "batch_records": 1, "timing_scope": "record", "brier": 9.16125463845674e-06, "nll": 0.0021425175946205854} |
| {"record_id": "jevbench:hard-sol-a-trap-04", "group_id": "jevbench:hard-sol-a-trap-04", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.995867133140564, "true": 0.004132878500968218}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.995867133140564, "record_seconds": 0.12042888812720776, "batch_seconds": 0.12042888812720776, "batch_records": 1, "timing_scope": "record", "brier": 3.416127583477646e-05, "nll": 0.004141430836170912} |
| {"record_id": "jevbench:hard-sol-a-trap-05", "group_id": "jevbench:hard-sol-a-trap-05", "question_id": "decision", "source": "jevbench/hard/trap", "type": "score", "prediction": "0", "probabilities": {"0": 0.7560700178146362, "1": 0.22364723682403564, "2": 0.019594529643654823, "3": 0.0006881640292704105}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.7560700178146362, "record_seconds": 0.3337820004671812, "batch_seconds": 0.3337820004671812, "batch_records": 1, "timing_scope": "record", "brier": 0.10990434885025024, "nll": 0.27962130308151245} |
| {"record_id": "jevbench:hard-sol-a-trap-06", "group_id": "jevbench:hard-sol-a-trap-06", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "keep_open_unresolved", "probabilities": {"await_internal_metrics": 0.0011018490185961127, "close_agent_verified": 0.0006594828446395695, "close_confirmed": 0.00042298814514651895, "keep_open_unresolved": 0.997815728187561}, "none_candidate_id": null, "predicted_none": false, "label": "keep_open_unresolved", "correct": true, "confidence": 0.997815728187561, "record_seconds": 0.32564973924309015, "batch_seconds": 0.32564973924309015, "batch_records": 1, "timing_scope": "record", "brier": 6.59895113130915e-06, "nll": 0.002186660887673497} |
| {"record_id": "jevbench:hard-sol-a-trap-08", "group_id": "jevbench:hard-sol-a-trap-08", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "late", "probabilities": {"canceled": 0.00026869072462432086, "indeterminate": 0.000448437815066427, "late": 0.9987534284591675, "on_time": 0.0005294438451528549}, "none_candidate_id": null, "predicted_none": false, "label": "late", "correct": true, "confidence": 0.9987534284591675, "record_seconds": 0.25520662032067776, "batch_seconds": 0.25520662032067776, "batch_records": 1, "timing_scope": "record", "brier": 2.107542513840599e-06, "nll": 0.0012473491951823235} |
| {"record_id": "jevbench:hard-sol-a-trap-10", "group_id": "jevbench:hard-sol-a-trap-10", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "blocked_second_review", "probabilities": {"blocked_failed_test": 0.00033418723614886403, "blocked_second_review": 0.9930480122566223, "ready_one_review": 0.005910051055252552, "ready_two_reviews": 0.000707764585968107}, "none_candidate_id": null, "predicted_none": false, "label": "blocked_second_review", "correct": true, "confidence": 0.9930480122566223, "record_seconds": 0.17494453582912683, "batch_seconds": 0.17494453582912683, "batch_records": 1, "timing_scope": "record", "brier": 8.387144771404564e-05, "nll": 0.006976265460252762} |
| {"record_id": "jevbench:hard-sol-a-trap-13", "group_id": "jevbench:hard-sol-a-trap-13", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9971907734870911, "true": 0.0028092325665056705}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9971907734870911, "record_seconds": 0.12280851975083351, "batch_seconds": 0.12280851975083351, "batch_records": 1, "timing_scope": "record", "brier": 1.578354203957133e-05, "nll": 0.0028131797444075346} |
| {"record_id": "jevbench:hard-sol-a-trap-15", "group_id": "jevbench:hard-sol-a-trap-15", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9710435271263123, "true": 0.028956444934010506}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9710435271263123, "record_seconds": 0.1766484435647726, "batch_seconds": 0.1766484435647726, "batch_records": 1, "timing_scope": "record", "brier": 0.0016769530484452844, "nll": 0.029383985325694084} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-01", "group_id": "jevbench:hard-sol-b-judge_hard-01", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.006944643799215555, "true": 0.9930554032325745}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9930554032325745, "record_seconds": 0.16016844380646944, "batch_seconds": 0.16016844380646944, "batch_records": 1, "timing_scope": "record", "brier": 9.645550017012283e-05, "nll": 0.006968822795897722} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-02", "group_id": "jevbench:hard-sol-b-judge_hard-02", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.015023602172732353, "true": 0.9849764108657837}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.9849764108657837, "record_seconds": 0.12334202323108912, "batch_seconds": 0.12334202323108912, "batch_records": 1, "timing_scope": "record", "brier": 1.9403570890426636, "nll": 4.1981329917907715} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-05", "group_id": "jevbench:hard-sol-b-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.09213860332965851, "true": 0.9078613519668579}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9078613519668579, "record_seconds": 0.1357569731771946, "batch_seconds": 0.1357569731771946, "batch_records": 1, "timing_scope": "record", "brier": 0.016979053616523743, "nll": 0.09666360914707184} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-08", "group_id": "jevbench:hard-sol-b-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.03618888184428215, "true": 0.9638110995292664}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.9638110995292664, "record_seconds": 0.12496165186166763, "batch_seconds": 0.12496165186166763, "batch_records": 1, "timing_scope": "record", "brier": 1.8578636646270752, "nll": 3.3190033435821533} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-10", "group_id": "jevbench:hard-sol-b-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.38365882635116577, "true": 0.6163411736488342}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.6163411736488342, "record_seconds": 0.1369711086153984, "batch_seconds": 0.1369711086153984, "batch_records": 1, "timing_scope": "record", "brier": 0.7597528696060181, "nll": 0.9580016136169434} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-14", "group_id": "jevbench:hard-sol-b-judge_hard-14", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9871307015419006, "true": 0.012869289144873619}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9871307015419006, "record_seconds": 0.14614797197282314, "batch_seconds": 0.14614797197282314, "batch_records": 1, "timing_scope": "record", "brier": 0.00033123744651675224, "nll": 0.012952825054526329} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-15", "group_id": "jevbench:hard-sol-b-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.004688525106757879, "true": 0.9953115582466125}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9953115582466125, "record_seconds": 0.12166408076882362, "batch_seconds": 0.12166408076882362, "batch_records": 1, "timing_scope": "record", "brier": 4.396375152282417e-05, "nll": 0.004699466750025749} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-18", "group_id": "jevbench:hard-sol-b-judge_hard-18", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9769133925437927, "true": 0.023086629807949066}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9769133925437927, "record_seconds": 0.10904811974614859, "batch_seconds": 0.10904811974614859, "batch_records": 1, "timing_scope": "record", "brier": 0.0010659839026629925, "nll": 0.023357277736067772} |
| {"record_id": "jevbench:hard-sol-b-long_policy-01", "group_id": "jevbench:hard-sol-b-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "approve_meals_only", "probabilities": {"approve_all": 0.01751815155148506, "approve_lodging_only": 0.010265389457345009, "approve_meals_only": 0.960317075252533, "reject_entire_claim": 0.011899438686668873}, "none_candidate_id": null, "predicted_none": false, "label": "reject_entire_claim", "correct": false, "confidence": 0.960317075252533, "record_seconds": 2.8698698384687304, "batch_seconds": 2.8698698384687304, "batch_records": 1, "timing_scope": "record", "brier": 1.8989639282226562, "nll": 4.4312639236450195} |
| {"record_id": "jevbench:hard-sol-b-long_policy-02", "group_id": "jevbench:hard-sol-b-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "deny_current_export", "probabilities": {"analytics_owner_only": 0.0003200683568138629, "deny_current_export": 0.9919237494468689, "privacy_and_security": 0.006527015473693609, "privacy_only": 0.0012291526654735208}, "none_candidate_id": null, "predicted_none": false, "label": "deny_current_export", "correct": true, "confidence": 0.9919237494468689, "record_seconds": 2.7438852228224277, "batch_seconds": 2.7438852228224277, "batch_records": 1, "timing_scope": "record", "brier": 0.00010944101813947782, "nll": 0.008109040558338165} |
| {"record_id": "jevbench:hard-sol-b-long_policy-05", "group_id": "jevbench:hard-sol-b-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "delete_only_profile", "probabilities": {"delete_everything": 0.0005361989024095237, "delete_nothing": 0.0008581121801398695, "delete_only_profile": 0.8822453022003174, "delete_profile_omar_april_and_table": 0.11636041104793549}, "none_candidate_id": null, "predicted_none": false, "label": "delete_profile_omar_april_and_table", "correct": false, "confidence": 0.8822453022003174, "record_seconds": 2.7069265274330974, "batch_seconds": 2.7069265274330974, "batch_records": 1, "timing_scope": "record", "brier": 1.5591766834259033, "nll": 2.1510629653930664} |
| {"record_id": "jevbench:hard-sol-b-long_policy-06", "group_id": "jevbench:hard-sol-b-long_policy-06", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "east_ward", "probabilities": {"east_ward": 0.6499592661857605, "harbor_neighbors": 0.008772952482104301, "no_award": 0.21995297074317932, "river_arts": 0.12131476402282715}, "none_candidate_id": null, "predicted_none": false, "label": "east_ward", "correct": true, "confidence": 0.6499592661857605, "record_seconds": 2.7162815425544977, "batch_seconds": 2.7162815425544977, "batch_records": 1, "timing_scope": "record", "brier": 0.18570205569267273, "nll": 0.43084558844566345} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-01", "group_id": "jevbench:hard-sol-b-routing_hard-01", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "office_file_repair", "probabilities": {"data_pipeline": 0.00010906495299423113, "desktop_support": 0.0002100882848026231, "finance_review": 0.00024126928474288434, "office_file_repair": 0.9987996816635132, "spreadsheet_formula": 0.0006399911362677813}, "none_candidate_id": null, "predicted_none": false, "label": "office_file_repair", "correct": true, "confidence": 0.9987996816635132, "record_seconds": 0.2959725260734558, "batch_seconds": 0.2959725260734558, "batch_records": 1, "timing_scope": "record", "brier": 1.964595867320895e-06, "nll": 0.0012010392965748906} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-02", "group_id": "jevbench:hard-sol-b-routing_hard-02", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "ledger_reconciliation", "probabilities": {"customer_support": 0.00020668929209932685, "fraud_review": 3.3257296308875084e-05, "ledger_reconciliation": 0.9993411898612976, "mobile_app": 0.0002938176621682942, "payment_disputes": 0.00012491321831475943}, "none_candidate_id": null, "predicted_none": false, "label": "ledger_reconciliation", "correct": true, "confidence": 0.9993411898612976, "record_seconds": 0.28843686170876026, "batch_seconds": 0.28843686170876026, "batch_records": 1, "timing_scope": "record", "brier": 5.797894573333906e-07, "nll": 0.0006590272532775998} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-03", "group_id": "jevbench:hard-sol-b-routing_hard-03", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "network_dns", "probabilities": {"api_platform": 0.00016113999299705029, "authoritative_dns": 0.00018350871687289327, "endpoint_devices": 0.0005206491914577782, "network_dns": 0.9989984631538391, "tls_certificates": 0.000136296046548523}, "none_candidate_id": null, "predicted_none": false, "label": "network_dns", "correct": true, "confidence": 0.9989984631538391, "record_seconds": 0.29022505413740873, "batch_seconds": 0.29022505413740873, "batch_records": 1, "timing_scope": "record", "brier": 1.352369736196124e-06, "nll": 0.0010020387126132846} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-07", "group_id": "jevbench:hard-sol-b-routing_hard-07", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "release_notarization", "probabilities": {"ci_infrastructure": 0.0001479423517594114, "code_signing": 0.0026833994779735804, "endpoint_security": 0.00029187099426053464, "macos_runtime": 0.0003216133627574891, "release_notarization": 0.9965552091598511}, "none_candidate_id": null, "predicted_none": false, "label": "release_notarization", "correct": true, "confidence": 0.9965552091598511, "record_seconds": 0.25923426635563374, "batch_seconds": 0.25923426635563374, "batch_records": 1, "timing_scope": "record", "brier": 1.927772791532334e-05, "nll": 0.003450737800449133} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-09", "group_id": "jevbench:hard-sol-b-routing_hard-09", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "source_access", "probabilities": {"git_client": 0.0002558950800448656, "identity_login": 7.680997077841312e-05, "repository_admin": 0.0002774069143924862, "source_access": 0.9992642998695374, "vpn_network": 0.00012556010915432125}, "none_candidate_id": null, "predicted_none": false, "label": "source_access", "correct": true, "confidence": 0.9992642998695374, "record_seconds": 0.25892690289765596, "batch_seconds": 0.25892690289765596, "batch_records": 1, "timing_scope": "record", "brier": 7.053566832837532e-07, "nll": 0.0007359709125012159} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-01", "group_id": "jevbench:hard-sol-b-temporal_numeric-01", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "within_window", "probabilities": {"before_open": 0.10135078430175781, "late_by_2h_or_more": 0.19995959103107452, "late_by_under_2h": 0.11032941937446594, "within_window": 0.5883601307868958}, "none_candidate_id": null, "predicted_none": false, "label": "late_by_under_2h", "correct": false, "confidence": 0.5883601307868958, "record_seconds": 0.2557181157171726, "batch_seconds": 0.2557181157171726, "batch_records": 1, "timing_scope": "record", "brier": 1.1879372596740723, "nll": 2.20428466796875} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-02", "group_id": "jevbench:hard-sol-b-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "under_100gb_below", "probabilities": {"at_least_100gb_below": 0.17190231382846832, "exactly_equal": 0.06473521888256073, "over_limit": 0.20967277884483337, "under_100gb_below": 0.5536896586418152}, "none_candidate_id": null, "predicted_none": false, "label": "over_limit", "correct": false, "confidence": 0.5536896586418152, "record_seconds": 0.2617741236463189, "batch_seconds": 0.2617741236463189, "batch_records": 1, "timing_scope": "record", "brier": 0.9649302959442139, "nll": 1.5622071027755737} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-03", "group_id": "jevbench:hard-sol-b-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "6_credits", "probabilities": {"12_credits": 0.22333276271820068, "15_credits": 0.19433312118053436, "6_credits": 0.3105340898036957, "9_credits": 0.27180007100105286}, "none_candidate_id": null, "predicted_none": false, "label": "9_credits", "correct": false, "confidence": 0.3105340898036957, "record_seconds": 0.19173468090593815, "batch_seconds": 0.19173468090593815, "batch_records": 1, "timing_scope": "record", "brier": 0.7143495082855225, "nll": 1.302688479423523} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-04", "group_id": "jevbench:hard-sol-b-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "narrow_fail", "probabilities": {"clear_fail": 0.3665792942047119, "narrow_fail": 0.6206615567207336, "pass": 0.012759244069457054}, "none_candidate_id": null, "predicted_none": false, "label": "pass", "correct": false, "confidence": 0.6206615567207336, "record_seconds": 0.183121875859797, "batch_seconds": 0.183121875859797, "batch_records": 1, "timing_scope": "record", "brier": 1.4942454099655151, "nll": 4.361499309539795} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-03", "group_id": "jevbench:hard-sol-c-judge_hard-03", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8147689700126648, "true": 0.185231015086174}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.8147689700126648, "record_seconds": 0.2316634440794587, "batch_seconds": 0.2316634440794587, "batch_records": 1, "timing_scope": "record", "brier": 0.06862106174230576, "nll": 0.2048506736755371} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-05", "group_id": "jevbench:hard-sol-c-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0800209492444992, "true": 0.9199790358543396}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9199790358543396, "record_seconds": 0.19232400506734848, "batch_seconds": 0.19232400506734848, "batch_records": 1, "timing_scope": "record", "brier": 0.012806707061827183, "nll": 0.08340439945459366} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-07", "group_id": "jevbench:hard-sol-c-judge_hard-07", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.34595680236816406, "true": 0.6540431976318359}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.6540431976318359, "record_seconds": 0.18073839507997036, "batch_seconds": 0.18073839507997036, "batch_records": 1, "timing_scope": "record", "brier": 0.8555449843406677, "nll": 1.0614413022994995} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-08", "group_id": "jevbench:hard-sol-c-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.3204328715801239, "true": 0.6795671582221985}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.6795671582221985, "record_seconds": 0.17925337329506874, "batch_seconds": 0.17925337329506874, "batch_records": 1, "timing_scope": "record", "brier": 0.9236229658126831, "nll": 1.138082504272461} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-09", "group_id": "jevbench:hard-sol-c-judge_hard-09", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.07322067022323608, "true": 0.9267793297767639}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9267793297767639, "record_seconds": 0.18899370171129704, "batch_seconds": 0.18899370171129704, "batch_records": 1, "timing_scope": "record", "brier": 0.010722532868385315, "nll": 0.07603979110717773} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-10", "group_id": "jevbench:hard-sol-c-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.08800455927848816, "true": 0.9119954705238342}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9119954705238342, "record_seconds": 0.22549306694418192, "batch_seconds": 0.22549306694418192, "batch_records": 1, "timing_scope": "record", "brier": 0.015489599667489529, "nll": 0.09212025254964828} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-11", "group_id": "jevbench:hard-sol-c-judge_hard-11", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9971049427986145, "true": 0.0028950294945389032}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9971049427986145, "record_seconds": 0.18775757309049368, "batch_seconds": 0.18775757309049368, "batch_records": 1, "timing_scope": "record", "brier": 1.6762551240390167e-05, "nll": 0.002899256069213152} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-13", "group_id": "jevbench:hard-sol-c-judge_hard-13", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.39238283038139343, "true": 0.6076171398162842}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.6076171398162842, "record_seconds": 0.18392246589064598, "batch_seconds": 0.18392246589064598, "batch_records": 1, "timing_scope": "record", "brier": 0.7383971810340881, "nll": 0.9355173110961914} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-15", "group_id": "jevbench:hard-sol-c-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.003637254936620593, "true": 0.9963627457618713}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9963627457618713, "record_seconds": 0.1616382533684373, "batch_seconds": 0.1616382533684373, "batch_records": 1, "timing_scope": "record", "brier": 2.645924178068526e-05, "nll": 0.0036438850220292807} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-07", "group_id": "jevbench:hard-sol-c-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "programme_review", "probabilities": {"partial_release": 0.022320209071040154, "programme_review": 0.93598473072052, "reject_draw": 0.024008963257074356, "release_draw": 0.017686089500784874}, "none_candidate_id": null, "predicted_none": false, "label": "programme_review", "correct": true, "confidence": 0.93598473072052, "record_seconds": 2.6441691145300865, "batch_seconds": 2.6441691145300865, "batch_records": 1, "timing_scope": "record", "brier": 0.0054853749461472034, "nll": 0.06615611910820007} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-09", "group_id": "jevbench:hard-sol-c-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "bus_substitution", "probabilities": {"bus_substitution": 0.9934013485908508, "call_fen_cross": 0.001531373243778944, "pass_fen_cross": 0.0014568301849067211, "terminate_oakmere": 0.0036104065366089344}, "none_candidate_id": null, "predicted_none": false, "label": "bus_substitution", "correct": true, "confidence": 0.9934013485908508, "record_seconds": 2.717715042643249, "batch_seconds": 2.717715042643249, "batch_records": 1, "timing_scope": "record", "brier": 6.10446950304322e-05, "nll": 0.006620518863201141} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-11", "group_id": "jevbench:hard-sol-c-multi_hop-11", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "coverage_excluded", "probabilities": {"adjuster_referral": 0.08371923863887787, "coverage_confirmed": 0.11251532286405563, "coverage_excluded": 0.6903890371322632, "documentation_needed": 0.1133764311671257}, "none_candidate_id": null, "predicted_none": false, "label": "coverage_excluded", "correct": true, "confidence": 0.6903890371322632, "record_seconds": 2.7830756586045027, "batch_seconds": 2.7830756586045027, "batch_records": 1, "timing_scope": "record", "brier": 0.12838177382946014, "nll": 0.37050002813339233} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-12", "group_id": "jevbench:hard-sol-c-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "conservation_hold", "probabilities": {"conservation_hold": 0.9935610294342041, "deny_request": 0.005790097173303366, "digital_copy_only": 0.0003241963859181851, "supervised_delivery": 0.0003246595151722431}, "none_candidate_id": null, "predicted_none": false, "label": "conservation_hold", "correct": true, "confidence": 0.9935610294342041, "record_seconds": 0.5395537242293358, "batch_seconds": 0.5395537242293358, "batch_records": 1, "timing_scope": "record", "brier": 7.519607606809586e-05, "nll": 0.0064597902819514275} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-13", "group_id": "jevbench:hard-sol-c-multi_hop-13", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "normal_release", "probabilities": {"engineering_review": 0.0007388157537207007, "normal_release": 0.9973672032356262, "rework": 0.0008478080853819847, "scrap_lot": 0.0010461816564202309}, "none_candidate_id": null, "predicted_none": false, "label": "normal_release", "correct": true, "confidence": 0.9973672032356262, "record_seconds": 2.70331441052258, "batch_seconds": 2.70331441052258, "batch_records": 1, "timing_scope": "record", "brier": 9.290742127632257e-06, "nll": 0.002636268734931946} |
|
|
