Invalid JSON:Unexpected non-whitespace character after JSONat line 2294, column 2
| { | |
| "model": "North-ML1/Aurora-Proelia", | |
| "checkpoint": "/home/arthur/ember-proelia/checkpoints/aurora-proelia-chatml-sft-strong-probe-20260815/final.pt", | |
| "runtime": "native AuroraForCausalLM", | |
| "device": "cuda", | |
| "seed": 20260815, | |
| "prompt_format": "Question: ...\\nAnswer:", | |
| "scoring": { | |
| "mmlu_arc": "conditional log-likelihood of answer letters A/B/C/D", | |
| "hellaswag": "conditional log-likelihood of each ending", | |
| "gsm8k": "greedy generation; last extracted number exact-match" | |
| }, | |
| "warning": "These are transparent local slices of public benchmark test/validation splits, not official full leaderboard evaluations.", | |
| "elapsed_seconds": 128.61, | |
| "benchmarks": { | |
| "mmlu": { | |
| "dataset": "cais/mmlu", | |
| "config": "all", | |
| "split": "test", | |
| "sample_rule": "first 1 rows per subject, sorted by subject", | |
| "n": 57, | |
| "correct": 16, | |
| "accuracy": 0.2807017543859649, | |
| "items": [ | |
| { | |
| "id": "abstract_algebra", | |
| "gold": 1, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -11.065407752990723, | |
| -13.722344398498535, | |
| -13.820038795471191, | |
| -8.746415138244629 | |
| ] | |
| }, | |
| { | |
| "id": "anatomy", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -8.992938995361328, | |
| -12.293392181396484, | |
| -11.663692474365234, | |
| -10.392932891845703 | |
| ] | |
| }, | |
| { | |
| "id": "astronomy", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -15.312116622924805, | |
| -16.695585250854492, | |
| -16.407697677612305, | |
| -15.772703170776367 | |
| ] | |
| }, | |
| { | |
| "id": "business_ethics", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -8.371504783630371, | |
| -11.833579063415527, | |
| -5.9145731925964355, | |
| -10.86881160736084 | |
| ] | |
| }, | |
| { | |
| "id": "clinical_knowledge", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -13.617753028869629, | |
| -13.94362735748291, | |
| -14.37102222442627, | |
| -13.762635231018066 | |
| ] | |
| }, | |
| { | |
| "id": "college_biology", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.839444637298584, | |
| -9.998029708862305, | |
| -10.59827995300293, | |
| -9.899320602416992 | |
| ] | |
| }, | |
| { | |
| "id": "college_chemistry", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.491821765899658, | |
| -9.227158546447754, | |
| -7.884720325469971, | |
| -10.37416934967041 | |
| ] | |
| }, | |
| { | |
| "id": "college_computer_science", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -13.934694290161133, | |
| -15.274904251098633, | |
| -14.76298713684082, | |
| -14.667703628540039 | |
| ] | |
| }, | |
| { | |
| "id": "college_mathematics", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -9.38847541809082, | |
| -8.894182205200195, | |
| -7.576257705688477, | |
| -11.495370864868164 | |
| ] | |
| }, | |
| { | |
| "id": "college_medicine", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.948405742645264, | |
| -9.367794036865234, | |
| -9.104778289794922, | |
| -12.021099090576172 | |
| ] | |
| }, | |
| { | |
| "id": "college_physics", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.774142742156982, | |
| -12.240840911865234, | |
| -10.31619644165039, | |
| -10.80251693725586 | |
| ] | |
| }, | |
| { | |
| "id": "computer_security", | |
| "gold": 2, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -9.164381980895996, | |
| -8.878569602966309, | |
| -9.307784080505371, | |
| -9.142569541931152 | |
| ] | |
| }, | |
| { | |
| "id": "conceptual_physics", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -10.650524139404297, | |
| -16.48568344116211, | |
| -17.593189239501953, | |
| -14.196720123291016 | |
| ] | |
| }, | |
| { | |
| "id": "econometrics", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -11.667923927307129, | |
| -15.554291725158691, | |
| -13.543992042541504, | |
| -15.381836891174316 | |
| ] | |
| }, | |
| { | |
| "id": "electrical_engineering", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -13.507604598999023, | |
| -11.796621322631836, | |
| -11.196439743041992, | |
| -12.321866989135742 | |
| ] | |
| }, | |
| { | |
| "id": "elementary_mathematics", | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -9.177167892456055, | |
| -11.000051498413086, | |
| -9.727285385131836, | |
| -8.292234420776367 | |
| ] | |
| }, | |
| { | |
| "id": "formal_logic", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -8.854928970336914, | |
| -10.767251968383789, | |
| -10.492792129516602, | |
| -11.700159072875977 | |
| ] | |
| }, | |
| { | |
| "id": "global_facts", | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -9.921621322631836, | |
| -12.06275749206543, | |
| -11.628408432006836, | |
| -9.798643112182617 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_biology", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -8.379108428955078, | |
| -10.377635955810547, | |
| -9.365474700927734, | |
| -9.146793365478516 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_chemistry", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -12.899845123291016, | |
| -14.593929290771484, | |
| -12.671749114990234, | |
| -13.124927520751953 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_computer_science", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -13.954312324523926, | |
| -17.87796401977539, | |
| -14.44018268585205, | |
| -16.66739273071289 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_european_history", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -8.392210960388184, | |
| -10.337569236755371, | |
| -8.25827693939209, | |
| -10.247694969177246 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_geography", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -12.850770950317383, | |
| -15.290803909301758, | |
| -13.659631729125977, | |
| -15.022645950317383 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_government_and_politics", | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -15.277093887329102, | |
| -16.450571060180664, | |
| -14.606401443481445, | |
| -13.837526321411133 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_macroeconomics", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -6.6448845863342285, | |
| -7.94866418838501, | |
| -7.96520471572876, | |
| -7.020662784576416 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_mathematics", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -9.252296447753906, | |
| -9.586601257324219, | |
| -8.409431457519531, | |
| -8.650627136230469 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_microeconomics", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -14.495097160339355, | |
| -14.228175163269043, | |
| -13.366610527038574, | |
| -13.880999565124512 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_physics", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.188655853271484, | |
| -12.201091766357422, | |
| -7.940624237060547, | |
| -10.224483489990234 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_psychology", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -8.603910446166992, | |
| -13.654096603393555, | |
| -12.107229232788086, | |
| -14.099546432495117 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_statistics", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -8.421165466308594, | |
| -9.866905212402344, | |
| -7.0621185302734375, | |
| -9.383644104003906 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_us_history", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -9.264945983886719, | |
| -10.157363891601562, | |
| -6.800559997558594, | |
| -8.552444458007812 | |
| ] | |
| }, | |
| { | |
| "id": "high_school_world_history", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.331644058227539, | |
| -10.24443244934082, | |
| -9.055864334106445, | |
| -9.592950820922852 | |
| ] | |
| }, | |
| { | |
| "id": "human_aging", | |
| "gold": 0, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -11.297323226928711, | |
| -13.392545700073242, | |
| -13.670263290405273, | |
| -10.73957633972168 | |
| ] | |
| }, | |
| { | |
| "id": "human_sexuality", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.16053581237793, | |
| -12.549161911010742, | |
| -11.716474533081055, | |
| -11.937253952026367 | |
| ] | |
| }, | |
| { | |
| "id": "international_law", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -12.278502464294434, | |
| -14.08774471282959, | |
| -12.842947959899902, | |
| -14.707045555114746 | |
| ] | |
| }, | |
| { | |
| "id": "jurisprudence", | |
| "gold": 3, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -17.552677154541016, | |
| -16.74979019165039, | |
| -18.41897964477539, | |
| -17.30539321899414 | |
| ] | |
| }, | |
| { | |
| "id": "logical_fallacies", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -6.788644313812256, | |
| -10.431733131408691, | |
| -9.424210548400879, | |
| -10.552140235900879 | |
| ] | |
| }, | |
| { | |
| "id": "machine_learning", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.600849151611328, | |
| -11.468860626220703, | |
| -12.78879165649414, | |
| -11.993694305419922 | |
| ] | |
| }, | |
| { | |
| "id": "management", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -6.842371463775635, | |
| -7.841776371002197, | |
| -7.36474084854126, | |
| -7.551119327545166 | |
| ] | |
| }, | |
| { | |
| "id": "marketing", | |
| "gold": 1, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -12.726282119750977, | |
| -15.85038948059082, | |
| -13.401376724243164, | |
| -12.384675979614258 | |
| ] | |
| }, | |
| { | |
| "id": "medical_genetics", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -13.769607543945312, | |
| -16.369468688964844, | |
| -13.299087524414062, | |
| -13.802482604980469 | |
| ] | |
| }, | |
| { | |
| "id": "miscellaneous", | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -7.125714302062988, | |
| -8.332486152648926, | |
| -7.317517280578613, | |
| -5.525883674621582 | |
| ] | |
| }, | |
| { | |
| "id": "moral_disputes", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.763257026672363, | |
| -13.08532428741455, | |
| -9.438855171203613, | |
| -11.360783576965332 | |
| ] | |
| }, | |
| { | |
| "id": "moral_scenarios", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.405111312866211, | |
| -10.830595016479492, | |
| -9.96485710144043, | |
| -11.952451705932617 | |
| ] | |
| }, | |
| { | |
| "id": "nutrition", | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -12.171109199523926, | |
| -11.023301124572754, | |
| -11.43376636505127, | |
| -10.981022834777832 | |
| ] | |
| }, | |
| { | |
| "id": "philosophy", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -9.276885986328125, | |
| -16.04876708984375, | |
| -14.734336853027344, | |
| -15.995231628417969 | |
| ] | |
| }, | |
| { | |
| "id": "prehistory", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -15.24170207977295, | |
| -16.234783172607422, | |
| -11.760371208190918, | |
| -16.808399200439453 | |
| ] | |
| }, | |
| { | |
| "id": "professional_accounting", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -7.388260841369629, | |
| -8.36519718170166, | |
| -6.917466163635254, | |
| -8.646897315979004 | |
| ] | |
| }, | |
| { | |
| "id": "professional_law", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -9.46272087097168, | |
| -11.342100143432617, | |
| -8.988119125366211, | |
| -11.000295639038086 | |
| ] | |
| }, | |
| { | |
| "id": "professional_medicine", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -9.14848518371582, | |
| -10.542520523071289, | |
| -7.1248040199279785, | |
| -9.842691421508789 | |
| ] | |
| }, | |
| { | |
| "id": "professional_psychology", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -9.68390941619873, | |
| -13.24897289276123, | |
| -11.055384635925293, | |
| -12.5396146774292 | |
| ] | |
| }, | |
| { | |
| "id": "public_relations", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.99325180053711, | |
| -13.374870300292969, | |
| -13.460803985595703, | |
| -13.733356475830078 | |
| ] | |
| }, | |
| { | |
| "id": "security_studies", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -11.936192512512207, | |
| -13.596524238586426, | |
| -12.959683418273926, | |
| -14.541020393371582 | |
| ] | |
| }, | |
| { | |
| "id": "sociology", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -11.086275100708008, | |
| -14.300546646118164, | |
| -13.146585464477539, | |
| -12.826700210571289 | |
| ] | |
| }, | |
| { | |
| "id": "us_foreign_policy", | |
| "gold": 0, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -11.321792602539062, | |
| -12.469329833984375, | |
| -10.786674499511719, | |
| -8.848953247070312 | |
| ] | |
| }, | |
| { | |
| "id": "virology", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -14.737914085388184, | |
| -19.15251922607422, | |
| -16.356002807617188, | |
| -16.827438354492188 | |
| ] | |
| }, | |
| { | |
| "id": "world_religions", | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -14.619362831115723, | |
| -15.485116004943848, | |
| -14.994767189025879, | |
| -12.115769386291504 | |
| ] | |
| } | |
| ] | |
| }, | |
| "arc_challenge": { | |
| "dataset": "allenai/ai2_arc", | |
| "config": "ARC-Challenge", | |
| "split": "test", | |
| "sample_rule": "first 50 rows", | |
| "n": 50, | |
| "correct": 15, | |
| "accuracy": 0.3, | |
| "items": [ | |
| { | |
| "id": "Mercury_7175875", | |
| "gold": 2, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -13.257099151611328, | |
| -12.892284393310547, | |
| -13.339046478271484, | |
| -13.150699615478516 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_409171", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -5.607930660247803, | |
| -9.590680122375488, | |
| -9.437291145324707, | |
| -12.3324556350708 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_408547", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -11.540667533874512, | |
| -13.786578178405762, | |
| -11.154475212097168, | |
| -14.782282829284668 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_407327", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -6.625519275665283, | |
| -9.365859985351562, | |
| -7.808945178985596, | |
| -8.823867797851562 | |
| ] | |
| }, | |
| { | |
| "id": "MCAS_2006_9_44", | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -7.209028720855713, | |
| -7.654547214508057, | |
| -6.35488748550415, | |
| -9.375509262084961 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7270393", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -6.907016754150391, | |
| -8.797840118408203, | |
| -6.689937591552734, | |
| -7.817386627197266 | |
| ] | |
| }, | |
| { | |
| "id": "MCAS_2014_5_7", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.3675217628479, | |
| -11.022520065307617, | |
| -10.54194450378418, | |
| -13.059621810913086 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7086660", | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -12.918753623962402, | |
| -13.41837215423584, | |
| -12.168532371520996, | |
| -12.031256675720215 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7168805", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -11.670303344726562, | |
| -11.43414306640625, | |
| -10.112014770507812, | |
| -11.487091064453125 | |
| ] | |
| }, | |
| { | |
| "id": "MCAS_2003_8_11", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -13.72336483001709, | |
| -16.018110275268555, | |
| -10.757201194763184, | |
| -14.797911643981934 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7250058", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.975772857666016, | |
| -10.70980453491211, | |
| -10.912708282470703, | |
| -12.006580352783203 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7012740", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -12.445462226867676, | |
| -15.087506294250488, | |
| -13.499478340148926, | |
| -14.967564582824707 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_LBS10610", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -10.429519653320312, | |
| -12.155929565429688, | |
| -9.80712890625, | |
| -9.881828308105469 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_407400", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.77410364151001, | |
| -11.508357048034668, | |
| -9.733233451843262, | |
| -11.378230094909668 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7212993", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -13.05376148223877, | |
| -15.286519050598145, | |
| -12.013554573059082, | |
| -14.143208503723145 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_413240", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -17.878520965576172, | |
| -21.556209564208984, | |
| -19.954063415527344, | |
| -18.31875991821289 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7186358", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.592679977416992, | |
| -11.186834335327148, | |
| -10.437093734741211, | |
| -10.743894577026367 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7166425", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -13.151106834411621, | |
| -14.948561668395996, | |
| -12.550688743591309, | |
| -14.81430721282959 | |
| ] | |
| }, | |
| { | |
| "id": "MDSA_2007_8_3", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.31587028503418, | |
| -11.113393783569336, | |
| -10.048635482788086, | |
| -10.456602096557617 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7094290", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -12.274971961975098, | |
| -14.554680824279785, | |
| -12.17855167388916, | |
| -13.716797828674316 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7186568", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.662038803100586, | |
| -13.914113998413086, | |
| -10.246809005737305, | |
| -13.753362655639648 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_402216", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.382719993591309, | |
| -11.716048240661621, | |
| -8.407286643981934, | |
| -10.41538143157959 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_404894", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -6.746318817138672, | |
| -8.517719268798828, | |
| -5.736660003662109, | |
| -7.205410003662109 | |
| ] | |
| }, | |
| { | |
| "id": "MCAS_2002_8_11", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -9.341812133789062, | |
| -11.586677551269531, | |
| -9.9390869140625, | |
| -10.084205627441406 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_405086", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -12.923977851867676, | |
| -14.6491060256958, | |
| -13.37151050567627, | |
| -13.287083625793457 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_408324", | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -15.907831192016602, | |
| -17.123109817504883, | |
| -14.713815689086914, | |
| -14.527193069458008 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7218820", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -8.5324068069458, | |
| -11.384137153625488, | |
| -8.22386646270752, | |
| -10.3728609085083 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_412202", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -14.737939834594727, | |
| -17.706117630004883, | |
| -18.98332405090332, | |
| -18.239458084106445 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_409139", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -12.48038387298584, | |
| -16.033302307128906, | |
| -13.108176231384277, | |
| -14.910483360290527 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_400687", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -12.036847114562988, | |
| -13.12477970123291, | |
| -11.944828987121582, | |
| -13.722100257873535 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7171605", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -12.876701354980469, | |
| -16.208396911621094, | |
| -14.424369812011719, | |
| -14.705284118652344 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7210245", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -13.27576732635498, | |
| -17.60330581665039, | |
| -12.7488431930542, | |
| -14.3162260055542 | |
| ] | |
| }, | |
| { | |
| "id": "AKDE&ED_2008_4_25", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -10.140519142150879, | |
| -11.45272159576416, | |
| -9.074936866760254, | |
| -10.698594093322754 | |
| ] | |
| }, | |
| { | |
| "id": "AKDE&ED_2008_4_19", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.919215202331543, | |
| -15.9137601852417, | |
| -15.312006950378418, | |
| -16.290857315063477 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_400402", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -7.694456577301025, | |
| -11.839841842651367, | |
| -10.010038375854492, | |
| -10.61762809753418 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7234308", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -8.998245239257812, | |
| -13.028724670410156, | |
| -11.51690673828125, | |
| -11.057670593261719 | |
| ] | |
| }, | |
| { | |
| "id": "ACTAAP_2014_5_8", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -9.374757766723633, | |
| -10.552026748657227, | |
| -11.106287002563477, | |
| -11.348367691040039 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_400407", | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -11.040645599365234, | |
| -11.55923843383789, | |
| -9.535724639892578, | |
| -11.937259674072266 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7116288", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -8.690303802490234, | |
| -10.50320053100586, | |
| -6.748226642608643, | |
| -10.19913101196289 | |
| ] | |
| }, | |
| { | |
| "id": "MCAS_2004_9_15-v1", | |
| "gold": 1, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.780317306518555, | |
| -12.197942733764648, | |
| -11.332799911499023, | |
| -11.70692253112793 | |
| ] | |
| }, | |
| { | |
| "id": "NYSEDREGENTS_2015_4_26", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -7.854938507080078, | |
| -11.25619888305664, | |
| -10.267627716064453, | |
| -11.087566375732422 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_401620", | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -11.677645683288574, | |
| -10.38996410369873, | |
| -9.348132133483887, | |
| -12.03480052947998 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_400877", | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -13.356064796447754, | |
| -16.007526397705078, | |
| -11.087029457092285, | |
| -12.589570045471191 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7174213", | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.955911636352539, | |
| -15.54490852355957, | |
| -11.857767105102539, | |
| -13.05388069152832 | |
| ] | |
| }, | |
| { | |
| "id": "NYSEDREGENTS_2008_8_34", | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -8.583789825439453, | |
| -8.559322357177734, | |
| -10.30838394165039, | |
| -8.716968536376953 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_7212398", | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -12.839554786682129, | |
| -12.759804725646973, | |
| -13.434563636779785, | |
| -13.291665077209473 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_401290", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -6.22521448135376, | |
| -7.722757816314697, | |
| -8.017831802368164, | |
| -8.484460830688477 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_402120", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -8.452454566955566, | |
| -11.386002540588379, | |
| -11.028359413146973, | |
| -12.88771915435791 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_184975", | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -10.78169059753418, | |
| -14.064840316772461, | |
| -10.904233932495117, | |
| -12.338041305541992 | |
| ] | |
| }, | |
| { | |
| "id": "Mercury_SC_400578", | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -9.3740234375, | |
| -12.418960571289062, | |
| -11.314460754394531, | |
| -12.545074462890625 | |
| ] | |
| } | |
| ] | |
| }, | |
| "hellaswag": { | |
| "dataset": "Rowan/hellaswag", | |
| "split": "validation", | |
| "sample_rule": "first 50 rows", | |
| "n": 50, | |
| "correct": 19, | |
| "accuracy": 0.38, | |
| "items": [ | |
| { | |
| "id": 24, | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -58.97417068481445, | |
| -56.32265853881836, | |
| -35.89630126953125, | |
| -46.27552795410156 | |
| ] | |
| }, | |
| { | |
| "id": 92, | |
| "gold": 3, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -52.01768493652344, | |
| -41.43523406982422, | |
| -49.460594177246094, | |
| -64.71932983398438 | |
| ] | |
| }, | |
| { | |
| "id": 106, | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -84.59806823730469, | |
| -81.92660522460938, | |
| -47.8038330078125, | |
| -152.38461303710938 | |
| ] | |
| }, | |
| { | |
| "id": 114, | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -38.21574401855469, | |
| -43.240745544433594, | |
| -53.74546813964844, | |
| -48.1297607421875 | |
| ] | |
| }, | |
| { | |
| "id": 116, | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -35.632606506347656, | |
| -30.693992614746094, | |
| -40.05568313598633, | |
| -37.1392707824707 | |
| ] | |
| }, | |
| { | |
| "id": 117, | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -45.845428466796875, | |
| -64.99568176269531, | |
| -29.720195770263672, | |
| -72.37071990966797 | |
| ] | |
| }, | |
| { | |
| "id": 149, | |
| "gold": 2, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -53.611968994140625, | |
| -41.80815124511719, | |
| -72.14735412597656, | |
| -48.25930404663086 | |
| ] | |
| }, | |
| { | |
| "id": 170, | |
| "gold": 0, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -101.25760650634766, | |
| -94.36492919921875, | |
| -70.56925964355469, | |
| -64.97718811035156 | |
| ] | |
| }, | |
| { | |
| "id": 180, | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -165.63174438476562, | |
| -69.18826293945312, | |
| -37.464599609375, | |
| -73.97086334228516 | |
| ] | |
| }, | |
| { | |
| "id": 182, | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -86.95623779296875, | |
| -60.08461380004883, | |
| -110.89420318603516, | |
| -108.59903717041016 | |
| ] | |
| }, | |
| { | |
| "id": 185, | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -63.292137145996094, | |
| -89.58827209472656, | |
| -44.102386474609375, | |
| -65.39309692382812 | |
| ] | |
| }, | |
| { | |
| "id": 186, | |
| "gold": 3, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -55.55475997924805, | |
| -91.25175476074219, | |
| -47.01008987426758, | |
| -98.15768432617188 | |
| ] | |
| }, | |
| { | |
| "id": 187, | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -110.35063171386719, | |
| -70.6715087890625, | |
| -49.93235397338867, | |
| -72.15544128417969 | |
| ] | |
| }, | |
| { | |
| "id": 188, | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -81.07919311523438, | |
| -117.24858093261719, | |
| -71.77798461914062, | |
| -72.8318862915039 | |
| ] | |
| }, | |
| { | |
| "id": 192, | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -60.681610107421875, | |
| -61.50605010986328, | |
| -77.38455963134766, | |
| -65.66329956054688 | |
| ] | |
| }, | |
| { | |
| "id": 200, | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -102.02816009521484, | |
| -108.43778991699219, | |
| -126.33024597167969, | |
| -73.75164794921875 | |
| ] | |
| }, | |
| { | |
| "id": 225, | |
| "gold": 2, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -79.86792755126953, | |
| -61.519248962402344, | |
| -87.25880432128906, | |
| -64.83087158203125 | |
| ] | |
| }, | |
| { | |
| "id": 245, | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -96.00947570800781, | |
| -120.17504119873047, | |
| -78.04824829101562, | |
| -100.17385864257812 | |
| ] | |
| }, | |
| { | |
| "id": 246, | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -78.3630142211914, | |
| -63.59217834472656, | |
| -64.74330139160156, | |
| -102.46669006347656 | |
| ] | |
| }, | |
| { | |
| "id": 247, | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -96.45269775390625, | |
| -82.45916748046875, | |
| -74.95116424560547, | |
| -91.07105255126953 | |
| ] | |
| }, | |
| { | |
| "id": 282, | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -75.71699523925781, | |
| -50.46632385253906, | |
| -74.38658905029297, | |
| -116.06935119628906 | |
| ] | |
| }, | |
| { | |
| "id": 354, | |
| "gold": 0, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -56.19131851196289, | |
| -62.31458282470703, | |
| -88.37478637695312, | |
| -50.45991516113281 | |
| ] | |
| }, | |
| { | |
| "id": 378, | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -43.91583251953125, | |
| -101.2091293334961, | |
| -62.915958404541016, | |
| -41.873687744140625 | |
| ] | |
| }, | |
| { | |
| "id": 380, | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -56.52827835083008, | |
| -78.88848876953125, | |
| -78.91925811767578, | |
| -48.379554748535156 | |
| ] | |
| }, | |
| { | |
| "id": 385, | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -89.74951171875, | |
| -83.21479797363281, | |
| -76.45021057128906, | |
| -127.01903533935547 | |
| ] | |
| }, | |
| { | |
| "id": 386, | |
| "gold": 3, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -190.13619995117188, | |
| -77.71086120605469, | |
| -106.107666015625, | |
| -111.29759216308594 | |
| ] | |
| }, | |
| { | |
| "id": 393, | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -54.82701110839844, | |
| -80.93632507324219, | |
| -109.81082153320312, | |
| -64.51954650878906 | |
| ] | |
| }, | |
| { | |
| "id": 398, | |
| "gold": 3, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -110.66639709472656, | |
| -64.37057495117188, | |
| -94.86198425292969, | |
| -77.98493957519531 | |
| ] | |
| }, | |
| { | |
| "id": 399, | |
| "gold": 1, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -57.262474060058594, | |
| -48.567928314208984, | |
| -75.16720581054688, | |
| -47.18919372558594 | |
| ] | |
| }, | |
| { | |
| "id": 400, | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -83.77692413330078, | |
| -84.8166275024414, | |
| -71.52482604980469, | |
| -70.779296875 | |
| ] | |
| }, | |
| { | |
| "id": 402, | |
| "gold": 1, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -59.79026794433594, | |
| -52.81324768066406, | |
| -38.623291015625, | |
| -65.29905700683594 | |
| ] | |
| }, | |
| { | |
| "id": 478, | |
| "gold": 0, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -51.93339920043945, | |
| -35.12632751464844, | |
| -68.02326202392578, | |
| -54.45841979980469 | |
| ] | |
| }, | |
| { | |
| "id": 479, | |
| "gold": 1, | |
| "predicted": 1, | |
| "correct": true, | |
| "scores": [ | |
| -118.08658599853516, | |
| -33.16276168823242, | |
| -88.72938537597656, | |
| -156.5254669189453 | |
| ] | |
| }, | |
| { | |
| "id": 491, | |
| "gold": 2, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -90.12571716308594, | |
| -52.412841796875, | |
| -80.73646545410156, | |
| -98.206787109375 | |
| ] | |
| }, | |
| { | |
| "id": 492, | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -93.69236755371094, | |
| -97.57264709472656, | |
| -70.7070541381836, | |
| -87.77411651611328 | |
| ] | |
| }, | |
| { | |
| "id": 493, | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -119.6843032836914, | |
| -61.70817947387695, | |
| -40.689002990722656, | |
| -89.13894653320312 | |
| ] | |
| }, | |
| { | |
| "id": 503, | |
| "gold": 3, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -71.94010925292969, | |
| -46.83222579956055, | |
| -102.84877014160156, | |
| -121.54273986816406 | |
| ] | |
| }, | |
| { | |
| "id": 527, | |
| "gold": 2, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -47.260093688964844, | |
| -57.814613342285156, | |
| -84.59222412109375, | |
| -51.496944427490234 | |
| ] | |
| }, | |
| { | |
| "id": 529, | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -63.52226638793945, | |
| -61.843055725097656, | |
| -60.18296432495117, | |
| -78.32892608642578 | |
| ] | |
| }, | |
| { | |
| "id": 560, | |
| "gold": 1, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -99.75664520263672, | |
| -95.23532104492188, | |
| -113.17375183105469, | |
| -91.73249053955078 | |
| ] | |
| }, | |
| { | |
| "id": 586, | |
| "gold": 2, | |
| "predicted": 2, | |
| "correct": true, | |
| "scores": [ | |
| -143.83102416992188, | |
| -99.50735473632812, | |
| -64.4426040649414, | |
| -72.21981811523438 | |
| ] | |
| }, | |
| { | |
| "id": 604, | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -96.4774398803711, | |
| -221.2063446044922, | |
| -158.43115234375, | |
| -81.52798461914062 | |
| ] | |
| }, | |
| { | |
| "id": 605, | |
| "gold": 3, | |
| "predicted": 3, | |
| "correct": true, | |
| "scores": [ | |
| -92.68693542480469, | |
| -68.42957305908203, | |
| -188.98281860351562, | |
| -64.8801498413086 | |
| ] | |
| }, | |
| { | |
| "id": 606, | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -89.72834777832031, | |
| -85.41148376464844, | |
| -190.0895233154297, | |
| -83.48924255371094 | |
| ] | |
| }, | |
| { | |
| "id": 615, | |
| "gold": 0, | |
| "predicted": 2, | |
| "correct": false, | |
| "scores": [ | |
| -81.62350463867188, | |
| -109.8926010131836, | |
| -74.27828979492188, | |
| -110.36332702636719 | |
| ] | |
| }, | |
| { | |
| "id": 616, | |
| "gold": 0, | |
| "predicted": 1, | |
| "correct": false, | |
| "scores": [ | |
| -138.66162109375, | |
| -70.92147827148438, | |
| -96.69862365722656, | |
| -129.30487060546875 | |
| ] | |
| }, | |
| { | |
| "id": 618, | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -54.799461364746094, | |
| -70.267333984375, | |
| -92.53612518310547, | |
| -183.44483947753906 | |
| ] | |
| }, | |
| { | |
| "id": 619, | |
| "gold": 0, | |
| "predicted": 0, | |
| "correct": true, | |
| "scores": [ | |
| -61.755313873291016, | |
| -82.50668334960938, | |
| -90.70120239257812, | |
| -72.40680694580078 | |
| ] | |
| }, | |
| { | |
| "id": 689, | |
| "gold": 3, | |
| "predicted": 0, | |
| "correct": false, | |
| "scores": [ | |
| -30.048839569091797, | |
| -51.66824722290039, | |
| -49.33673095703125, | |
| -47.256568908691406 | |
| ] | |
| }, | |
| { | |
| "id": 708, | |
| "gold": 2, | |
| "predicted": 3, | |
| "correct": false, | |
| "scores": [ | |
| -115.7948989868164, | |
| -117.76850891113281, | |
| -121.9887466430664, | |
| -85.19407653808594 | |
| ] | |
| } | |
| ] | |
| }, | |
| "gsm8k": { | |
| "dataset": "openai/gsm8k", | |
| "config": "main", | |
| "split": "test", | |
| "sample_rule": "first 50 rows", | |
| "n": 50, | |
| "correct": 0, | |
| "accuracy": 0.0, | |
| "items": [ | |
| { | |
| "id": 0, | |
| "gold": "18", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "I can answer this question with the help of an equation." | |
| }, | |
| { | |
| "id": 1, | |
| "gold": "3", | |
| "predicted": "2", | |
| "correct": false, | |
| "response": "There are 2 bolts in total." | |
| }, | |
| { | |
| "id": 2, | |
| "gold": "70000", | |
| "predicted": "150", | |
| "correct": false, | |
| "response": "Josh decides to try flipping a house. He buys a house for $80,000 and then puts in $50,000 in repairs. This increased the value of the house by 150%. How much profit did he make?" | |
| }, | |
| { | |
| "id": 3, | |
| "gold": "540", | |
| "predicted": "60", | |
| "correct": false, | |
| "response": "James decides to run 3 sprints 3 times a week. He runs 60 meters each sprint. How many total meters does he run a week?" | |
| }, | |
| { | |
| "id": 4, | |
| "gold": "20", | |
| "predicted": "15", | |
| "correct": false, | |
| "response": "I need to give her chickens 15 cups of feed." | |
| }, | |
| { | |
| "id": 5, | |
| "gold": "64", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "Kylar needs to pay for each glass." | |
| }, | |
| { | |
| "id": 6, | |
| "gold": "260", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "Toulouse has twice as many sheep as Charleston. Charleston" | |
| }, | |
| { | |
| "id": 7, | |
| "gold": "160", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "She needs to restart the download from the beginning." | |
| }, | |
| { | |
| "id": 8, | |
| "gold": "45", | |
| "predicted": "2", | |
| "correct": false, | |
| "response": "John drives for 3 hours at a speed of 60 mph and then turns around because he realizes he forgot something very important at home. He tries to get home in 4 hours but spends the first 2 hours in standstill traffic. He spends the next half-hour driving at a speed of" | |
| }, | |
| { | |
| "id": 9, | |
| "gold": "460", | |
| "predicted": "10", | |
| "correct": false, | |
| "response": "$10." | |
| }, | |
| { | |
| "id": 10, | |
| "gold": "366", | |
| "predicted": "60", | |
| "correct": false, | |
| "response": "The program had 60 downloads in the first month. The number of downloads in" | |
| }, | |
| { | |
| "id": 11, | |
| "gold": "694", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "The total cost is the sum of the costs of the pastries and the costs of the" | |
| }, | |
| { | |
| "id": 12, | |
| "gold": "13", | |
| "predicted": "3", | |
| "correct": false, | |
| "response": "I will tell you the answer is $3." | |
| }, | |
| { | |
| "id": 13, | |
| "gold": "18", | |
| "predicted": "2", | |
| "correct": false, | |
| "response": "2." | |
| }, | |
| { | |
| "id": 14, | |
| "gold": "60", | |
| "predicted": "25", | |
| "correct": false, | |
| "response": "In a dance class of 20 students, 20% enrolled in contemporary dance, 25% of the remaining enrolled in jazz dance, and the rest enrolled in hip-hop dance." | |
| }, | |
| { | |
| "id": 15, | |
| "gold": "125", | |
| "predicted": "8000", | |
| "correct": false, | |
| "response": "The merchant would make a choice of purchase between 2 purchase plans: jewelry worth $5,000 or electronic gadgets worth $8,000. His financial advisor speculates that the jewelry" | |
| }, | |
| { | |
| "id": 16, | |
| "gold": "230", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "The distance covered by each train in the two days is the same." | |
| }, | |
| { | |
| "id": 17, | |
| "gold": "57500", | |
| "predicted": "15", | |
| "correct": false, | |
| "response": "Jill gets paid $20 per hour to teach and $30 to be a cheerleading coach. If she works 50 weeks a year, 35 hours a week as a teacher and 15 hours a week as" | |
| }, | |
| { | |
| "id": 18, | |
| "gold": "7", | |
| "predicted": "4", | |
| "correct": false, | |
| "response": "I want to know how many eggs will I eat in 4 weeks." | |
| }, | |
| { | |
| "id": 19, | |
| "gold": "6", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "Hello! I am Marissa." | |
| }, | |
| { | |
| "id": 20, | |
| "gold": "15", | |
| "predicted": "24", | |
| "correct": false, | |
| "response": "I have 10 liters of orange drink that are two-thirds water and I wish to add it to 15 liters of pineapple drink that is three-fifths water. But as I pour it, I spill one liter of the orange drink. How much water is in the remaining 24 liters?" | |
| }, | |
| { | |
| "id": 21, | |
| "gold": "14", | |
| "predicted": "1947", | |
| "correct": false, | |
| "response": "In 1947, Samantha was born." | |
| }, | |
| { | |
| "id": 22, | |
| "gold": "7", | |
| "predicted": "2", | |
| "correct": false, | |
| "response": "Billy sells DVDs. He has 8 customers on Tuesday. His first 3 customers buy one DVD each. His next 2 customers buy 2 DVDs each. His" | |
| }, | |
| { | |
| "id": 23, | |
| "gold": "8", | |
| "predicted": "00", | |
| "correct": false, | |
| "response": "A candle melts by 2 centimeters every hour that it burns. How many centimeters shorter will a candle be after burning from 1:00 PM to 5:00 PM?" | |
| }, | |
| { | |
| "id": 24, | |
| "gold": "26", | |
| "predicted": "25", | |
| "correct": false, | |
| "response": "Kyle bought last year's best-selling book for $19.50. This is with a 25% discount from the original price. What was the original price of the book?" | |
| }, | |
| { | |
| "id": 25, | |
| "gold": "2", | |
| "predicted": "50", | |
| "correct": false, | |
| "response": "Marie ordered one chicken meal that costs $12, 5 packs of milk that costs $3 each, 4 apples that cost $1.50 each, and some boxes of pizza. Marie paid a total of $50." | |
| }, | |
| { | |
| "id": 26, | |
| "gold": "243", | |
| "predicted": "16.50", | |
| "correct": false, | |
| "response": "Mishka bought 3 pairs of shorts, 3 pairs of pants, and 3 pairs of shoes. One pair of shorts costs $16.50. One pair of" | |
| }, | |
| { | |
| "id": 27, | |
| "gold": "16", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "Question asked: How much will she spend on ice cream?" | |
| }, | |
| { | |
| "id": 28, | |
| "gold": "25", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "I can answer this question with the help of the formula: ${distance}={time}\\cdot{time}$" | |
| }, | |
| { | |
| "id": 29, | |
| "gold": "104", | |
| "predicted": "33", | |
| "correct": false, | |
| "response": "Gloria has to choose between the boots and two pairs of high heels that together cost five dollars less than the boots. If one pair of heels costs $33 and the other costs twice as much, how many dollars are the boots?" | |
| }, | |
| { | |
| "id": 30, | |
| "gold": "109", | |
| "predicted": "162", | |
| "correct": false, | |
| "response": "Allen's age 10 years from now is 162 years." | |
| }, | |
| { | |
| "id": 31, | |
| "gold": "80", | |
| "predicted": "25", | |
| "correct": false, | |
| "response": "Gunter is trying to count the jelly beans in a jar. He asks his friends how many they think are in the jar. One says 80. Another says 20 more than half the first one. A third says 25% more than the first one." | |
| }, | |
| { | |
| "id": 32, | |
| "gold": "35", | |
| "predicted": "10", | |
| "correct": false, | |
| "response": "John spends a total of 10 hours a day on a dog. He spends a total" | |
| }, | |
| { | |
| "id": 33, | |
| "gold": "70", | |
| "predicted": "30", | |
| "correct": false, | |
| "response": "There are 110 coins. There are 30 more gold coins than silver coins. How many gold coins does Gretchen have?" | |
| }, | |
| { | |
| "id": 34, | |
| "gold": "23", | |
| "predicted": "5", | |
| "correct": false, | |
| "response": "Siobhan has 2 fewer jewels than Aaron. Aaron has 5 more jewels than" | |
| }, | |
| { | |
| "id": 35, | |
| "gold": "9", | |
| "predicted": "20", | |
| "correct": false, | |
| "response": "Mike scored 50 points in the first 20 minutes." | |
| }, | |
| { | |
| "id": 36, | |
| "gold": "75", | |
| "predicted": "4", | |
| "correct": false, | |
| "response": "Terry eats 2 yogurts a day. They are currently on sale at 4 yogurts" | |
| }, | |
| { | |
| "id": 37, | |
| "gold": "2", | |
| "predicted": "5", | |
| "correct": false, | |
| "response": "John has 13 lego sets and he sells them for $15 each. He ends up buying 8 video games for $20 each and has $5 left. How many lego sets does he still have?" | |
| }, | |
| { | |
| "id": 38, | |
| "gold": "10", | |
| "predicted": "3", | |
| "correct": false, | |
| "response": "John runs 60 miles a week. He runs 3 days a week." | |
| }, | |
| { | |
| "id": 39, | |
| "gold": "18", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "If Dana can run at a rate of speed four times faster than she can walk, but she can skip at a rate of" | |
| }, | |
| { | |
| "id": 40, | |
| "gold": "8", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "Brandon's iPhone is four times as old as Ben's iPhone" | |
| }, | |
| { | |
| "id": 41, | |
| "gold": "200", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "I cannot determine that from the given information." | |
| }, | |
| { | |
| "id": 42, | |
| "gold": "26", | |
| "predicted": "14", | |
| "correct": false, | |
| "response": "There were 14 pieces of pie remaining." | |
| }, | |
| { | |
| "id": 43, | |
| "gold": "48", | |
| "predicted": "1800", | |
| "correct": false, | |
| "response": "When it comes to calories, a bag of chips has 250 calories per serving. If a 300g bag has 5 servings, how many grams can you eat if your daily calorie target is 2000 and you have already consumed 1800 calories?" | |
| }, | |
| { | |
| "id": 44, | |
| "gold": "20", | |
| "predicted": null, | |
| "correct": false, | |
| "response": "I cannot determine that from the given information." | |
| }, | |
| { | |
| "id": 45, | |
| "gold": "104", | |
| "predicted": "4", | |
| "correct": false, | |
| "response": "The total number of hours she spent writing articles is 4." | |
| }, | |
| { | |
| "id": 46, | |
| "gold": "163", | |
| "predicted": "24", | |
| "correct": false, | |
| "response": "There are 24 Post-it notes in the package that she purchased." | |
| }, | |
| { | |
| "id": 47, | |
| "gold": "800", | |
| "predicted": "40", | |
| "correct": false, | |
| "response": "John spent $200 on blue ties that cost $40 each." | |
| }, | |
| { | |
| "id": 48, | |
| "gold": "8", | |
| "predicted": "6", | |
| "correct": false, | |
| "response": "Tracy used a piece of wire 4 feet long to support tomato plants in the garden. The wire was cut into pieces 6 inches long. How many pieces did she obtain?" | |
| }, | |
| { | |
| "id": 49, | |
| "gold": "30", | |
| "predicted": "7", | |
| "correct": false, | |
| "response": "The total number of unoccupied units is 7." | |
| } | |
| ] | |
| } | |
| } | |
| }\n |