Upload folder using huggingface_hub

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_11/0.4.8_2025-03-15T11-25-38.029337_lambada_openai.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "alias": "lambada_openai",
+      "perplexity,none": 22.798507649915244,
+      "perplexity_stderr,none": 0.7375150682316036,
+      "acc,none": 0.38404812730448284,
+      "acc_stderr,none": 0.006776076316867708
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_11/0.4.8_2025-03-15T11-40-26.868929_hellaswag.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "hellaswag": {
+      "alias": "hellaswag",
+      "acc,none": 0.29197371041625175,
+      "acc_stderr,none": 0.004537410615572915,
+      "acc_norm,none": 0.3192591117307309,
+      "acc_norm_stderr,none": 0.004652368273845513
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_11/0.4.8_2025-03-15T11-41-59.115770_piqa.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "piqa": {
+      "alias": "piqa",
+      "acc,none": 0.6137105549510338,
+      "acc_stderr,none": 0.011360138833823677,
+      "acc_norm,none": 0.6186071817192601,
+      "acc_norm_stderr,none": 0.011332850406528682
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_11/0.4.8_2025-03-15T11-45-05.505108_arc_easy.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_easy": {
+      "alias": "arc_easy",
+      "acc,none": 0.44234006734006737,
+      "acc_stderr,none": 0.010191334444220846,
+      "acc_norm,none": 0.3939393939393939,
+      "acc_norm_stderr,none": 0.010026305355981804
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_11/0.4.8_2025-03-15T11-46-42.690976_arc_challenge.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_challenge": {
+      "alias": "arc_challenge",
+      "acc,none": 0.19880546075085323,
+      "acc_stderr,none": 0.011662850198175536,
+      "acc_norm,none": 0.22781569965870307,
+      "acc_norm_stderr,none": 0.012256708602326919
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-15T23-56-38.085921_glue.json

@@ -0,0 +1,69 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "glue"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "cola": {
+      "alias": "cola",
+      "mcc,none": 0.05873054109498616,
+      "mcc_stderr,none": 0.03406044652133965
+    },
+    "mnli": {
+      "alias": "mnli",
+      "acc,none": 0.3186958736627611,
+      "acc_stderr,none": 0.004703657632807151
+    },
+    "mnli_mismatch": {
+      "alias": "mnli_mismatch",
+      "acc,none": 0.32414564686737185,
+      "acc_stderr,none": 0.00472060656953387
+    },
+    "mrpc": {
+      "alias": "mrpc",
+      "acc,none": 0.6838235294117647,
+      "acc_stderr,none": 0.023048336668420193,
+      "f1,none": 0.8122270742358079,
+      "f1_stderr,none": 0.01642422915504594
+    },
+    "qnli": {
+      "alias": "qnli",
+      "acc,none": 0.5024711696869851,
+      "acc_stderr,none": 0.006765327922882504
+    },
+    "qqp": {
+      "alias": "qqp",
+      "acc,none": 0.37380657927281724,
+      "acc_stderr,none": 0.002406197221605358,
+      "f1,none": 0.5285562652464573,
+      "f1_stderr,none": 0.0025938199201718647
+    },
+    "rte": {
+      "alias": "rte",
+      "acc,none": 0.5018050541516246,
+      "acc_stderr,none": 0.030096267148976626
+    },
+    "sst2": {
+      "alias": "sst2",
+      "acc,none": 0.5091743119266054,
+      "acc_stderr,none": 0.016939001525351532
+    },
+    "wnli": {
+      "alias": "wnli",
+      "acc,none": 0.4647887323943662,
+      "acc_stderr,none": 0.05961305784972239
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-15T23-57-53.763827_winogrande.json

@@ -0,0 +1,25 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "winogrande": {
+      "alias": "winogrande",
+      "acc,none": 0.5098658247829518,
+      "acc_stderr,none": 0.014049749833367589
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-15T23-59-54.095066_sciq.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "sciq"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "sciq": {
+      "alias": "sciq",
+      "acc,none": 0.756,
+      "acc_stderr,none": 0.013588548437881431,
+      "acc_norm,none": 0.672,
+      "acc_norm_stderr,none": 0.014853842487270334
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T00-22-58.345845_mmlu.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.2315909414613303,
+      "acc_stderr,none": 0.0035530263924027183,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "acc,none": 0.2452709883103082,
+      "acc_stderr,none": 0.006271012587014002,
+      "alias": " - humanities"
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.31746031746031744,
+      "acc_stderr,none": 0.04163453031302859
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.23030303030303031,
+      "acc_stderr,none": 0.0328766675860349
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.27450980392156865,
+      "acc_stderr,none": 0.03132179803083292
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.270042194092827,
+      "acc_stderr,none": 0.028900721906293426
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2396694214876033,
+      "acc_stderr,none": 0.03896878985070417
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04186091791394607
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.22085889570552147,
+      "acc_stderr,none": 0.032591773927421776
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.24855491329479767,
+      "acc_stderr,none": 0.023267528432100174
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.23798882681564246,
+      "acc_stderr,none": 0.014242630070574885
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.19614147909967847,
+      "acc_stderr,none": 0.022552447780478026
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.2222222222222222,
+      "acc_stderr,none": 0.023132376234543325
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.2470664928292047,
+      "acc_stderr,none": 0.011015752255279338
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.30994152046783624,
+      "acc_stderr,none": 0.03546976959393161
+    },
+    "mmlu_other": {
+      "acc,none": 0.24589636305117477,
+      "acc_stderr,none": 0.0077088095866843925,
+      "alias": " - other"
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.20754716981132076,
+      "acc_stderr,none": 0.02495991802891127
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.24277456647398843,
+      "acc_stderr,none": 0.0326926380614177
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.19,
+      "acc_stderr,none": 0.039427724440366234
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.31390134529147984,
+      "acc_stderr,none": 0.03114679648297246
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.1650485436893204,
+      "acc_stderr,none": 0.036756688322331886
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.2905982905982906,
+      "acc_stderr,none": 0.029745048572674064
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.33,
+      "acc_stderr,none": 0.04725815626252604
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.2388250319284802,
+      "acc_stderr,none": 0.015246803197398682
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.238562091503268,
+      "acc_stderr,none": 0.02440439492808787
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.24822695035460993,
+      "acc_stderr,none": 0.025770015644290406
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1948529411764706,
+      "acc_stderr,none": 0.024060599423487424
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.28313253012048195,
+      "acc_stderr,none": 0.03507295431370518
+    },
+    "mmlu_social_sciences": {
+      "acc,none": 0.2183945401364966,
+      "acc_stderr,none": 0.007444613745005196,
+      "alias": " - social sciences"
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.22807017543859648,
+      "acc_stderr,none": 0.03947152782669415
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.17676767676767677,
+      "acc_stderr,none": 0.027178752639044915
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.20207253886010362,
+      "acc_stderr,none": 0.02897908979429673
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2076923076923077,
+      "acc_stderr,none": 0.02056753956724681
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.2184873949579832,
+      "acc_stderr,none": 0.026841514322958938
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.017149858514250958
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.2595419847328244,
+      "acc_stderr,none": 0.0384487613978527
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.2549019607843137,
+      "acc_stderr,none": 0.017630827375148383
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.19090909090909092,
+      "acc_stderr,none": 0.03764425585984924
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.17959183673469387,
+      "acc_stderr,none": 0.024573293589585637
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.23383084577114427,
+      "acc_stderr,none": 0.0299294154083484
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.28,
+      "acc_stderr,none": 0.045126085985421276
+    },
+    "mmlu_stem": {
+      "acc,none": 0.20995876942594355,
+      "acc_stderr,none": 0.007233045009100066,
+      "alias": " - stem"
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.04229525846816506
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.18518518518518517,
+      "acc_stderr,none": 0.03355677216313142
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.18421052631578946,
+      "acc_stderr,none": 0.0315469804508223
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.24305555555555555,
+      "acc_stderr,none": 0.03586879280080342
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.14,
+      "acc_stderr,none": 0.034873508801977725
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.0416333199893227
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.21568627450980393,
+      "acc_stderr,none": 0.04092563958237655
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.044084400227680794
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.26382978723404255,
+      "acc_stderr,none": 0.02880998985410298
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.23448275862068965,
+      "acc_stderr,none": 0.035306258743465914
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.21693121693121692,
+      "acc_stderr,none": 0.021227082449445045
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.17096774193548386,
+      "acc_stderr,none": 0.02141724293632157
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.1625615763546798,
+      "acc_stderr,none": 0.02596030006460558
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.2111111111111111,
+      "acc_stderr,none": 0.024882116857655078
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.1986754966887417,
+      "acc_stderr,none": 0.032578473844367746
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.12962962962962962,
+      "acc_stderr,none": 0.022907883151288597
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.29464285714285715,
+      "acc_stderr,none": 0.04327040932578728
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T00-33-26.008450_lambada_multilingual.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "lambada_multilingual"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai_mt_de": {
+      "alias": "lambada_openai_mt_de",
+      "perplexity,none": 438.1123448025884,
+      "perplexity_stderr,none": 27.631396537461868,
+      "acc,none": 0.17970114496409859,
+      "acc_stderr,none": 0.005349011697308402
+    },
+    "lambada_openai_mt_en": {
+      "alias": "lambada_openai_mt_en",
+      "perplexity,none": 21.873502960243776,
+      "perplexity_stderr,none": 0.7063940196221459,
+      "acc,none": 0.39239278090432755,
+      "acc_stderr,none": 0.006802742619162039
+    },
+    "lambada_openai_mt_es": {
+      "alias": "lambada_openai_mt_es",
+      "perplexity,none": 520.7083247325162,
+      "perplexity_stderr,none": 31.60312563024137,
+      "acc,none": 0.17581991073161266,
+      "acc_stderr,none": 0.005303433892159959
+    },
+    "lambada_openai_mt_fr": {
+      "alias": "lambada_openai_mt_fr",
+      "perplexity,none": 315.69320350745926,
+      "perplexity_stderr,none": 19.029382127916357,
+      "acc,none": 0.21075101882398603,
+      "acc_stderr,none": 0.005682035322435383
+    },
+    "lambada_openai_mt_it": {
+      "alias": "lambada_openai_mt_it",
+      "perplexity,none": 753.7215739042271,
+      "perplexity_stderr,none": 49.15367277380386,
+      "acc,none": 0.16475839316902774,
+      "acc_stderr,none": 0.005168230882239359
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T09-56-09.522548_pawsx.json

@@ -0,0 +1,60 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "pawsx"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "pawsx": {
+      "acc,none": 0.48514285714285715,
+      "acc_stderr,none": 0.0042164415074101135,
+      "alias": "pawsx"
+    },
+    "paws_de": {
+      "alias": " - paws_de",
+      "acc,none": 0.4985,
+      "acc_stderr,none": 0.011183085696839198
+    },
+    "paws_en": {
+      "alias": " - paws_en",
+      "acc,none": 0.5015,
+      "acc_stderr,none": 0.011183085696839203
+    },
+    "paws_es": {
+      "alias": " - paws_es",
+      "acc,none": 0.532,
+      "acc_stderr,none": 0.011160209457602887
+    },
+    "paws_fr": {
+      "alias": " - paws_fr",
+      "acc,none": 0.4715,
+      "acc_stderr,none": 0.0111649542364288
+    },
+    "paws_ja": {
+      "alias": " - paws_ja",
+      "acc,none": 0.4395,
+      "acc_stderr,none": 0.011100968009384213
+    },
+    "paws_ko": {
+      "alias": " - paws_ko",
+      "acc,none": 0.445,
+      "acc_stderr,none": 0.011115272135099217
+    },
+    "paws_zh": {
+      "alias": " - paws_zh",
+      "acc,none": 0.508,
+      "acc_stderr,none": 0.011181704488030004
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T10-00-53.579856_xcopa.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "xcopa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xcopa": {
+      "acc,none": 0.5310909090909091,
+      "acc_stderr,none": 0.006728483885990303,
+      "alias": "xcopa"
+    },
+    "xcopa_et": {
+      "alias": " - xcopa_et",
+      "acc,none": 0.504,
+      "acc_stderr,none": 0.022382357781962132
+    },
+    "xcopa_ht": {
+      "alias": " - xcopa_ht",
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.02237859698923078
+    },
+    "xcopa_id": {
+      "alias": " - xcopa_id",
+      "acc,none": 0.544,
+      "acc_stderr,none": 0.022296238348407056
+    },
+    "xcopa_it": {
+      "alias": " - xcopa_it",
+      "acc,none": 0.514,
+      "acc_stderr,none": 0.022374298166353185
+    },
+    "xcopa_qu": {
+      "alias": " - xcopa_qu",
+      "acc,none": 0.502,
+      "acc_stderr,none": 0.02238289498648352
+    },
+    "xcopa_sw": {
+      "alias": " - xcopa_sw",
+      "acc,none": 0.544,
+      "acc_stderr,none": 0.022296238348407056
+    },
+    "xcopa_ta": {
+      "alias": " - xcopa_ta",
+      "acc,none": 0.53,
+      "acc_stderr,none": 0.022342748192502846
+    },
+    "xcopa_th": {
+      "alias": " - xcopa_th",
+      "acc,none": 0.578,
+      "acc_stderr,none": 0.022109039310618556
+    },
+    "xcopa_tr": {
+      "alias": " - xcopa_tr",
+      "acc,none": 0.562,
+      "acc_stderr,none": 0.022210326363977417
+    },
+    "xcopa_vi": {
+      "alias": " - xcopa_vi",
+      "acc,none": 0.53,
+      "acc_stderr,none": 0.022342748192502843
+    },
+    "xcopa_zh": {
+      "alias": " - xcopa_zh",
+      "acc,none": 0.524,
+      "acc_stderr,none": 0.0223572738810164
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T11-13-06.581208_xnli.json

@@ -0,0 +1,100 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "xnli"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xnli": {
+      "acc,none": 0.3610709504685408,
+      "acc_stderr,none": 0.002478789488090545,
+      "alias": "xnli"
+    },
+    "xnli_ar": {
+      "alias": " - xnli_ar",
+      "acc,none": 0.3433734939759036,
+      "acc_stderr,none": 0.009517658993060705
+    },
+    "xnli_bg": {
+      "alias": " - xnli_bg",
+      "acc,none": 0.3457831325301205,
+      "acc_stderr,none": 0.009533455033752768
+    },
+    "xnli_de": {
+      "alias": " - xnli_de",
+      "acc,none": 0.39718875502008033,
+      "acc_stderr,none": 0.009807915070677289
+    },
+    "xnli_el": {
+      "alias": " - xnli_el",
+      "acc,none": 0.3353413654618474,
+      "acc_stderr,none": 0.009463034891512703
+    },
+    "xnli_en": {
+      "alias": " - xnli_en",
+      "acc,none": 0.46224899598393576,
+      "acc_stderr,none": 0.009993466360872784
+    },
+    "xnli_es": {
+      "alias": " - xnli_es",
+      "acc,none": 0.344578313253012,
+      "acc_stderr,none": 0.009525590900110655
+    },
+    "xnli_fr": {
+      "alias": " - xnli_fr",
+      "acc,none": 0.41044176706827307,
+      "acc_stderr,none": 0.00985999467258512
+    },
+    "xnli_hi": {
+      "alias": " - xnli_hi",
+      "acc,none": 0.348995983935743,
+      "acc_stderr,none": 0.009554095988300674
+    },
+    "xnli_ru": {
+      "alias": " - xnli_ru",
+      "acc,none": 0.3795180722891566,
+      "acc_stderr,none": 0.009726763372837137
+    },
+    "xnli_sw": {
+      "alias": " - xnli_sw",
+      "acc,none": 0.3285140562248996,
+      "acc_stderr,none": 0.009414190734131762
+    },
+    "xnli_th": {
+      "alias": " - xnli_th",
+      "acc,none": 0.3285140562248996,
+      "acc_stderr,none": 0.009414190734131762
+    },
+    "xnli_tr": {
+      "alias": " - xnli_tr",
+      "acc,none": 0.35903614457831323,
+      "acc_stderr,none": 0.00961553399241459
+    },
+    "xnli_ur": {
+      "alias": " - xnli_ur",
+      "acc,none": 0.3333333333333333,
+      "acc_stderr,none": 0.009448900914617614
+    },
+    "xnli_vi": {
+      "alias": " - xnli_vi",
+      "acc,none": 0.36305220883534134,
+      "acc_stderr,none": 0.00963882313398499
+    },
+    "xnli_zh": {
+      "alias": " - xnli_zh",
+      "acc,none": 0.336144578313253,
+      "acc_stderr,none": 0.00946863466929353
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T11-52-46.021580_xstorycloze.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "xstorycloze"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xstorycloze": {
+      "acc,none": 0.5168762409000662,
+      "acc_stderr,none": 0.0038693051060829046,
+      "alias": "xstorycloze"
+    },
+    "xstorycloze_ar": {
+      "alias": " - xstorycloze_ar",
+      "acc,none": 0.47253474520185307,
+      "acc_stderr,none": 0.012847698270388213
+    },
+    "xstorycloze_en": {
+      "alias": " - xstorycloze_en",
+      "acc,none": 0.6029119788219722,
+      "acc_stderr,none": 0.012591627740247462
+    },
+    "xstorycloze_es": {
+      "alias": " - xstorycloze_es",
+      "acc,none": 0.5241561879549967,
+      "acc_stderr,none": 0.01285210005730961
+    },
+    "xstorycloze_eu": {
+      "alias": " - xstorycloze_eu",
+      "acc,none": 0.5168762409000662,
+      "acc_stderr,none": 0.012859793919977602
+    },
+    "xstorycloze_hi": {
+      "alias": " - xstorycloze_hi",
+      "acc,none": 0.514890800794176,
+      "acc_stderr,none": 0.012861417842074004
+    },
+    "xstorycloze_id": {
+      "alias": " - xstorycloze_id",
+      "acc,none": 0.500992720052945,
+      "acc_stderr,none": 0.012867099955422933
+    },
+    "xstorycloze_my": {
+      "alias": " - xstorycloze_my",
+      "acc,none": 0.49503639973527463,
+      "acc_stderr,none": 0.012866491277589943
+    },
+    "xstorycloze_ru": {
+      "alias": " - xstorycloze_ru",
+      "acc,none": 0.4943745863666446,
+      "acc_stderr,none": 0.01286631092307252
+    },
+    "xstorycloze_sw": {
+      "alias": " - xstorycloze_sw",
+      "acc,none": 0.5056254136333554,
+      "acc_stderr,none": 0.012866310923072511
+    },
+    "xstorycloze_te": {
+      "alias": " - xstorycloze_te",
+      "acc,none": 0.5420251489080079,
+      "acc_stderr,none": 0.012821595164245275
+    },
+    "xstorycloze_zh": {
+      "alias": " - xstorycloze_zh",
+      "acc,none": 0.5162144275314361,
+      "acc_stderr,none": 0.012860357805055855
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T11-55-53.206548_xwinograd.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.5927174645987863,
+      "acc_stderr,none": 0.007316470702661161,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "alias": " - xwinograd_en",
+      "acc,none": 0.6468817204301075,
+      "acc_stderr,none": 0.009914126992783658
+    },
+    "xwinograd_fr": {
+      "alias": " - xwinograd_fr",
+      "acc,none": 0.5421686746987951,
+      "acc_stderr,none": 0.05501904358494246
+    },
+    "xwinograd_jp": {
+      "alias": " - xwinograd_jp",
+      "acc,none": 0.5078206465067779,
+      "acc_stderr,none": 0.01615229055184455
+    },
+    "xwinograd_pt": {
+      "alias": " - xwinograd_pt",
+      "acc,none": 0.5285171102661597,
+      "acc_stderr,none": 0.030839820992717426
+    },
+    "xwinograd_ru": {
+      "alias": " - xwinograd_ru",
+      "acc,none": 0.546031746031746,
+      "acc_stderr,none": 0.028096800277810533
+    },
+    "xwinograd_zh": {
+      "alias": " - xwinograd_zh",
+      "acc,none": 0.5753968253968254,
+      "acc_stderr,none": 0.022038973193044563
+    }
+  }
+}

lm_eval/RWKV-5-World-0.1B-v1-20230803-ctx4096/pad_261/0.4.8_2025-03-16T12-17-19.970767_mmlu.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/RWKV-5-World-0.1B-v1-20230803-ctx4096.pth",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 5,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": null,
+  "pad_token_ids": [
+    261
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.2315909414613303,
+      "acc_stderr,none": 0.0035530263924027183,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "acc,none": 0.2452709883103082,
+      "acc_stderr,none": 0.006271012587014002,
+      "alias": " - humanities"
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.31746031746031744,
+      "acc_stderr,none": 0.04163453031302859
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.23030303030303031,
+      "acc_stderr,none": 0.0328766675860349
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.27450980392156865,
+      "acc_stderr,none": 0.03132179803083292
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.270042194092827,
+      "acc_stderr,none": 0.028900721906293426
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2396694214876033,
+      "acc_stderr,none": 0.03896878985070417
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04186091791394607
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.22085889570552147,
+      "acc_stderr,none": 0.032591773927421776
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.24855491329479767,
+      "acc_stderr,none": 0.023267528432100174
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.23798882681564246,
+      "acc_stderr,none": 0.014242630070574885
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.19614147909967847,
+      "acc_stderr,none": 0.022552447780478026
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.2222222222222222,
+      "acc_stderr,none": 0.023132376234543325
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.2470664928292047,
+      "acc_stderr,none": 0.011015752255279338
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.30994152046783624,
+      "acc_stderr,none": 0.03546976959393161
+    },
+    "mmlu_other": {
+      "acc,none": 0.24589636305117477,
+      "acc_stderr,none": 0.0077088095866843925,
+      "alias": " - other"
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.20754716981132076,
+      "acc_stderr,none": 0.02495991802891127
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.24277456647398843,
+      "acc_stderr,none": 0.0326926380614177
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.19,
+      "acc_stderr,none": 0.039427724440366234
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.31390134529147984,
+      "acc_stderr,none": 0.03114679648297246
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.1650485436893204,
+      "acc_stderr,none": 0.036756688322331886
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.2905982905982906,
+      "acc_stderr,none": 0.029745048572674064
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.33,
+      "acc_stderr,none": 0.04725815626252604
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.2388250319284802,
+      "acc_stderr,none": 0.015246803197398682
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.238562091503268,
+      "acc_stderr,none": 0.02440439492808787
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.24822695035460993,
+      "acc_stderr,none": 0.025770015644290406
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1948529411764706,
+      "acc_stderr,none": 0.024060599423487424
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.28313253012048195,
+      "acc_stderr,none": 0.03507295431370518
+    },
+    "mmlu_social_sciences": {
+      "acc,none": 0.2183945401364966,
+      "acc_stderr,none": 0.007444613745005196,
+      "alias": " - social sciences"
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.22807017543859648,
+      "acc_stderr,none": 0.03947152782669415
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.17676767676767677,
+      "acc_stderr,none": 0.027178752639044915
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.20207253886010362,
+      "acc_stderr,none": 0.02897908979429673
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2076923076923077,
+      "acc_stderr,none": 0.02056753956724681
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.2184873949579832,
+      "acc_stderr,none": 0.026841514322958938
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.017149858514250958
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.2595419847328244,
+      "acc_stderr,none": 0.0384487613978527
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.2549019607843137,
+      "acc_stderr,none": 0.017630827375148383
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.19090909090909092,
+      "acc_stderr,none": 0.03764425585984924
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.17959183673469387,
+      "acc_stderr,none": 0.024573293589585637
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.23383084577114427,
+      "acc_stderr,none": 0.0299294154083484
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.28,
+      "acc_stderr,none": 0.045126085985421276
+    },
+    "mmlu_stem": {
+      "acc,none": 0.20995876942594355,
+      "acc_stderr,none": 0.007233045009100066,
+      "alias": " - stem"
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.04229525846816506
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.18518518518518517,
+      "acc_stderr,none": 0.03355677216313142
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.18421052631578946,
+      "acc_stderr,none": 0.0315469804508223
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.24305555555555555,
+      "acc_stderr,none": 0.03586879280080342
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.14,
+      "acc_stderr,none": 0.034873508801977725
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.0416333199893227
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.21568627450980393,
+      "acc_stderr,none": 0.04092563958237655
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.044084400227680794
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.26382978723404255,
+      "acc_stderr,none": 0.02880998985410298
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.23448275862068965,
+      "acc_stderr,none": 0.035306258743465914
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.21693121693121692,
+      "acc_stderr,none": 0.021227082449445045
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.17096774193548386,
+      "acc_stderr,none": 0.02141724293632157
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.1625615763546798,
+      "acc_stderr,none": 0.02596030006460558
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.2111111111111111,
+      "acc_stderr,none": 0.024882116857655078
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.1986754966887417,
+      "acc_stderr,none": 0.032578473844367746
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.12962962962962962,
+      "acc_stderr,none": 0.022907883151288597
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.29464285714285715,
+      "acc_stderr,none": 0.04327040932578728
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T14-14-42.123700_lambada_openai.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "alias": "lambada_openai",
+      "perplexity,none": 8.874438771398895,
+      "perplexity_stderr,none": 0.25110180223575285,
+      "acc,none": 0.5402678051620415,
+      "acc_stderr,none": 0.0069433502956647445
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T14-39-58.725375_hellaswag.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "hellaswag": {
+      "alias": "hellaswag",
+      "acc,none": 0.34176458872734516,
+      "acc_stderr,none": 0.004733317847006545,
+      "acc_norm,none": 0.40928101971718783,
+      "acc_norm_stderr,none": 0.004906962980328275
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T14-42-10.675354_piqa.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "piqa": {
+      "alias": "piqa",
+      "acc,none": 0.6653971708378672,
+      "acc_stderr,none": 0.011009071725162505,
+      "acc_norm,none": 0.6724700761697497,
+      "acc_norm_stderr,none": 0.010949830482825478
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T14-46-46.674313_arc_easy.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_easy": {
+      "alias": "arc_easy",
+      "acc,none": 0.5395622895622896,
+      "acc_stderr,none": 0.010227616386289006,
+      "acc_norm,none": 0.4877946127946128,
+      "acc_norm_stderr,none": 0.010256726235129009
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T14-49-11.748710_arc_challenge.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_challenge": {
+      "alias": "arc_challenge",
+      "acc,none": 0.23976109215017063,
+      "acc_stderr,none": 0.012476304127453932,
+      "acc_norm,none": 0.2627986348122867,
+      "acc_norm_stderr,none": 0.012862523175351331
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T16-08-18.050596_glue.json

@@ -0,0 +1,69 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "glue"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "cola": {
+      "alias": "cola",
+      "mcc,none": 0.0024763761493444563,
+      "mcc_stderr,none": 0.030693706731630372
+    },
+    "mnli": {
+      "alias": "mnli",
+      "acc,none": 0.34212939378502294,
+      "acc_stderr,none": 0.004788973218637058
+    },
+    "mnli_mismatch": {
+      "alias": "mnli_mismatch",
+      "acc,none": 0.34662327095199347,
+      "acc_stderr,none": 0.004799675113044456
+    },
+    "mrpc": {
+      "alias": "mrpc",
+      "acc,none": 0.6838235294117647,
+      "acc_stderr,none": 0.023048336668420193,
+      "f1,none": 0.8122270742358079,
+      "f1_stderr,none": 0.01642422915504585
+    },
+    "qnli": {
+      "alias": "qnli",
+      "acc,none": 0.5066813106351822,
+      "acc_stderr,none": 0.006764806510150313
+    },
+    "qqp": {
+      "alias": "qqp",
+      "acc,none": 0.610907741775909,
+      "acc_stderr,none": 0.0024247535821838137,
+      "f1,none": 0.057628946264901455,
+      "f1_stderr,none": 0.002516531840072046
+    },
+    "rte": {
+      "alias": "rte",
+      "acc,none": 0.5270758122743683,
+      "acc_stderr,none": 0.030052303463143706
+    },
+    "sst2": {
+      "alias": "sst2",
+      "acc,none": 0.5149082568807339,
+      "acc_stderr,none": 0.0169343211533256
+    },
+    "wnli": {
+      "alias": "wnli",
+      "acc,none": 0.4647887323943662,
+      "acc_stderr,none": 0.0596130578497224
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T16-09-43.473147_winogrande.json

@@ -0,0 +1,25 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "winogrande": {
+      "alias": "winogrande",
+      "acc,none": 0.531965272296764,
+      "acc_stderr,none": 0.01402373922116638
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T16-12-44.780584_sciq.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "sciq"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "sciq": {
+      "alias": "sciq",
+      "acc,none": 0.869,
+      "acc_stderr,none": 0.010674874844837957,
+      "acc_norm,none": 0.809,
+      "acc_norm_stderr,none": 0.012436787112179517
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T16-52-46.693108_mmlu.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.23764421022646345,
+      "acc_stderr,none": 0.003587790530224594,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "acc,none": 0.24782146652497344,
+      "acc_stderr,none": 0.0062902893350153324,
+      "alias": " - humanities"
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.3253968253968254,
+      "acc_stderr,none": 0.041905964388711366
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.22424242424242424,
+      "acc_stderr,none": 0.03256866661681102
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.2107843137254902,
+      "acc_stderr,none": 0.028626547912437388
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.270042194092827,
+      "acc_stderr,none": 0.028900721906293433
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2396694214876033,
+      "acc_stderr,none": 0.038968789850704164
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.32407407407407407,
+      "acc_stderr,none": 0.04524596007030048
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.20245398773006135,
+      "acc_stderr,none": 0.03157065078911902
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.2398843930635838,
+      "acc_stderr,none": 0.022989592543123567
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.25027932960893856,
+      "acc_stderr,none": 0.014487500852850412
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.19935691318327975,
+      "acc_stderr,none": 0.022691033780549656
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.23148148148148148,
+      "acc_stderr,none": 0.023468429832451152
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.25358539765319427,
+      "acc_stderr,none": 0.01111171533610114
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.2982456140350877,
+      "acc_stderr,none": 0.035087719298245654
+    },
+    "mmlu_other": {
+      "acc,none": 0.24460894753781784,
+      "acc_stderr,none": 0.007691719275922855,
+      "alias": " - other"
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.31,
+      "acc_stderr,none": 0.04648231987117316
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.19245283018867926,
+      "acc_stderr,none": 0.02426297983937226
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.2023121387283237,
+      "acc_stderr,none": 0.03063114553919882
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.24,
+      "acc_stderr,none": 0.042923469599092816
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.3273542600896861,
+      "acc_stderr,none": 0.03149384670994132
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.20388349514563106,
+      "acc_stderr,none": 0.03989139859531771
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.2905982905982906,
+      "acc_stderr,none": 0.029745048572674064
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.29,
+      "acc_stderr,none": 0.04560480215720684
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.23243933588761176,
+      "acc_stderr,none": 0.015104550008905704
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.25163398692810457,
+      "acc_stderr,none": 0.024848018263875195
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.2695035460992908,
+      "acc_stderr,none": 0.026469036818590627
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1801470588235294,
+      "acc_stderr,none": 0.02334516361654486
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.26506024096385544,
+      "acc_stderr,none": 0.03436024037944967
+    },
+    "mmlu_social_sciences": {
+      "acc,none": 0.22099447513812154,
+      "acc_stderr,none": 0.007486677746029817,
+      "alias": " - social sciences"
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.21929824561403508,
+      "acc_stderr,none": 0.0389243110651875
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.19696969696969696,
+      "acc_stderr,none": 0.028335609732463355
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.23316062176165803,
+      "acc_stderr,none": 0.030516111371476008
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.020280805062535726
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.20168067226890757,
+      "acc_stderr,none": 0.026064313406304534
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.21467889908256882,
+      "acc_stderr,none": 0.01760430414925649
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.22900763358778625,
+      "acc_stderr,none": 0.036853466317118506
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.24673202614379086,
+      "acc_stderr,none": 0.0174408203674025
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.22727272727272727,
+      "acc_stderr,none": 0.04013964554072775
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.21224489795918366,
+      "acc_stderr,none": 0.026176967197866767
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.21393034825870647,
+      "acc_stderr,none": 0.0289969096933289
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.0446196043338474
+    },
+    "mmlu_stem": {
+      "acc,none": 0.23184268950206152,
+      "acc_stderr,none": 0.00750773023393252,
+      "alias": " - stem"
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.04163331998932269
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.23703703703703705,
+      "acc_stderr,none": 0.03673731683969506
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.21052631578947367,
+      "acc_stderr,none": 0.033176727875331574
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.2916666666666667,
+      "acc_stderr,none": 0.03800968060554859
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.21,
+      "acc_stderr,none": 0.040936018074033256
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.24,
+      "acc_stderr,none": 0.04292346959909284
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.21568627450980393,
+      "acc_stderr,none": 0.04092563958237655
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.28936170212765955,
+      "acc_stderr,none": 0.02964400657700962
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.25517241379310346,
+      "acc_stderr,none": 0.03632984052707842
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.21693121693121692,
+      "acc_stderr,none": 0.02122708244944506
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.1935483870967742,
+      "acc_stderr,none": 0.022475258525536057
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.1921182266009852,
+      "acc_stderr,none": 0.027719315709614778
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.04229525846816506
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.26296296296296295,
+      "acc_stderr,none": 0.026842057873833706
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.2251655629139073,
+      "acc_stderr,none": 0.03410435282008937
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.16203703703703703,
+      "acc_stderr,none": 0.02513045365226846
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.2857142857142857,
+      "acc_stderr,none": 0.04287858751340456
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T17-10-04.238325_lambada_multilingual.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "lambada_multilingual"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai_mt_de": {
+      "alias": "lambada_openai_mt_de",
+      "perplexity,none": 122.582684053421,
+      "perplexity_stderr,none": 7.611410508434413,
+      "acc,none": 0.2905103823015719,
+      "acc_stderr,none": 0.006325078561068121
+    },
+    "lambada_openai_mt_en": {
+      "alias": "lambada_openai_mt_en",
+      "perplexity,none": 8.874438778769504,
+      "perplexity_stderr,none": 0.2511018027503425,
+      "acc,none": 0.5402678051620415,
+      "acc_stderr,none": 0.0069433502956647445
+    },
+    "lambada_openai_mt_es": {
+      "alias": "lambada_openai_mt_es",
+      "perplexity,none": 134.88945576369062,
+      "perplexity_stderr,none": 7.962805036133316,
+      "acc,none": 0.3062293809431399,
+      "acc_stderr,none": 0.006421603000130909
+    },
+    "lambada_openai_mt_fr": {
+      "alias": "lambada_openai_mt_fr",
+      "perplexity,none": 73.64587512871607,
+      "perplexity_stderr,none": 4.301058908781159,
+      "acc,none": 0.3685231903745391,
+      "acc_stderr,none": 0.006720834282814074
+    },
+    "lambada_openai_mt_it": {
+      "alias": "lambada_openai_mt_it",
+      "perplexity,none": 113.04863122926666,
+      "perplexity_stderr,none": 7.0611616362975695,
+      "acc,none": 0.33630894624490587,
+      "acc_stderr,none": 0.006582096796438631
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T17-31-49.237829_pawsx.json

@@ -0,0 +1,60 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "pawsx"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "pawsx": {
+      "acc,none": 0.4947857142857143,
+      "acc_stderr,none": 0.0042167946968630675,
+      "alias": "pawsx"
+    },
+    "paws_de": {
+      "alias": " - paws_de",
+      "acc,none": 0.545,
+      "acc_stderr,none": 0.011137752231145222
+    },
+    "paws_en": {
+      "alias": " - paws_en",
+      "acc,none": 0.533,
+      "acc_stderr,none": 0.011158752568250671
+    },
+    "paws_es": {
+      "alias": " - paws_es",
+      "acc,none": 0.515,
+      "acc_stderr,none": 0.011178102477052804
+    },
+    "paws_fr": {
+      "alias": " - paws_fr",
+      "acc,none": 0.463,
+      "acc_stderr,none": 0.011152474561478177
+    },
+    "paws_ja": {
+      "alias": " - paws_ja",
+      "acc,none": 0.4795,
+      "acc_stderr,none": 0.011173732641806813
+    },
+    "paws_ko": {
+      "alias": " - paws_ko",
+      "acc,none": 0.447,
+      "acc_stderr,none": 0.011120131683767735
+    },
+    "paws_zh": {
+      "alias": " - paws_zh",
+      "acc,none": 0.481,
+      "acc_stderr,none": 0.01117505887995606
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T17-37-40.081757_xcopa.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xcopa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xcopa": {
+      "acc,none": 0.5401818181818182,
+      "acc_stderr,none": 0.006718472971138079,
+      "alias": "xcopa"
+    },
+    "xcopa_et": {
+      "alias": " - xcopa_et",
+      "acc,none": 0.506,
+      "acc_stderr,none": 0.022381462412439324
+    },
+    "xcopa_ht": {
+      "alias": " - xcopa_ht",
+      "acc,none": 0.522,
+      "acc_stderr,none": 0.02236139673920788
+    },
+    "xcopa_id": {
+      "alias": " - xcopa_id",
+      "acc,none": 0.57,
+      "acc_stderr,none": 0.022162634426652835
+    },
+    "xcopa_it": {
+      "alias": " - xcopa_it",
+      "acc,none": 0.548,
+      "acc_stderr,none": 0.02227969410784342
+    },
+    "xcopa_qu": {
+      "alias": " - xcopa_qu",
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.02237859698923078
+    },
+    "xcopa_sw": {
+      "alias": " - xcopa_sw",
+      "acc,none": 0.55,
+      "acc_stderr,none": 0.022270877485360437
+    },
+    "xcopa_ta": {
+      "alias": " - xcopa_ta",
+      "acc,none": 0.512,
+      "acc_stderr,none": 0.02237662679792717
+    },
+    "xcopa_th": {
+      "alias": " - xcopa_th",
+      "acc,none": 0.534,
+      "acc_stderr,none": 0.02233126442325838
+    },
+    "xcopa_tr": {
+      "alias": " - xcopa_tr",
+      "acc,none": 0.538,
+      "acc_stderr,none": 0.02231833811987053
+    },
+    "xcopa_vi": {
+      "alias": " - xcopa_vi",
+      "acc,none": 0.576,
+      "acc_stderr,none": 0.022122993778135404
+    },
+    "xcopa_zh": {
+      "alias": " - xcopa_zh",
+      "acc,none": 0.576,
+      "acc_stderr,none": 0.022122993778135404
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T19-11-48.808152_xnli.json

@@ -0,0 +1,100 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xnli"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xnli": {
+      "acc,none": 0.3847657295850067,
+      "acc_stderr,none": 0.0025079690852275174,
+      "alias": "xnli"
+    },
+    "xnli_ar": {
+      "alias": " - xnli_ar",
+      "acc,none": 0.3333333333333333,
+      "acc_stderr,none": 0.009448900914617616
+    },
+    "xnli_bg": {
+      "alias": " - xnli_bg",
+      "acc,none": 0.37028112449799194,
+      "acc_stderr,none": 0.009678915409840288
+    },
+    "xnli_de": {
+      "alias": " - xnli_de",
+      "acc,none": 0.43012048192771085,
+      "acc_stderr,none": 0.00992371167540806
+    },
+    "xnli_el": {
+      "alias": " - xnli_el",
+      "acc,none": 0.3582329317269076,
+      "acc_stderr,none": 0.009610788482973929
+    },
+    "xnli_en": {
+      "alias": " - xnli_en",
+      "acc,none": 0.4943775100401606,
+      "acc_stderr,none": 0.010021439203777294
+    },
+    "xnli_es": {
+      "alias": " - xnli_es",
+      "acc,none": 0.39759036144578314,
+      "acc_stderr,none": 0.009809602996075804
+    },
+    "xnli_fr": {
+      "alias": " - xnli_fr",
+      "acc,none": 0.42248995983935744,
+      "acc_stderr,none": 0.009900919227857794
+    },
+    "xnli_hi": {
+      "alias": " - xnli_hi",
+      "acc,none": 0.3650602409638554,
+      "acc_stderr,none": 0.009650194822749635
+    },
+    "xnli_ru": {
+      "alias": " - xnli_ru",
+      "acc,none": 0.43775100401606426,
+      "acc_stderr,none": 0.009944099734290161
+    },
+    "xnli_sw": {
+      "alias": " - xnli_sw",
+      "acc,none": 0.3409638554216867,
+      "acc_stderr,none": 0.009501591178361544
+    },
+    "xnli_th": {
+      "alias": " - xnli_th",
+      "acc,none": 0.348995983935743,
+      "acc_stderr,none": 0.009554095988300678
+    },
+    "xnli_tr": {
+      "alias": " - xnli_tr",
+      "acc,none": 0.3718875502008032,
+      "acc_stderr,none": 0.009687507958631799
+    },
+    "xnli_ur": {
+      "alias": " - xnli_ur",
+      "acc,none": 0.3345381526104418,
+      "acc_stderr,none": 0.009457404390939166
+    },
+    "xnli_vi": {
+      "alias": " - xnli_vi",
+      "acc,none": 0.38393574297188754,
+      "acc_stderr,none": 0.009748321202534375
+    },
+    "xnli_zh": {
+      "alias": " - xnli_zh",
+      "acc,none": 0.3819277108433735,
+      "acc_stderr,none": 0.00973862791451752
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T19-42-29.131682_xstorycloze.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xstorycloze"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xstorycloze": {
+      "acc,none": 0.5407616870224415,
+      "acc_stderr,none": 0.003849468335294993,
+      "alias": "xstorycloze"
+    },
+    "xstorycloze_ar": {
+      "alias": " - xstorycloze_ar",
+      "acc,none": 0.48974189278623426,
+      "acc_stderr,none": 0.012864417047980472
+    },
+    "xstorycloze_en": {
+      "alias": " - xstorycloze_en",
+      "acc,none": 0.6657842488418266,
+      "acc_stderr,none": 0.012139246810918228
+    },
+    "xstorycloze_es": {
+      "alias": " - xstorycloze_es",
+      "acc,none": 0.5698213103904699,
+      "acc_stderr,none": 0.012741052817471078
+    },
+    "xstorycloze_eu": {
+      "alias": " - xstorycloze_eu",
+      "acc,none": 0.5188616810059563,
+      "acc_stderr,none": 0.012857966762464996
+    },
+    "xstorycloze_hi": {
+      "alias": " - xstorycloze_hi",
+      "acc,none": 0.5168762409000662,
+      "acc_stderr,none": 0.012859793919977608
+    },
+    "xstorycloze_id": {
+      "alias": " - xstorycloze_id",
+      "acc,none": 0.5360688285903376,
+      "acc_stderr,none": 0.012833602406620017
+    },
+    "xstorycloze_my": {
+      "alias": " - xstorycloze_my",
+      "acc,none": 0.4930509596293845,
+      "acc_stderr,none": 0.01286588257096072
+    },
+    "xstorycloze_ru": {
+      "alias": " - xstorycloze_ru",
+      "acc,none": 0.5519523494374586,
+      "acc_stderr,none": 0.012797478885304733
+    },
+    "xstorycloze_sw": {
+      "alias": " - xstorycloze_sw",
+      "acc,none": 0.5029781601588352,
+      "acc_stderr,none": 0.012866897066011228
+    },
+    "xstorycloze_te": {
+      "alias": " - xstorycloze_te",
+      "acc,none": 0.5473196558570483,
+      "acc_stderr,none": 0.012809372866181954
+    },
+    "xstorycloze_zh": {
+      "alias": " - xstorycloze_zh",
+      "acc,none": 0.5559232296492389,
+      "acc_stderr,none": 0.012786390539820832
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T19-47-11.016382_xwinograd.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.6556529557203866,
+      "acc_stderr,none": 0.007021810569184652,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "alias": " - xwinograd_en",
+      "acc,none": 0.7290322580645161,
+      "acc_stderr,none": 0.009219643045448322
+    },
+    "xwinograd_fr": {
+      "alias": " - xwinograd_fr",
+      "acc,none": 0.6144578313253012,
+      "acc_stderr,none": 0.0537495779731939
+    },
+    "xwinograd_jp": {
+      "alias": " - xwinograd_jp",
+      "acc,none": 0.5411887382690302,
+      "acc_stderr,none": 0.01609936161806395
+    },
+    "xwinograd_pt": {
+      "alias": " - xwinograd_pt",
+      "acc,none": 0.55893536121673,
+      "acc_stderr,none": 0.0306747666644263
+    },
+    "xwinograd_ru": {
+      "alias": " - xwinograd_ru",
+      "acc,none": 0.5746031746031746,
+      "acc_stderr,none": 0.027900777694976245
+    },
+    "xwinograd_zh": {
+      "alias": " - xwinograd_zh",
+      "acc,none": 0.6428571428571429,
+      "acc_stderr,none": 0.021364573561124416
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T20-21-44.743544_lambada_openai.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "alias": "lambada_openai",
+      "perplexity,none": 8.874438771398895,
+      "perplexity_stderr,none": 0.25110180223575285,
+      "acc,none": 0.5402678051620415,
+      "acc_stderr,none": 0.0069433502956647445
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T22-02-39.723956_xnli.json

@@ -0,0 +1,100 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xnli"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xnli": {
+      "acc,none": 0.3847657295850067,
+      "acc_stderr,none": 0.0025079690852275174,
+      "alias": "xnli"
+    },
+    "xnli_ar": {
+      "alias": " - xnli_ar",
+      "acc,none": 0.3333333333333333,
+      "acc_stderr,none": 0.009448900914617616
+    },
+    "xnli_bg": {
+      "alias": " - xnli_bg",
+      "acc,none": 0.37028112449799194,
+      "acc_stderr,none": 0.009678915409840288
+    },
+    "xnli_de": {
+      "alias": " - xnli_de",
+      "acc,none": 0.43012048192771085,
+      "acc_stderr,none": 0.00992371167540806
+    },
+    "xnli_el": {
+      "alias": " - xnli_el",
+      "acc,none": 0.3582329317269076,
+      "acc_stderr,none": 0.009610788482973929
+    },
+    "xnli_en": {
+      "alias": " - xnli_en",
+      "acc,none": 0.4943775100401606,
+      "acc_stderr,none": 0.010021439203777294
+    },
+    "xnli_es": {
+      "alias": " - xnli_es",
+      "acc,none": 0.39759036144578314,
+      "acc_stderr,none": 0.009809602996075804
+    },
+    "xnli_fr": {
+      "alias": " - xnli_fr",
+      "acc,none": 0.42248995983935744,
+      "acc_stderr,none": 0.009900919227857794
+    },
+    "xnli_hi": {
+      "alias": " - xnli_hi",
+      "acc,none": 0.3650602409638554,
+      "acc_stderr,none": 0.009650194822749635
+    },
+    "xnli_ru": {
+      "alias": " - xnli_ru",
+      "acc,none": 0.43775100401606426,
+      "acc_stderr,none": 0.009944099734290161
+    },
+    "xnli_sw": {
+      "alias": " - xnli_sw",
+      "acc,none": 0.3409638554216867,
+      "acc_stderr,none": 0.009501591178361544
+    },
+    "xnli_th": {
+      "alias": " - xnli_th",
+      "acc,none": 0.348995983935743,
+      "acc_stderr,none": 0.009554095988300678
+    },
+    "xnli_tr": {
+      "alias": " - xnli_tr",
+      "acc,none": 0.3718875502008032,
+      "acc_stderr,none": 0.009687507958631799
+    },
+    "xnli_ur": {
+      "alias": " - xnli_ur",
+      "acc,none": 0.3345381526104418,
+      "acc_stderr,none": 0.009457404390939166
+    },
+    "xnli_vi": {
+      "alias": " - xnli_vi",
+      "acc,none": 0.38393574297188754,
+      "acc_stderr,none": 0.009748321202534375
+    },
+    "xnli_zh": {
+      "alias": " - xnli_zh",
+      "acc,none": 0.3819277108433735,
+      "acc_stderr,none": 0.00973862791451752
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T22-35-32.643193_xstorycloze.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xstorycloze"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xstorycloze": {
+      "acc,none": 0.5407616870224415,
+      "acc_stderr,none": 0.003849468335294993,
+      "alias": "xstorycloze"
+    },
+    "xstorycloze_ar": {
+      "alias": " - xstorycloze_ar",
+      "acc,none": 0.48974189278623426,
+      "acc_stderr,none": 0.012864417047980472
+    },
+    "xstorycloze_en": {
+      "alias": " - xstorycloze_en",
+      "acc,none": 0.6657842488418266,
+      "acc_stderr,none": 0.012139246810918228
+    },
+    "xstorycloze_es": {
+      "alias": " - xstorycloze_es",
+      "acc,none": 0.5698213103904699,
+      "acc_stderr,none": 0.012741052817471078
+    },
+    "xstorycloze_eu": {
+      "alias": " - xstorycloze_eu",
+      "acc,none": 0.5188616810059563,
+      "acc_stderr,none": 0.012857966762464996
+    },
+    "xstorycloze_hi": {
+      "alias": " - xstorycloze_hi",
+      "acc,none": 0.5168762409000662,
+      "acc_stderr,none": 0.012859793919977608
+    },
+    "xstorycloze_id": {
+      "alias": " - xstorycloze_id",
+      "acc,none": 0.5360688285903376,
+      "acc_stderr,none": 0.012833602406620017
+    },
+    "xstorycloze_my": {
+      "alias": " - xstorycloze_my",
+      "acc,none": 0.4930509596293845,
+      "acc_stderr,none": 0.01286588257096072
+    },
+    "xstorycloze_ru": {
+      "alias": " - xstorycloze_ru",
+      "acc,none": 0.5519523494374586,
+      "acc_stderr,none": 0.012797478885304733
+    },
+    "xstorycloze_sw": {
+      "alias": " - xstorycloze_sw",
+      "acc,none": 0.5029781601588352,
+      "acc_stderr,none": 0.012866897066011228
+    },
+    "xstorycloze_te": {
+      "alias": " - xstorycloze_te",
+      "acc,none": 0.5473196558570483,
+      "acc_stderr,none": 0.012809372866181954
+    },
+    "xstorycloze_zh": {
+      "alias": " - xstorycloze_zh",
+      "acc,none": 0.5559232296492389,
+      "acc_stderr,none": 0.012786390539820832
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T22-39-50.695003_xwinograd.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.6556529557203866,
+      "acc_stderr,none": 0.007021810569184652,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "alias": " - xwinograd_en",
+      "acc,none": 0.7290322580645161,
+      "acc_stderr,none": 0.009219643045448322
+    },
+    "xwinograd_fr": {
+      "alias": " - xwinograd_fr",
+      "acc,none": 0.6144578313253012,
+      "acc_stderr,none": 0.0537495779731939
+    },
+    "xwinograd_jp": {
+      "alias": " - xwinograd_jp",
+      "acc,none": 0.5411887382690302,
+      "acc_stderr,none": 0.01609936161806395
+    },
+    "xwinograd_pt": {
+      "alias": " - xwinograd_pt",
+      "acc,none": 0.55893536121673,
+      "acc_stderr,none": 0.0306747666644263
+    },
+    "xwinograd_ru": {
+      "alias": " - xwinograd_ru",
+      "acc,none": 0.5746031746031746,
+      "acc_stderr,none": 0.027900777694976245
+    },
+    "xwinograd_zh": {
+      "alias": " - xwinograd_zh",
+      "acc,none": 0.6428571428571429,
+      "acc_stderr,none": 0.021364573561124416
+    }
+  }
+}

lm_eval/RWKV-5-World-0.4B-v2-20231113-ctx4096/pad_11/0.4.8_2025-03-15T23-45-11.895035_mmlu.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/RWKV-5-World-0.4B-v2-20231113-ctx4096",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 5,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.23764421022646345,
+      "acc_stderr,none": 0.003587790530224594,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "acc,none": 0.24782146652497344,
+      "acc_stderr,none": 0.0062902893350153324,
+      "alias": " - humanities"
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.3253968253968254,
+      "acc_stderr,none": 0.041905964388711366
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.22424242424242424,
+      "acc_stderr,none": 0.03256866661681102
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.2107843137254902,
+      "acc_stderr,none": 0.028626547912437388
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.270042194092827,
+      "acc_stderr,none": 0.028900721906293433
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2396694214876033,
+      "acc_stderr,none": 0.038968789850704164
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.32407407407407407,
+      "acc_stderr,none": 0.04524596007030048
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.20245398773006135,
+      "acc_stderr,none": 0.03157065078911902
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.2398843930635838,
+      "acc_stderr,none": 0.022989592543123567
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.25027932960893856,
+      "acc_stderr,none": 0.014487500852850412
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.19935691318327975,
+      "acc_stderr,none": 0.022691033780549656
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.23148148148148148,
+      "acc_stderr,none": 0.023468429832451152
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.25358539765319427,
+      "acc_stderr,none": 0.01111171533610114
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.2982456140350877,
+      "acc_stderr,none": 0.035087719298245654
+    },
+    "mmlu_other": {
+      "acc,none": 0.24460894753781784,
+      "acc_stderr,none": 0.007691719275922855,
+      "alias": " - other"
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.31,
+      "acc_stderr,none": 0.04648231987117316
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.19245283018867926,
+      "acc_stderr,none": 0.02426297983937226
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.2023121387283237,
+      "acc_stderr,none": 0.03063114553919882
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.24,
+      "acc_stderr,none": 0.042923469599092816
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.3273542600896861,
+      "acc_stderr,none": 0.03149384670994132
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.20388349514563106,
+      "acc_stderr,none": 0.03989139859531771
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.2905982905982906,
+      "acc_stderr,none": 0.029745048572674064
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.29,
+      "acc_stderr,none": 0.04560480215720684
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.23243933588761176,
+      "acc_stderr,none": 0.015104550008905704
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.25163398692810457,
+      "acc_stderr,none": 0.024848018263875195
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.2695035460992908,
+      "acc_stderr,none": 0.026469036818590627
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1801470588235294,
+      "acc_stderr,none": 0.02334516361654486
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.26506024096385544,
+      "acc_stderr,none": 0.03436024037944967
+    },
+    "mmlu_social_sciences": {
+      "acc,none": 0.22099447513812154,
+      "acc_stderr,none": 0.007486677746029817,
+      "alias": " - social sciences"
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.21929824561403508,
+      "acc_stderr,none": 0.0389243110651875
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.19696969696969696,
+      "acc_stderr,none": 0.028335609732463355
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.23316062176165803,
+      "acc_stderr,none": 0.030516111371476008
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.020280805062535726
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.20168067226890757,
+      "acc_stderr,none": 0.026064313406304534
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.21467889908256882,
+      "acc_stderr,none": 0.01760430414925649
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.22900763358778625,
+      "acc_stderr,none": 0.036853466317118506
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.24673202614379086,
+      "acc_stderr,none": 0.0174408203674025
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.22727272727272727,
+      "acc_stderr,none": 0.04013964554072775
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.21224489795918366,
+      "acc_stderr,none": 0.026176967197866767
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.21393034825870647,
+      "acc_stderr,none": 0.0289969096933289
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.0446196043338474
+    },
+    "mmlu_stem": {
+      "acc,none": 0.23184268950206152,
+      "acc_stderr,none": 0.00750773023393252,
+      "alias": " - stem"
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.04163331998932269
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.23703703703703705,
+      "acc_stderr,none": 0.03673731683969506
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.21052631578947367,
+      "acc_stderr,none": 0.033176727875331574
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.2916666666666667,
+      "acc_stderr,none": 0.03800968060554859
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.21,
+      "acc_stderr,none": 0.040936018074033256
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.24,
+      "acc_stderr,none": 0.04292346959909284
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.21568627450980393,
+      "acc_stderr,none": 0.04092563958237655
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.28936170212765955,
+      "acc_stderr,none": 0.02964400657700962
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.25517241379310346,
+      "acc_stderr,none": 0.03632984052707842
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.21693121693121692,
+      "acc_stderr,none": 0.02122708244944506
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.1935483870967742,
+      "acc_stderr,none": 0.022475258525536057
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.1921182266009852,
+      "acc_stderr,none": 0.027719315709614778
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.04229525846816506
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.26296296296296295,
+      "acc_stderr,none": 0.026842057873833706
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.2251655629139073,
+      "acc_stderr,none": 0.03410435282008937
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.16203703703703703,
+      "acc_stderr,none": 0.02513045365226846
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.2857142857142857,
+      "acc_stderr,none": 0.04287858751340456
+    }
+  }
+}

lm_eval/RWKV-x060-World-1B6-v2.1-20240328-ctx4096/pad_11/0.4.8_2025-03-16T09-41-13.419478_lambada_openai.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "alias": "lambada_openai",
+      "perplexity,none": 4.637882037692772,
+      "perplexity_stderr,none": 0.10579675670275654,
+      "acc,none": 0.6741703861828061,
+      "acc_stderr,none": 0.0065296843174760975
+    }
+  }
+}

lm_eval/RWKV-x060-World-1B6-v2.1-20240328-ctx4096/pad_11/0.4.8_2025-03-16T10-43-54.649276_hellaswag.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "hellaswag": {
+      "alias": "hellaswag",
+      "acc,none": 0.4644493128858793,
+      "acc_stderr,none": 0.004977152746478598,
+      "acc_norm,none": 0.6107349133638718,
+      "acc_norm_stderr,none": 0.004865871290143343
+    }
+  }
+}

lm_eval/RWKV-x060-World-1B6-v2.1-20240328-ctx4096/pad_11/0.4.8_2025-03-16T10-47-38.776674_piqa.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "piqa": {
+      "alias": "piqa",
+      "acc,none": 0.7437431991294886,
+      "acc_stderr,none": 0.010185787831565084,
+      "acc_norm,none": 0.7453754080522307,
+      "acc_norm_stderr,none": 0.010164432237060482
+    }
+  }
+}

lm_eval/RWKV-x060-World-1B6-v2.1-20240328-ctx4096/pad_11/0.4.8_2025-03-16T10-55-41.507473_arc_easy.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_easy": {
+      "alias": "arc_easy",
+      "acc,none": 0.6426767676767676,
+      "acc_stderr,none": 0.00983320561246312,
+      "acc_norm,none": 0.6216329966329966,
+      "acc_norm_stderr,none": 0.009951575683331947
+    }
+  }
+}

lm_eval/RWKV-x060-World-1B6-v2.1-20240328-ctx4096/pad_11/0.4.8_2025-03-16T11-00-09.309689_arc_challenge.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-1B6-v2.1-20240328-ctx4096",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_challenge": {
+      "alias": "arc_challenge",
+      "acc,none": 0.3097269624573379,
+      "acc_stderr,none": 0.01351205841523836,
+      "acc_norm,none": 0.3370307167235495,
+      "acc_norm_stderr,none": 0.013813476652902274
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T00-45-26.618704_lambada_openai.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "alias": "lambada_openai",
+      "perplexity,none": 3.8612709402913734,
+      "perplexity_stderr,none": 0.08105989741229948,
+      "acc,none": 0.7172520861634,
+      "acc_stderr,none": 0.006274045840971217
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T02-36-50.849530_hellaswag.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "hellaswag": {
+      "alias": "hellaswag",
+      "acc,none": 0.5053774148575981,
+      "acc_stderr,none": 0.004989492828168538,
+      "acc_norm,none": 0.6835291774546903,
+      "acc_norm_stderr,none": 0.004641484273335084
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T02-43-09.170340_piqa.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "piqa": {
+      "alias": "piqa",
+      "acc,none": 0.763873775843308,
+      "acc_stderr,none": 0.009908965890558216,
+      "acc_norm,none": 0.7633297062023939,
+      "acc_norm_stderr,none": 0.009916841655042806
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T02-57-01.574833_arc_easy.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_easy": {
+      "alias": "arc_easy",
+      "acc,none": 0.7117003367003367,
+      "acc_stderr,none": 0.009294774252029625,
+      "acc_norm,none": 0.6717171717171717,
+      "acc_norm_stderr,none": 0.009635749509262161
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T03-04-40.682395_arc_challenge.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_challenge": {
+      "alias": "arc_challenge",
+      "acc,none": 0.35580204778157,
+      "acc_stderr,none": 0.01399057113791876,
+      "acc_norm,none": 0.39078498293515357,
+      "acc_norm_stderr,none": 0.014258563880513778
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T08-11-37.430416_glue.json

@@ -0,0 +1,69 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "glue"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "cola": {
+      "alias": "cola",
+      "mcc,none": 0.07139041138969343,
+      "mcc_stderr,none": 0.03374782039086001
+    },
+    "mnli": {
+      "alias": "mnli",
+      "acc,none": 0.37748344370860926,
+      "acc_stderr,none": 0.004893294795433131
+    },
+    "mnli_mismatch": {
+      "alias": "mnli_mismatch",
+      "acc,none": 0.3817127746135069,
+      "acc_stderr,none": 0.004899645239995305
+    },
+    "mrpc": {
+      "alias": "mrpc",
+      "acc,none": 0.6397058823529411,
+      "acc_stderr,none": 0.023796963985532167,
+      "f1,none": 0.7111984282907662,
+      "f1_stderr,none": 0.02254781842194292
+    },
+    "qnli": {
+      "alias": "qnli",
+      "acc,none": 0.5317591067179206,
+      "acc_stderr,none": 0.006751749019242126
+    },
+    "qqp": {
+      "alias": "qqp",
+      "acc,none": 0.6318080633193174,
+      "acc_stderr,none": 0.002398740231240921,
+      "f1,none": 0.0,
+      "f1_stderr,none": 0.0
+    },
+    "rte": {
+      "alias": "rte",
+      "acc,none": 0.6137184115523465,
+      "acc_stderr,none": 0.029307720385270512
+    },
+    "sst2": {
+      "alias": "sst2",
+      "acc,none": 0.893348623853211,
+      "acc_stderr,none": 0.010458867008246879
+    },
+    "wnli": {
+      "alias": "wnli",
+      "acc,none": 0.43661971830985913,
+      "acc_stderr,none": 0.0592793555841297
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T08-14-56.977956_winogrande.json

@@ -0,0 +1,25 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "winogrande": {
+      "alias": "winogrande",
+      "acc,none": 0.6629834254143646,
+      "acc_stderr,none": 0.01328495576939525
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T08-29-25.494712_sciq.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "sciq"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "sciq": {
+      "alias": "sciq",
+      "acc,none": 0.922,
+      "acc_stderr,none": 0.008484573530118585,
+      "acc_norm,none": 0.886,
+      "acc_norm_stderr,none": 0.01005510343582333
+    }
+  }
+}

lm_eval/RWKV-x060-World-3B-v2.1-20240417-ctx4096/pad_11/0.4.8_2025-03-15T16-15-02.568125_mmlu.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/RWKV-x060-World-3B-v2.1-20240417-ctx4096.pth",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.8",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.28300811850163793,
+      "acc_stderr,none": 0.0037926401279525677,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "acc,none": 0.29330499468650373,
+      "acc_stderr,none": 0.00661831036231422,
+      "alias": " - humanities"
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.23809523809523808,
+      "acc_stderr,none": 0.03809523809523811
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.37575757575757573,
+      "acc_stderr,none": 0.03781887353205982
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.29411764705882354,
+      "acc_stderr,none": 0.031980016601150726
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.2616033755274262,
+      "acc_stderr,none": 0.028609516716994934
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.38016528925619836,
+      "acc_stderr,none": 0.04431324501968432
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.2777777777777778,
+      "acc_stderr,none": 0.04330043749650741
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.3312883435582822,
+      "acc_stderr,none": 0.03697983910025588
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.3063583815028902,
+      "acc_stderr,none": 0.024818350129436596
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.2324022346368715,
+      "acc_stderr,none": 0.014125968754673403
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.33762057877813506,
+      "acc_stderr,none": 0.02685882587948854
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.3425925925925926,
+      "acc_stderr,none": 0.02640614597362568
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.2907431551499348,
+      "acc_stderr,none": 0.011598062372851981
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.3508771929824561,
+      "acc_stderr,none": 0.03660298834049163
+    },
+    "mmlu_other": {
+      "acc,none": 0.28451882845188287,
+      "acc_stderr,none": 0.008058064756071836,
+      "alias": " - other"
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.32,
+      "acc_stderr,none": 0.046882617226215034
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.24150943396226415,
+      "acc_stderr,none": 0.02634148037111836
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.2543352601156069,
+      "acc_stderr,none": 0.0332055644308557
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.32,
+      "acc_stderr,none": 0.04688261722621505
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.2242152466367713,
+      "acc_stderr,none": 0.02799153425851952
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.20388349514563106,
+      "acc_stderr,none": 0.0398913985953177
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.3247863247863248,
+      "acc_stderr,none": 0.030679022765498835
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.19,
+      "acc_stderr,none": 0.03942772444036623
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.351213282247765,
+      "acc_stderr,none": 0.01706998205149943
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.29411764705882354,
+      "acc_stderr,none": 0.026090162504279046
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.2765957446808511,
+      "acc_stderr,none": 0.026684564340461
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.22794117647058823,
+      "acc_stderr,none": 0.025483081468029804
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.2469879518072289,
+      "acc_stderr,none": 0.03357351982064536
+    },
+    "mmlu_social_sciences": {
+      "acc,none": 0.27494312642183943,
+      "acc_stderr,none": 0.008044129845426675,
+      "alias": " - social sciences"
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.24561403508771928,
+      "acc_stderr,none": 0.040493392977481425
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.29797979797979796,
+      "acc_stderr,none": 0.03258630383836556
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.31088082901554404,
+      "acc_stderr,none": 0.03340361906276586
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.22564102564102564,
+      "acc_stderr,none": 0.021193632525148533
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.23529411764705882,
+      "acc_stderr,none": 0.02755361446786379
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.27155963302752295,
+      "acc_stderr,none": 0.019069098363191445
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.2900763358778626,
+      "acc_stderr,none": 0.03980066246467766
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.29248366013071897,
+      "acc_stderr,none": 0.01840341571010978
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.36363636363636365,
+      "acc_stderr,none": 0.046075820907199756
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.2612244897959184,
+      "acc_stderr,none": 0.028123429335142787
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.3034825870646766,
+      "acc_stderr,none": 0.03251006816458619
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_stem": {
+      "acc,none": 0.27402473834443386,
+      "acc_stderr,none": 0.007942050435080712,
+      "alias": " - stem"
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.042295258468165065
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.35555555555555557,
+      "acc_stderr,none": 0.04135176749720386
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.2894736842105263,
+      "acc_stderr,none": 0.03690677986137282
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.2777777777777778,
+      "acc_stderr,none": 0.03745554791462457
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.19,
+      "acc_stderr,none": 0.039427724440366234
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.0446196043338474
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.0440844002276808
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.21568627450980393,
+      "acc_stderr,none": 0.04092563958237656
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.31,
+      "acc_stderr,none": 0.04648231987117316
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.2425531914893617,
+      "acc_stderr,none": 0.028020226271200217
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.3103448275862069,
+      "acc_stderr,none": 0.038552896163789485
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.2698412698412698,
+      "acc_stderr,none": 0.022860838309232072
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.2870967741935484,
+      "acc_stderr,none": 0.02573654274559452
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.30049261083743845,
+      "acc_stderr,none": 0.03225799476233485
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.33,
+      "acc_stderr,none": 0.047258156262526045
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.25925925925925924,
+      "acc_stderr,none": 0.026719240783712184
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.2913907284768212,
+      "acc_stderr,none": 0.03710185726119994
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.22685185185185186,
+      "acc_stderr,none": 0.028561650102422263
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.30357142857142855,
+      "acc_stderr,none": 0.04364226155841044
+    }
+  }
+}