Upload folder using huggingface_hub

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/arc_challenge_2025-03-10T17-01-44.603102.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "arc_challenge": {
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.012653835621466646,
+      "acc_norm,none": 0.2815699658703072,
+      "acc_norm_stderr,none": 0.013143376735009022,
+      "alias": "arc_challenge"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/arc_challenge_2025-03-11T22-06-12.454703.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "arc_challenge"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_challenge": {
+      "acc,none": 0.2636518771331058,
+      "acc_stderr,none": 0.012875929151297044,
+      "acc_norm,none": 0.29436860068259385,
+      "acc_norm_stderr,none": 0.013318528460539422,
+      "alias": "arc_challenge"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/arc_easy_2025-03-10T17-00-09.413507.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "arc_easy": {
+      "acc,none": 0.5597643097643098,
+      "acc_stderr,none": 0.010186228624515655,
+      "acc_norm,none": 0.47895622895622897,
+      "acc_norm_stderr,none": 0.010250692602022576,
+      "alias": "arc_easy"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/arc_easy_2025-03-11T22-04-44.986766.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "arc_easy"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "arc_easy": {
+      "acc,none": 0.5841750841750841,
+      "acc_stderr,none": 0.010113348244647876,
+      "acc_norm,none": 0.5294612794612794,
+      "acc_norm_stderr,none": 0.010241957728409676,
+      "alias": "arc_easy"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/glue_2025-03-10T18-04-42.259056.json

@@ -0,0 +1,78 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "glue"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "glue": {
+      "acc,none": 0.37153108623153885,
+      "acc_stderr,none": 0.001850330650439922,
+      "f1,none": 0.535035415960775,
+      "f1_stderr,none": 0.002562079856834694,
+      "mcc,none": 0.03494082007962207,
+      "mcc_stderr,none": 0.031408601572730226,
+      "alias": "glue"
+    },
+    "cola": {
+      "mcc,none": 0.03494082007962207,
+      "mcc_stderr,none": 0.031408601572730226,
+      "alias": " - cola"
+    },
+    "mnli": {
+      "acc,none": 0.3182883341823739,
+      "acc_stderr,none": 0.004702054913568257,
+      "alias": " - mnli"
+    },
+    "mnli_mismatch": {
+      "acc,none": 0.31814483319772174,
+      "acc_stderr,none": 0.004697422861392529,
+      "alias": " - mnli_mismatch"
+    },
+    "mrpc": {
+      "acc,none": 0.4117647058823529,
+      "acc_stderr,none": 0.024395116363488303,
+      "f1,none": 0.38461538461538464,
+      "f1_stderr,none": 0.031397994005677686,
+      "alias": " - mrpc"
+    },
+    "qnli": {
+      "acc,none": 0.5167490389895661,
+      "acc_stderr,none": 0.006761613680941321,
+      "alias": " - qnli"
+    },
+    "qqp": {
+      "acc,none": 0.3706158792975513,
+      "acc_stderr,none": 0.002402002209823611,
+      "f1,none": 0.5365533821440279,
+      "f1_stderr,none": 0.002568480759283663,
+      "alias": " - qqp"
+    },
+    "rte": {
+      "acc,none": 0.49097472924187724,
+      "acc_stderr,none": 0.030091559826331334,
+      "alias": " - rte"
+    },
+    "sst2": {
+      "acc,none": 0.6387614678899083,
+      "acc_stderr,none": 0.01627636093868883,
+      "alias": " - sst2"
+    },
+    "wnli": {
+      "acc,none": 0.49295774647887325,
+      "acc_stderr,none": 0.05975550263548289,
+      "alias": " - wnli"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/glue_2025-03-12T10-56-43.655674.json

@@ -0,0 +1,78 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "glue"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "glue": {
+      "f1,none": 0.5346106647322794,
+      "f1_stderr,none": 0.002567335583973214,
+      "acc,none": 0.37517865650309673,
+      "acc_stderr,none": 0.0018509720446141336,
+      "mcc,none": 0.0024285045843052164,
+      "mcc_stderr,none": 0.030955008759027437,
+      "alias": "glue"
+    },
+    "cola": {
+      "mcc,none": 0.0024285045843052164,
+      "mcc_stderr,none": 0.03095500875902744,
+      "alias": " - cola"
+    },
+    "mnli": {
+      "acc,none": 0.31798267957208354,
+      "acc_stderr,none": 0.004700850152962885,
+      "alias": " - mnli"
+    },
+    "mnli_mismatch": {
+      "acc,none": 0.31814483319772174,
+      "acc_stderr,none": 0.004697422861392529,
+      "alias": " - mnli_mismatch"
+    },
+    "mrpc": {
+      "acc,none": 0.40441176470588236,
+      "acc_stderr,none": 0.024326954407515665,
+      "f1,none": 0.3816793893129771,
+      "f1_stderr,none": 0.03140466535806043,
+      "alias": " - mrpc"
+    },
+    "qnli": {
+      "acc,none": 0.518762584660443,
+      "acc_stderr,none": 0.006760645556775843,
+      "alias": " - qnli"
+    },
+    "qqp": {
+      "acc,none": 0.3746970071728914,
+      "acc_stderr,none": 0.0024073479491393064,
+      "f1,none": 0.5361539731757885,
+      "f1_stderr,none": 0.0025738212531611635,
+      "alias": " - qqp"
+    },
+    "rte": {
+      "acc,none": 0.4620938628158845,
+      "acc_stderr,none": 0.030009848912529113,
+      "alias": " - rte"
+    },
+    "sst2": {
+      "acc,none": 0.7282110091743119,
+      "acc_stderr,none": 0.01507424165684193,
+      "alias": " - sst2"
+    },
+    "wnli": {
+      "acc,none": 0.5633802816901409,
+      "acc_stderr,none": 0.0592793555841297,
+      "alias": " - wnli"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/hellaswag_2025-03-10T16-52-48.967622.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.34793865763792076,
+      "acc_stderr,none": 0.0047534298066454345,
+      "acc_norm,none": 0.4287990440151364,
+      "acc_norm_stderr,none": 0.004938930143234452,
+      "alias": "hellaswag"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/hellaswag_2025-03-11T22-00-37.986152.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "hellaswag"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.34793865763792076,
+      "acc_stderr,none": 0.004753429806645434,
+      "acc_norm,none": 0.4279028082055367,
+      "acc_norm_stderr,none": 0.004937635112830291,
+      "alias": "hellaswag"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_multilingual_2025-03-10T18-36-31.528678.json

@@ -0,0 +1,62 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_multilingual"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "lambada_multilingual": {
+      "acc,none": 0.3149233456239084,
+      "acc_stderr,none": 0.0028367730218709125,
+      "perplexity,none": 172.520659113721,
+      "perplexity_stderr,none": 5.367739924435885,
+      "alias": "lambada_multilingual"
+    },
+    "lambada_openai_mt_de": {
+      "perplexity,none": 217.9862646485565,
+      "perplexity_stderr,none": 13.330594833675963,
+      "acc,none": 0.24180089268387348,
+      "acc_stderr,none": 0.005965305048434238,
+      "alias": " - lambada_openai_mt_de"
+    },
+    "lambada_openai_mt_en": {
+      "perplexity,none": 12.364936956333898,
+      "perplexity_stderr,none": 0.3764505210612126,
+      "acc,none": 0.49117019212109453,
+      "acc_stderr,none": 0.006964891360529504,
+      "alias": " - lambada_openai_mt_en"
+    },
+    "lambada_openai_mt_es": {
+      "perplexity,none": 270.97982131432593,
+      "perplexity_stderr,none": 16.264671005675716,
+      "acc,none": 0.24936929943722103,
+      "acc_stderr,none": 0.006027631959331146,
+      "alias": " - lambada_openai_mt_es"
+    },
+    "lambada_openai_mt_fr": {
+      "perplexity,none": 131.45772697351666,
+      "perplexity_stderr,none": 7.755992287298453,
+      "acc,none": 0.31651465165922765,
+      "acc_stderr,none": 0.006479978824925184,
+      "alias": " - lambada_openai_mt_fr"
+    },
+    "lambada_openai_mt_it": {
+      "perplexity,none": 229.81454567587204,
+      "perplexity_stderr,none": 14.757180777900542,
+      "acc,none": 0.27576169221812535,
+      "acc_stderr,none": 0.0062261529732960475,
+      "alias": " - lambada_openai_mt_it"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_multilingual_2025-03-12T11-43-36.700098.json

@@ -0,0 +1,62 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_multilingual"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_multilingual": {
+      "acc,none": 0.3162817776052785,
+      "acc_stderr,none": 0.0028416091446470397,
+      "perplexity,none": 170.41860290665036,
+      "perplexity_stderr,none": 5.292397572601908,
+      "alias": "lambada_multilingual"
+    },
+    "lambada_openai_mt_de": {
+      "perplexity,none": 213.62853047491745,
+      "perplexity_stderr,none": 13.048252432195138,
+      "acc,none": 0.246070250339608,
+      "acc_stderr,none": 0.006000771208158281,
+      "alias": " - lambada_openai_mt_de"
+    },
+    "lambada_openai_mt_en": {
+      "perplexity,none": 12.595873481361684,
+      "perplexity_stderr,none": 0.38225300601901746,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": " - lambada_openai_mt_en"
+    },
+    "lambada_openai_mt_es": {
+      "perplexity,none": 270.8065908859296,
+      "perplexity_stderr,none": 16.220288193051754,
+      "acc,none": 0.24665243547448087,
+      "acc_stderr,none": 0.00600554563121515,
+      "alias": " - lambada_openai_mt_es"
+    },
+    "lambada_openai_mt_fr": {
+      "perplexity,none": 129.6995246749841,
+      "perplexity_stderr,none": 7.629690056718077,
+      "acc,none": 0.3188433921987192,
+      "acc_stderr,none": 0.006492684061449839,
+      "alias": " - lambada_openai_mt_fr"
+    },
+    "lambada_openai_mt_it": {
+      "perplexity,none": 225.36249501605897,
+      "perplexity_stderr,none": 14.440355720685359,
+      "acc,none": 0.2800310498738599,
+      "acc_stderr,none": 0.006255644360929012,
+      "alias": " - lambada_openai_mt_it"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-10T11-51-01.125594.json

@@ -0,0 +1,16 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.595873463983008,
+      "perplexity_stderr,none": 0.38225300476755747,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-10T11-56-47.764011.json

@@ -0,0 +1,16 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.595873481361684,
+      "perplexity_stderr,none": 0.38225300601901746,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-10T12-29-06.749140.json

@@ -0,0 +1,16 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.595873463983008,
+      "perplexity_stderr,none": 0.38225300476755747,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-10T13-05-32.078254.json

@@ -0,0 +1,16 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.595873481361684,
+      "perplexity_stderr,none": 0.38225300601901746,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-10T13-34-48.078208.json

@@ -0,0 +1,16 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.364936961529702,
+      "perplexity_stderr,none": 0.376450521079655,
+      "acc,none": 0.49117019212109453,
+      "acc_stderr,none": 0.006964891360529504,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/lambada_openai_2025-03-11T21-36-27.701706.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "lambada_openai"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "lambada_openai": {
+      "perplexity,none": 12.595873481361684,
+      "perplexity_stderr,none": 0.38225300601901746,
+      "acc,none": 0.48981176013972444,
+      "acc_stderr,none": 0.006964531366864929,
+      "alias": "lambada_openai"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/mmlu_2025-03-10T18-27-09.707005.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "mmlu": {
+      "acc,none": 0.2506053268765133,
+      "acc_stderr,none": 0.0036568725365202018,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "alias": " - humanities",
+      "acc,none": 0.2516471838469713,
+      "acc_stderr,none": 0.006330269741621689
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.25396825396825395,
+      "acc_stderr,none": 0.03893259610604673
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.28484848484848485,
+      "acc_stderr,none": 0.03524390844511782
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.25980392156862747,
+      "acc_stderr,none": 0.030778554678693268
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.2616033755274262,
+      "acc_stderr,none": 0.028609516716994927
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.256198347107438,
+      "acc_stderr,none": 0.03984979653302871
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.26851851851851855,
+      "acc_stderr,none": 0.04284467968052191
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.2392638036809816,
+      "acc_stderr,none": 0.0335195387952127
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.2514450867052023,
+      "acc_stderr,none": 0.02335736578587403
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.264804469273743,
+      "acc_stderr,none": 0.014756906483260666
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.20257234726688103,
+      "acc_stderr,none": 0.022827317491059672
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.23148148148148148,
+      "acc_stderr,none": 0.023468429832451166
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.2542372881355932,
+      "acc_stderr,none": 0.011121129007840671
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.22807017543859648,
+      "acc_stderr,none": 0.032180937956023566
+    },
+    "mmlu_other": {
+      "alias": " - other",
+      "acc,none": 0.26359832635983266,
+      "acc_stderr,none": 0.007894767937155992
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.23,
+      "acc_stderr,none": 0.04229525846816506
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.22641509433962265,
+      "acc_stderr,none": 0.025757559893106723
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.24277456647398843,
+      "acc_stderr,none": 0.0326926380614177
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.28,
+      "acc_stderr,none": 0.04512608598542127
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.3273542600896861,
+      "acc_stderr,none": 0.03149384670994131
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.2912621359223301,
+      "acc_stderr,none": 0.044986763205729224
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.27350427350427353,
+      "acc_stderr,none": 0.029202540153431166
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.27458492975734355,
+      "acc_stderr,none": 0.015959829933084032
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.2875816993464052,
+      "acc_stderr,none": 0.02591780611714716
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.25177304964539005,
+      "acc_stderr,none": 0.0258921511567094
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1801470588235294,
+      "acc_stderr,none": 0.02334516361654484
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.27710843373493976,
+      "acc_stderr,none": 0.034843315926805875
+    },
+    "mmlu_social_sciences": {
+      "alias": " - social_sciences",
+      "acc,none": 0.24406889827754305,
+      "acc_stderr,none": 0.007741968672346153
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.21929824561403508,
+      "acc_stderr,none": 0.03892431106518752
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.25252525252525254,
+      "acc_stderr,none": 0.030954055470365907
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.24870466321243523,
+      "acc_stderr,none": 0.03119584087770029
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2230769230769231,
+      "acc_stderr,none": 0.02110773012724399
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.27310924369747897,
+      "acc_stderr,none": 0.028942004040998164
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.22752293577981653,
+      "acc_stderr,none": 0.0179744635787765
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.19083969465648856,
+      "acc_stderr,none": 0.03446513350752598
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.28104575163398693,
+      "acc_stderr,none": 0.018185218954318075
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.2909090909090909,
+      "acc_stderr,none": 0.04350271442923243
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.21224489795918366,
+      "acc_stderr,none": 0.026176967197866767
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.24875621890547264,
+      "acc_stderr,none": 0.030567675938916718
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.21,
+      "acc_stderr,none": 0.040936018074033256
+    },
+    "mmlu_stem": {
+      "alias": " - stem",
+      "acc,none": 0.24262607040913417,
+      "acc_stderr,none": 0.007641847233625673
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.2074074074074074,
+      "acc_stderr,none": 0.03502553170678317
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.21052631578947367,
+      "acc_stderr,none": 0.03317672787533157
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.2986111111111111,
+      "acc_stderr,none": 0.03827052357950756
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.040201512610368445
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.044619604333847394
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.23529411764705882,
+      "acc_stderr,none": 0.04220773659171451
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.04408440022768079
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.2297872340425532,
+      "acc_stderr,none": 0.02750175294441242
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.25517241379310346,
+      "acc_stderr,none": 0.03632984052707841
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.2671957671957672,
+      "acc_stderr,none": 0.02278967314577656
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.24193548387096775,
+      "acc_stderr,none": 0.024362599693031093
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.2315270935960591,
+      "acc_stderr,none": 0.029678333141444455
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.04408440022768078
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.24074074074074073,
+      "acc_stderr,none": 0.026067159222275798
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.23841059602649006,
+      "acc_stderr,none": 0.0347918557259966
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.18981481481481483,
+      "acc_stderr,none": 0.026744714834691933
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.29464285714285715,
+      "acc_stderr,none": 0.04327040932578728
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/mmlu_2025-03-12T11-34-06.812808.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.25231448511608034,
+      "acc_stderr,none": 0.003663630582486529,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "alias": " - humanities",
+      "acc,none": 0.24930924548352817,
+      "acc_stderr,none": 0.006310523103517653
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.2222222222222222,
+      "acc_stderr,none": 0.037184890068181146
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.2727272727272727,
+      "acc_stderr,none": 0.03477691162163659
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.2647058823529412,
+      "acc_stderr,none": 0.03096451792692341
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.26582278481012656,
+      "acc_stderr,none": 0.028756799629658342
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2644628099173554,
+      "acc_stderr,none": 0.04026187527591206
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.2962962962962963,
+      "acc_stderr,none": 0.04414343666854933
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.26380368098159507,
+      "acc_stderr,none": 0.03462419931615624
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.21676300578034682,
+      "acc_stderr,none": 0.022183477668412856
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.25139664804469275,
+      "acc_stderr,none": 0.014508979453553979
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.21864951768488747,
+      "acc_stderr,none": 0.023475581417861102
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.23765432098765432,
+      "acc_stderr,none": 0.023683591837008557
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.25554106910039115,
+      "acc_stderr,none": 0.011139857833598511
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.22807017543859648,
+      "acc_stderr,none": 0.03218093795602357
+    },
+    "mmlu_other": {
+      "alias": " - other",
+      "acc,none": 0.26263276472481495,
+      "acc_stderr,none": 0.007885730301877262
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.044619604333847394
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.2490566037735849,
+      "acc_stderr,none": 0.026616482980501704
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.24855491329479767,
+      "acc_stderr,none": 0.03295304696818318
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.34977578475336324,
+      "acc_stderr,none": 0.03200736719484503
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.3106796116504854,
+      "acc_stderr,none": 0.045821241601615506
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.27350427350427353,
+      "acc_stderr,none": 0.029202540153431166
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.27330779054916987,
+      "acc_stderr,none": 0.01593668106262856
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.25163398692810457,
+      "acc_stderr,none": 0.024848018263875192
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.24113475177304963,
+      "acc_stderr,none": 0.025518731049537766
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.1875,
+      "acc_stderr,none": 0.023709788253811766
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.27710843373493976,
+      "acc_stderr,none": 0.034843315926805875
+    },
+    "mmlu_social_sciences": {
+      "alias": " - social_sciences",
+      "acc,none": 0.2463438414039649,
+      "acc_stderr,none": 0.007756497563842694
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.23684210526315788,
+      "acc_stderr,none": 0.039994238792813365
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.26262626262626265,
+      "acc_stderr,none": 0.031353050095330855
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.21761658031088082,
+      "acc_stderr,none": 0.02977866303775295
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.2512820512820513,
+      "acc_stderr,none": 0.02199201666237056
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.2184873949579832,
+      "acc_stderr,none": 0.026841514322958945
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.23669724770642203,
+      "acc_stderr,none": 0.01822407811729909
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.1450381679389313,
+      "acc_stderr,none": 0.03088466108951539
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.2908496732026144,
+      "acc_stderr,none": 0.018373116915903966
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.3090909090909091,
+      "acc_stderr,none": 0.044262946482000985
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.21224489795918366,
+      "acc_stderr,none": 0.026176967197866767
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.263681592039801,
+      "acc_stderr,none": 0.03115715086935556
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.04163331998932269
+    },
+    "mmlu_stem": {
+      "alias": " - stem",
+      "acc,none": 0.2524579765302886,
+      "acc_stderr,none": 0.0077368608002292985
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.28,
+      "acc_stderr,none": 0.04512608598542127
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.22962962962962963,
+      "acc_stderr,none": 0.036333844140734636
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.17105263157894737,
+      "acc_stderr,none": 0.030643607071677098
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.3194444444444444,
+      "acc_stderr,none": 0.03899073687357335
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.041633319989322716
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.0440844002276808
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.19,
+      "acc_stderr,none": 0.03942772444036623
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.24509803921568626,
+      "acc_stderr,none": 0.042801058373643945
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.25,
+      "acc_stderr,none": 0.04351941398892446
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.24680851063829787,
+      "acc_stderr,none": 0.028185441301234106
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.2413793103448276,
+      "acc_stderr,none": 0.03565998174135303
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.2804232804232804,
+      "acc_stderr,none": 0.023135287974325625
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.24838709677419354,
+      "acc_stderr,none": 0.024580028921481006
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.2857142857142857,
+      "acc_stderr,none": 0.03178529710642749
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.32,
+      "acc_stderr,none": 0.04688261722621504
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.25925925925925924,
+      "acc_stderr,none": 0.02671924078371217
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.24503311258278146,
+      "acc_stderr,none": 0.03511807571804724
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.2222222222222222,
+      "acc_stderr,none": 0.02835321286686344
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.24107142857142858,
+      "acc_stderr,none": 0.04059867246952687
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/mmlu_2025-03-12T18-34-12.292504.json

@@ -0,0 +1,330 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "mmlu"
+  ],
+  "num_fewshot": 5,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "mmlu": {
+      "acc,none": 0.25324027916251246,
+      "acc_stderr,none": 0.003666930438461816,
+      "alias": "mmlu"
+    },
+    "mmlu_humanities": {
+      "alias": " - humanities",
+      "acc,none": 0.2501594048884166,
+      "acc_stderr,none": 0.00631067205291355
+    },
+    "mmlu_formal_logic": {
+      "alias": "  - formal_logic",
+      "acc,none": 0.16666666666666666,
+      "acc_stderr,none": 0.03333333333333336
+    },
+    "mmlu_high_school_european_history": {
+      "alias": "  - high_school_european_history",
+      "acc,none": 0.21212121212121213,
+      "acc_stderr,none": 0.031922715695482995
+    },
+    "mmlu_high_school_us_history": {
+      "alias": "  - high_school_us_history",
+      "acc,none": 0.27941176470588236,
+      "acc_stderr,none": 0.031493281045079556
+    },
+    "mmlu_high_school_world_history": {
+      "alias": "  - high_school_world_history",
+      "acc,none": 0.25316455696202533,
+      "acc_stderr,none": 0.028304657943035313
+    },
+    "mmlu_international_law": {
+      "alias": "  - international_law",
+      "acc,none": 0.2644628099173554,
+      "acc_stderr,none": 0.04026187527591204
+    },
+    "mmlu_jurisprudence": {
+      "alias": "  - jurisprudence",
+      "acc,none": 0.3425925925925926,
+      "acc_stderr,none": 0.04587904741301812
+    },
+    "mmlu_logical_fallacies": {
+      "alias": "  - logical_fallacies",
+      "acc,none": 0.3006134969325153,
+      "acc_stderr,none": 0.03602511318806771
+    },
+    "mmlu_moral_disputes": {
+      "alias": "  - moral_disputes",
+      "acc,none": 0.24855491329479767,
+      "acc_stderr,none": 0.023267528432100174
+    },
+    "mmlu_moral_scenarios": {
+      "alias": "  - moral_scenarios",
+      "acc,none": 0.23016759776536314,
+      "acc_stderr,none": 0.014078339253425819
+    },
+    "mmlu_philosophy": {
+      "alias": "  - philosophy",
+      "acc,none": 0.24758842443729903,
+      "acc_stderr,none": 0.024513879973621967
+    },
+    "mmlu_prehistory": {
+      "alias": "  - prehistory",
+      "acc,none": 0.2839506172839506,
+      "acc_stderr,none": 0.025089478523765127
+    },
+    "mmlu_professional_law": {
+      "alias": "  - professional_law",
+      "acc,none": 0.24837027379400262,
+      "acc_stderr,none": 0.011035212598034503
+    },
+    "mmlu_world_religions": {
+      "alias": "  - world_religions",
+      "acc,none": 0.2573099415204678,
+      "acc_stderr,none": 0.03352799844161865
+    },
+    "mmlu_other": {
+      "alias": " - other",
+      "acc,none": 0.271000965561635,
+      "acc_stderr,none": 0.007979305318220201
+    },
+    "mmlu_business_ethics": {
+      "alias": "  - business_ethics",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.0440844002276808
+    },
+    "mmlu_clinical_knowledge": {
+      "alias": "  - clinical_knowledge",
+      "acc,none": 0.23018867924528302,
+      "acc_stderr,none": 0.025907897122408173
+    },
+    "mmlu_college_medicine": {
+      "alias": "  - college_medicine",
+      "acc,none": 0.2543352601156069,
+      "acc_stderr,none": 0.0332055644308557
+    },
+    "mmlu_global_facts": {
+      "alias": "  - global_facts",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.04461960433384739
+    },
+    "mmlu_human_aging": {
+      "alias": "  - human_aging",
+      "acc,none": 0.32286995515695066,
+      "acc_stderr,none": 0.03138147637575498
+    },
+    "mmlu_management": {
+      "alias": "  - management",
+      "acc,none": 0.2524271844660194,
+      "acc_stderr,none": 0.04301250399690877
+    },
+    "mmlu_marketing": {
+      "alias": "  - marketing",
+      "acc,none": 0.2606837606837607,
+      "acc_stderr,none": 0.028760348956523414
+    },
+    "mmlu_medical_genetics": {
+      "alias": "  - medical_genetics",
+      "acc,none": 0.27,
+      "acc_stderr,none": 0.044619604333847394
+    },
+    "mmlu_miscellaneous": {
+      "alias": "  - miscellaneous",
+      "acc,none": 0.2707535121328225,
+      "acc_stderr,none": 0.01588988836256049
+    },
+    "mmlu_nutrition": {
+      "alias": "  - nutrition",
+      "acc,none": 0.2973856209150327,
+      "acc_stderr,none": 0.02617390850671858
+    },
+    "mmlu_professional_accounting": {
+      "alias": "  - professional_accounting",
+      "acc,none": 0.2801418439716312,
+      "acc_stderr,none": 0.02678917235114025
+    },
+    "mmlu_professional_medicine": {
+      "alias": "  - professional_medicine",
+      "acc,none": 0.2867647058823529,
+      "acc_stderr,none": 0.02747227447323382
+    },
+    "mmlu_virology": {
+      "alias": "  - virology",
+      "acc,none": 0.2289156626506024,
+      "acc_stderr,none": 0.03270745277352477
+    },
+    "mmlu_social_sciences": {
+      "alias": " - social_sciences",
+      "acc,none": 0.24861878453038674,
+      "acc_stderr,none": 0.007797463983867868
+    },
+    "mmlu_econometrics": {
+      "alias": "  - econometrics",
+      "acc,none": 0.3157894736842105,
+      "acc_stderr,none": 0.043727482902780064
+    },
+    "mmlu_high_school_geography": {
+      "alias": "  - high_school_geography",
+      "acc,none": 0.2474747474747475,
+      "acc_stderr,none": 0.03074630074212451
+    },
+    "mmlu_high_school_government_and_politics": {
+      "alias": "  - high_school_government_and_politics",
+      "acc,none": 0.24870466321243523,
+      "acc_stderr,none": 0.0311958408777003
+    },
+    "mmlu_high_school_macroeconomics": {
+      "alias": "  - high_school_macroeconomics",
+      "acc,none": 0.26153846153846155,
+      "acc_stderr,none": 0.022282141204204423
+    },
+    "mmlu_high_school_microeconomics": {
+      "alias": "  - high_school_microeconomics",
+      "acc,none": 0.23529411764705882,
+      "acc_stderr,none": 0.02755361446786381
+    },
+    "mmlu_high_school_psychology": {
+      "alias": "  - high_school_psychology",
+      "acc,none": 0.22935779816513763,
+      "acc_stderr,none": 0.018025349724618684
+    },
+    "mmlu_human_sexuality": {
+      "alias": "  - human_sexuality",
+      "acc,none": 0.22137404580152673,
+      "acc_stderr,none": 0.03641297081313729
+    },
+    "mmlu_professional_psychology": {
+      "alias": "  - professional_psychology",
+      "acc,none": 0.26143790849673204,
+      "acc_stderr,none": 0.01777694715752805
+    },
+    "mmlu_public_relations": {
+      "alias": "  - public_relations",
+      "acc,none": 0.2818181818181818,
+      "acc_stderr,none": 0.043091187099464585
+    },
+    "mmlu_security_studies": {
+      "alias": "  - security_studies",
+      "acc,none": 0.2530612244897959,
+      "acc_stderr,none": 0.027833023871399677
+    },
+    "mmlu_sociology": {
+      "alias": "  - sociology",
+      "acc,none": 0.21393034825870647,
+      "acc_stderr,none": 0.02899690969332891
+    },
+    "mmlu_us_foreign_policy": {
+      "alias": "  - us_foreign_policy",
+      "acc,none": 0.24,
+      "acc_stderr,none": 0.042923469599092816
+    },
+    "mmlu_stem": {
+      "alias": " - stem",
+      "acc,none": 0.24484617824294322,
+      "acc_stderr,none": 0.007634541481754456
+    },
+    "mmlu_abstract_algebra": {
+      "alias": "  - abstract_algebra",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.0416333199893227
+    },
+    "mmlu_anatomy": {
+      "alias": "  - anatomy",
+      "acc,none": 0.2962962962962963,
+      "acc_stderr,none": 0.03944624162501117
+    },
+    "mmlu_astronomy": {
+      "alias": "  - astronomy",
+      "acc,none": 0.15789473684210525,
+      "acc_stderr,none": 0.02967416752010144
+    },
+    "mmlu_college_biology": {
+      "alias": "  - college_biology",
+      "acc,none": 0.2708333333333333,
+      "acc_stderr,none": 0.037161774375660164
+    },
+    "mmlu_college_chemistry": {
+      "alias": "  - college_chemistry",
+      "acc,none": 0.22,
+      "acc_stderr,none": 0.041633319989322695
+    },
+    "mmlu_college_computer_science": {
+      "alias": "  - college_computer_science",
+      "acc,none": 0.18,
+      "acc_stderr,none": 0.038612291966536955
+    },
+    "mmlu_college_mathematics": {
+      "alias": "  - college_mathematics",
+      "acc,none": 0.26,
+      "acc_stderr,none": 0.04408440022768079
+    },
+    "mmlu_college_physics": {
+      "alias": "  - college_physics",
+      "acc,none": 0.19607843137254902,
+      "acc_stderr,none": 0.03950581861179964
+    },
+    "mmlu_computer_security": {
+      "alias": "  - computer_security",
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_conceptual_physics": {
+      "alias": "  - conceptual_physics",
+      "acc,none": 0.28936170212765955,
+      "acc_stderr,none": 0.029644006577009618
+    },
+    "mmlu_electrical_engineering": {
+      "alias": "  - electrical_engineering",
+      "acc,none": 0.2,
+      "acc_stderr,none": 0.0333333333333333
+    },
+    "mmlu_elementary_mathematics": {
+      "alias": "  - elementary_mathematics",
+      "acc,none": 0.2777777777777778,
+      "acc_stderr,none": 0.023068188848261124
+    },
+    "mmlu_high_school_biology": {
+      "alias": "  - high_school_biology",
+      "acc,none": 0.22903225806451613,
+      "acc_stderr,none": 0.02390491431178266
+    },
+    "mmlu_high_school_chemistry": {
+      "alias": "  - high_school_chemistry",
+      "acc,none": 0.19704433497536947,
+      "acc_stderr,none": 0.027986724666736212
+    },
+    "mmlu_high_school_computer_science": {
+      "alias": "  - high_school_computer_science",
+      "acc,none": 0.33,
+      "acc_stderr,none": 0.04725815626252605
+    },
+    "mmlu_high_school_mathematics": {
+      "alias": "  - high_school_mathematics",
+      "acc,none": 0.2074074074074074,
+      "acc_stderr,none": 0.024720713193952155
+    },
+    "mmlu_high_school_physics": {
+      "alias": "  - high_school_physics",
+      "acc,none": 0.1986754966887417,
+      "acc_stderr,none": 0.03257847384436776
+    },
+    "mmlu_high_school_statistics": {
+      "alias": "  - high_school_statistics",
+      "acc,none": 0.3148148148148148,
+      "acc_stderr,none": 0.0316746870682898
+    },
+    "mmlu_machine_learning": {
+      "alias": "  - machine_learning",
+      "acc,none": 0.2767857142857143,
+      "acc_stderr,none": 0.04246624336697624
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/pawsx_2025-03-10T18-52-23.966216.json

@@ -0,0 +1,60 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "pawsx"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "pawsx": {
+      "acc,none": 0.5330714285714285,
+      "acc_stderr,none": 0.004214214249376192,
+      "alias": "pawsx"
+    },
+    "paws_de": {
+      "acc,none": 0.5185,
+      "acc_stderr,none": 0.011175478542788575,
+      "alias": " - paws_de"
+    },
+    "paws_en": {
+      "acc,none": 0.5315,
+      "acc_stderr,none": 0.011160921022883286,
+      "alias": " - paws_en"
+    },
+    "paws_es": {
+      "acc,none": 0.493,
+      "acc_stderr,none": 0.011182040020027774,
+      "alias": " - paws_es"
+    },
+    "paws_fr": {
+      "acc,none": 0.5445,
+      "acc_stderr,none": 0.011138757154883975,
+      "alias": " - paws_fr"
+    },
+    "paws_ja": {
+      "acc,none": 0.5565,
+      "acc_stderr,none": 0.011111507899646485,
+      "alias": " - paws_ja"
+    },
+    "paws_ko": {
+      "acc,none": 0.539,
+      "acc_stderr,none": 0.01114906502023434,
+      "alias": " - paws_ko"
+    },
+    "paws_zh": {
+      "acc,none": 0.5485,
+      "acc_stderr,none": 0.011130400617630761,
+      "alias": " - paws_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/pawsx_2025-03-12T12-00-24.813841.json

@@ -0,0 +1,60 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "pawsx"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "pawsx": {
+      "acc,none": 0.5385714285714286,
+      "acc_stderr,none": 0.004212723202834257,
+      "alias": "pawsx"
+    },
+    "paws_de": {
+      "acc,none": 0.535,
+      "acc_stderr,none": 0.011155703691943108,
+      "alias": " - paws_de"
+    },
+    "paws_en": {
+      "acc,none": 0.5275,
+      "acc_stderr,none": 0.01116620871686354,
+      "alias": " - paws_en"
+    },
+    "paws_es": {
+      "acc,none": 0.5125,
+      "acc_stderr,none": 0.011179640744835734,
+      "alias": " - paws_es"
+    },
+    "paws_fr": {
+      "acc,none": 0.548,
+      "acc_stderr,none": 0.011131484850525782,
+      "alias": " - paws_fr"
+    },
+    "paws_ja": {
+      "acc,none": 0.5515,
+      "acc_stderr,none": 0.011123656901911277,
+      "alias": " - paws_ja"
+    },
+    "paws_ko": {
+      "acc,none": 0.5495,
+      "acc_stderr,none": 0.011128198119942874,
+      "alias": " - paws_ko"
+    },
+    "paws_zh": {
+      "acc,none": 0.546,
+      "acc_stderr,none": 0.011135708419359798,
+      "alias": " - paws_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/piqa_2025-03-10T16-57-25.214478.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "piqa": {
+      "acc,none": 0.6800870511425462,
+      "acc_stderr,none": 0.01088287358209206,
+      "acc_norm,none": 0.6828073993471164,
+      "acc_norm_stderr,none": 0.01085815545438087,
+      "alias": "piqa"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/piqa_2025-03-11T22-02-02.648211.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "piqa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "piqa": {
+      "acc,none": 0.6871599564744287,
+      "acc_stderr,none": 0.010817714425701086,
+      "acc_norm,none": 0.6866158868335147,
+      "acc_norm_stderr,none": 0.010822829929195487,
+      "alias": "piqa"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/rwkv7-g1-0.1b-20250307-ctx4096_blimp.json

@@ -0,0 +1,360 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "blimp"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "blimp": {
+      "acc,none": 0.817820895522388,
+      "acc_stderr,none": 0.0013328875580462674,
+      "alias": "blimp"
+    },
+    "blimp_adjunct_island": {
+      "acc,none": 0.897,
+      "acc_stderr,none": 0.00961683333969581,
+      "alias": " - blimp_adjunct_island"
+    },
+    "blimp_anaphor_gender_agreement": {
+      "acc,none": 0.98,
+      "acc_stderr,none": 0.004429403980178343,
+      "alias": " - blimp_anaphor_gender_agreement"
+    },
+    "blimp_anaphor_number_agreement": {
+      "acc,none": 0.994,
+      "acc_stderr,none": 0.0024433521993298276,
+      "alias": " - blimp_anaphor_number_agreement"
+    },
+    "blimp_animate_subject_passive": {
+      "acc,none": 0.792,
+      "acc_stderr,none": 0.01284137457209692,
+      "alias": " - blimp_animate_subject_passive"
+    },
+    "blimp_animate_subject_trans": {
+      "acc,none": 0.895,
+      "acc_stderr,none": 0.00969892102602495,
+      "alias": " - blimp_animate_subject_trans"
+    },
+    "blimp_causative": {
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.01339490288966001,
+      "alias": " - blimp_causative"
+    },
+    "blimp_complex_NP_island": {
+      "acc,none": 0.604,
+      "acc_stderr,none": 0.015473313265859401,
+      "alias": " - blimp_complex_NP_island"
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "acc,none": 0.759,
+      "acc_stderr,none": 0.013531522534515455,
+      "alias": " - blimp_coordinate_structure_constraint_complex_left_branch"
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "acc,none": 0.833,
+      "acc_stderr,none": 0.011800434324644579,
+      "alias": " - blimp_coordinate_structure_constraint_object_extraction"
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "acc,none": 0.989,
+      "acc_stderr,none": 0.003299983316607816,
+      "alias": " - blimp_determiner_noun_agreement_1"
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "acc,none": 0.973,
+      "acc_stderr,none": 0.005128089049275289,
+      "alias": " - blimp_determiner_noun_agreement_2"
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "acc,none": 0.937,
+      "acc_stderr,none": 0.00768700787628643,
+      "alias": " - blimp_determiner_noun_agreement_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "acc,none": 0.954,
+      "acc_stderr,none": 0.006627814717380713,
+      "alias": " - blimp_determiner_noun_agreement_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "acc,none": 0.946,
+      "acc_stderr,none": 0.007150883521295433,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "acc,none": 0.916,
+      "acc_stderr,none": 0.008776162089491115,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "acc,none": 0.937,
+      "acc_stderr,none": 0.00768700787628643,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "acc,none": 0.964,
+      "acc_stderr,none": 0.0058939578161655475,
+      "alias": " - blimp_determiner_noun_agreement_with_adjective_1"
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "acc,none": 0.864,
+      "acc_stderr,none": 0.010845350230472992,
+      "alias": " - blimp_distractor_agreement_relational_noun"
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "acc,none": 0.716,
+      "acc_stderr,none": 0.014267009061031313,
+      "alias": " - blimp_distractor_agreement_relative_clause"
+    },
+    "blimp_drop_argument": {
+      "acc,none": 0.761,
+      "acc_stderr,none": 0.01349300044693759,
+      "alias": " - blimp_drop_argument"
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "acc,none": 0.83,
+      "acc_stderr,none": 0.01188449583454167,
+      "alias": " - blimp_ellipsis_n_bar_1"
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "acc,none": 0.89,
+      "acc_stderr,none": 0.009899393819724439,
+      "alias": " - blimp_ellipsis_n_bar_2"
+    },
+    "blimp_existential_there_object_raising": {
+      "acc,none": 0.878,
+      "acc_stderr,none": 0.010354864712936687,
+      "alias": " - blimp_existential_there_object_raising"
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "acc,none": 0.989,
+      "acc_stderr,none": 0.0032999833166078153,
+      "alias": " - blimp_existential_there_quantifiers_1"
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "acc,none": 0.3,
+      "acc_stderr,none": 0.014498627873361427,
+      "alias": " - blimp_existential_there_quantifiers_2"
+    },
+    "blimp_existential_there_subject_raising": {
+      "acc,none": 0.904,
+      "acc_stderr,none": 0.009320454434783226,
+      "alias": " - blimp_existential_there_subject_raising"
+    },
+    "blimp_expletive_it_object_raising": {
+      "acc,none": 0.799,
+      "acc_stderr,none": 0.012679107214617322,
+      "alias": " - blimp_expletive_it_object_raising"
+    },
+    "blimp_inchoative": {
+      "acc,none": 0.666,
+      "acc_stderr,none": 0.014922019523732961,
+      "alias": " - blimp_inchoative"
+    },
+    "blimp_intransitive": {
+      "acc,none": 0.817,
+      "acc_stderr,none": 0.012233587399477825,
+      "alias": " - blimp_intransitive"
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "acc,none": 0.995,
+      "acc_stderr,none": 0.0022315868748448812,
+      "alias": " - blimp_irregular_past_participle_adjectives"
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "acc,none": 0.876,
+      "acc_stderr,none": 0.010427498872343961,
+      "alias": " - blimp_irregular_past_participle_verbs"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.931,
+      "acc_stderr,none": 0.008018934050315155,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_1"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.921,
+      "acc_stderr,none": 0.00853415677333344,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_2"
+    },
+    "blimp_left_branch_island_echo_question": {
+      "acc,none": 0.425,
+      "acc_stderr,none": 0.01564032031704011,
+      "alias": " - blimp_left_branch_island_echo_question"
+    },
+    "blimp_left_branch_island_simple_question": {
+      "acc,none": 0.793,
+      "acc_stderr,none": 0.012818553557843984,
+      "alias": " - blimp_left_branch_island_simple_question"
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "acc,none": 0.697,
+      "acc_stderr,none": 0.014539683710535246,
+      "alias": " - blimp_matrix_question_npi_licensor_present"
+    },
+    "blimp_npi_present_1": {
+      "acc,none": 0.745,
+      "acc_stderr,none": 0.013790038620872833,
+      "alias": " - blimp_npi_present_1"
+    },
+    "blimp_npi_present_2": {
+      "acc,none": 0.763,
+      "acc_stderr,none": 0.013454070462577964,
+      "alias": " - blimp_npi_present_2"
+    },
+    "blimp_only_npi_licensor_present": {
+      "acc,none": 0.979,
+      "acc_stderr,none": 0.00453647215130651,
+      "alias": " - blimp_only_npi_licensor_present"
+    },
+    "blimp_only_npi_scope": {
+      "acc,none": 0.721,
+      "acc_stderr,none": 0.014190150117612035,
+      "alias": " - blimp_only_npi_scope"
+    },
+    "blimp_passive_1": {
+      "acc,none": 0.904,
+      "acc_stderr,none": 0.009320454434783219,
+      "alias": " - blimp_passive_1"
+    },
+    "blimp_passive_2": {
+      "acc,none": 0.899,
+      "acc_stderr,none": 0.009533618929340992,
+      "alias": " - blimp_passive_2"
+    },
+    "blimp_principle_A_c_command": {
+      "acc,none": 0.682,
+      "acc_stderr,none": 0.014734079309311901,
+      "alias": " - blimp_principle_A_c_command"
+    },
+    "blimp_principle_A_case_1": {
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0,
+      "alias": " - blimp_principle_A_case_1"
+    },
+    "blimp_principle_A_case_2": {
+      "acc,none": 0.97,
+      "acc_stderr,none": 0.00539714082909918,
+      "alias": " - blimp_principle_A_case_2"
+    },
+    "blimp_principle_A_domain_1": {
+      "acc,none": 0.984,
+      "acc_stderr,none": 0.003969856390319422,
+      "alias": " - blimp_principle_A_domain_1"
+    },
+    "blimp_principle_A_domain_2": {
+      "acc,none": 0.708,
+      "acc_stderr,none": 0.014385511563477338,
+      "alias": " - blimp_principle_A_domain_2"
+    },
+    "blimp_principle_A_domain_3": {
+      "acc,none": 0.588,
+      "acc_stderr,none": 0.015572363292015086,
+      "alias": " - blimp_principle_A_domain_3"
+    },
+    "blimp_principle_A_reconstruction": {
+      "acc,none": 0.274,
+      "acc_stderr,none": 0.014111099288259583,
+      "alias": " - blimp_principle_A_reconstruction"
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.006049181150584933,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_1"
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.913,
+      "acc_stderr,none": 0.008916866630745902,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_2"
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "acc,none": 0.993,
+      "acc_stderr,none": 0.0026377941462437772,
+      "alias": " - blimp_sentential_negation_npi_licensor_present"
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "acc,none": 0.828,
+      "acc_stderr,none": 0.011939788882495321,
+      "alias": " - blimp_sentential_negation_npi_scope"
+    },
+    "blimp_sentential_subject_island": {
+      "acc,none": 0.267,
+      "acc_stderr,none": 0.013996674851796264,
+      "alias": " - blimp_sentential_subject_island"
+    },
+    "blimp_superlative_quantifiers_1": {
+      "acc,none": 0.882,
+      "acc_stderr,none": 0.010206869264381795,
+      "alias": " - blimp_superlative_quantifiers_1"
+    },
+    "blimp_superlative_quantifiers_2": {
+      "acc,none": 0.801,
+      "acc_stderr,none": 0.01263164908309918,
+      "alias": " - blimp_superlative_quantifiers_2"
+    },
+    "blimp_tough_vs_raising_1": {
+      "acc,none": 0.728,
+      "acc_stderr,none": 0.014078856992462616,
+      "alias": " - blimp_tough_vs_raising_1"
+    },
+    "blimp_tough_vs_raising_2": {
+      "acc,none": 0.847,
+      "acc_stderr,none": 0.011389500459665525,
+      "alias": " - blimp_tough_vs_raising_2"
+    },
+    "blimp_transitive": {
+      "acc,none": 0.894,
+      "acc_stderr,none": 0.009739551265785115,
+      "alias": " - blimp_transitive"
+    },
+    "blimp_wh_island": {
+      "acc,none": 0.768,
+      "acc_stderr,none": 0.013354937452281569,
+      "alias": " - blimp_wh_island"
+    },
+    "blimp_wh_questions_object_gap": {
+      "acc,none": 0.802,
+      "acc_stderr,none": 0.012607733934175303,
+      "alias": " - blimp_wh_questions_object_gap"
+    },
+    "blimp_wh_questions_subject_gap": {
+      "acc,none": 0.934,
+      "acc_stderr,none": 0.007855297938697594,
+      "alias": " - blimp_wh_questions_subject_gap"
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "acc,none": 0.885,
+      "acc_stderr,none": 0.010093407594904614,
+      "alias": " - blimp_wh_questions_subject_gap_long_distance"
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "acc,none": 0.954,
+      "acc_stderr,none": 0.006627814717380708,
+      "alias": " - blimp_wh_vs_that_no_gap"
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "acc,none": 0.935,
+      "acc_stderr,none": 0.007799733061832023,
+      "alias": " - blimp_wh_vs_that_no_gap_long_distance"
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "acc,none": 0.536,
+      "acc_stderr,none": 0.015778243024904586,
+      "alias": " - blimp_wh_vs_that_with_gap"
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "acc,none": 0.46,
+      "acc_stderr,none": 0.01576859691439438,
+      "alias": " - blimp_wh_vs_that_with_gap_long_distance"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/sciq_2025-03-10T18-07-00.999754.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "sciq"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "sciq": {
+      "acc,none": 0.863,
+      "acc_stderr,none": 0.010878848714333308,
+      "acc_norm,none": 0.797,
+      "acc_norm_stderr,none": 0.012726073744598288,
+      "alias": "sciq"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/sciq_2025-03-12T11-12-57.165000.json

@@ -0,0 +1,27 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "sciq"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "sciq": {
+      "acc,none": 0.867,
+      "acc_stderr,none": 0.01074366913239733,
+      "acc_norm,none": 0.805,
+      "acc_norm_stderr,none": 0.012535235623319325,
+      "alias": "sciq"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/winogrande_2025-03-10T18-05-31.048143.json

@@ -0,0 +1,25 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "winogrande": {
+      "acc,none": 0.5374901341752171,
+      "acc_stderr,none": 0.014012928183336573,
+      "alias": "winogrande"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/winogrande_2025-03-12T11-11-17.816730.json

@@ -0,0 +1,25 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "winogrande"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "winogrande": {
+      "acc,none": 0.5335438042620363,
+      "acc_stderr,none": 0.014020826677598098,
+      "alias": "winogrande"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xcopa_2025-03-10T18-56-23.087378.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xcopa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "xcopa": {
+      "acc,none": 0.5330909090909091,
+      "acc_stderr,none": 0.006722874288986094,
+      "alias": "xcopa"
+    },
+    "xcopa_et": {
+      "acc,none": 0.51,
+      "acc_stderr,none": 0.02237859698923078,
+      "alias": " - xcopa_et"
+    },
+    "xcopa_ht": {
+      "acc,none": 0.502,
+      "acc_stderr,none": 0.022382894986483524,
+      "alias": " - xcopa_ht"
+    },
+    "xcopa_id": {
+      "acc,none": 0.54,
+      "acc_stderr,none": 0.02231133324528967,
+      "alias": " - xcopa_id"
+    },
+    "xcopa_it": {
+      "acc,none": 0.556,
+      "acc_stderr,none": 0.02224224437573102,
+      "alias": " - xcopa_it"
+    },
+    "xcopa_qu": {
+      "acc,none": 0.482,
+      "acc_stderr,none": 0.02236856511738799,
+      "alias": " - xcopa_qu"
+    },
+    "xcopa_sw": {
+      "acc,none": 0.54,
+      "acc_stderr,none": 0.022311333245289666,
+      "alias": " - xcopa_sw"
+    },
+    "xcopa_ta": {
+      "acc,none": 0.55,
+      "acc_stderr,none": 0.022270877485360444,
+      "alias": " - xcopa_ta"
+    },
+    "xcopa_th": {
+      "acc,none": 0.552,
+      "acc_stderr,none": 0.02226169729227013,
+      "alias": " - xcopa_th"
+    },
+    "xcopa_tr": {
+      "acc,none": 0.514,
+      "acc_stderr,none": 0.022374298166353185,
+      "alias": " - xcopa_tr"
+    },
+    "xcopa_vi": {
+      "acc,none": 0.528,
+      "acc_stderr,none": 0.022347949832668093,
+      "alias": " - xcopa_vi"
+    },
+    "xcopa_zh": {
+      "acc,none": 0.59,
+      "acc_stderr,none": 0.022017482578127676,
+      "alias": " - xcopa_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xcopa_2025-03-12T12-05-27.136379.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xcopa"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xcopa": {
+      "acc,none": 0.5358181818181819,
+      "acc_stderr,none": 0.006720464517893033,
+      "alias": "xcopa"
+    },
+    "xcopa_et": {
+      "acc,none": 0.508,
+      "acc_stderr,none": 0.022380208834928035,
+      "alias": " - xcopa_et"
+    },
+    "xcopa_ht": {
+      "acc,none": 0.504,
+      "acc_stderr,none": 0.02238235778196214,
+      "alias": " - xcopa_ht"
+    },
+    "xcopa_id": {
+      "acc,none": 0.546,
+      "acc_stderr,none": 0.02228814759117695,
+      "alias": " - xcopa_id"
+    },
+    "xcopa_it": {
+      "acc,none": 0.558,
+      "acc_stderr,none": 0.02223197069632112,
+      "alias": " - xcopa_it"
+    },
+    "xcopa_qu": {
+      "acc,none": 0.482,
+      "acc_stderr,none": 0.02236856511738799,
+      "alias": " - xcopa_qu"
+    },
+    "xcopa_sw": {
+      "acc,none": 0.534,
+      "acc_stderr,none": 0.02233126442325838,
+      "alias": " - xcopa_sw"
+    },
+    "xcopa_ta": {
+      "acc,none": 0.546,
+      "acc_stderr,none": 0.02228814759117695,
+      "alias": " - xcopa_ta"
+    },
+    "xcopa_th": {
+      "acc,none": 0.556,
+      "acc_stderr,none": 0.022242244375731017,
+      "alias": " - xcopa_th"
+    },
+    "xcopa_tr": {
+      "acc,none": 0.526,
+      "acc_stderr,none": 0.02235279165091416,
+      "alias": " - xcopa_tr"
+    },
+    "xcopa_vi": {
+      "acc,none": 0.544,
+      "acc_stderr,none": 0.022296238348407063,
+      "alias": " - xcopa_vi"
+    },
+    "xcopa_zh": {
+      "acc,none": 0.59,
+      "acc_stderr,none": 0.022017482578127676,
+      "alias": " - xcopa_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xnli_2025-03-10T22-52-22.630412.json

@@ -0,0 +1,105 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xnli"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "xnli": {
+      "acc,none": 0.3740557129367328,
+      "acc_stderr,none": 0.0023440725323408814,
+      "alias": "xnli"
+    },
+    "xnli_ar": {
+      "acc,none": 0.3345381526104418,
+      "acc_stderr,none": 0.009457404390939166,
+      "alias": " - xnli_ar"
+    },
+    "xnli_bg": {
+      "acc,none": 0.3718875502008032,
+      "acc_stderr,none": 0.00968750795863181,
+      "alias": " - xnli_bg"
+    },
+    "xnli_de": {
+      "acc,none": 0.42369477911646586,
+      "acc_stderr,none": 0.0099046785408289,
+      "alias": " - xnli_de"
+    },
+    "xnli_el": {
+      "acc,none": 0.3654618473895582,
+      "acc_stderr,none": 0.009652447412833482,
+      "alias": " - xnli_el"
+    },
+    "xnli_en": {
+      "acc,none": 0.4855421686746988,
+      "acc_stderr,none": 0.010017882185606005,
+      "alias": " - xnli_en"
+    },
+    "xnli_es": {
+      "acc,none": 0.3550200803212851,
+      "acc_stderr,none": 0.00959151273097429,
+      "alias": " - xnli_es"
+    },
+    "xnli_eu": {
+      "acc,none": 0.36167664670658684,
+      "acc_stderr,none": 0.006788988682090094,
+      "alias": " - xnli_eu"
+    },
+    "xnli_fr": {
+      "acc,none": 0.41445783132530123,
+      "acc_stderr,none": 0.009874311310483537,
+      "alias": " - xnli_fr"
+    },
+    "xnli_hi": {
+      "acc,none": 0.3526104417670683,
+      "acc_stderr,none": 0.009576746271768752,
+      "alias": " - xnli_hi"
+    },
+    "xnli_ru": {
+      "acc,none": 0.41726907630522087,
+      "acc_stderr,none": 0.009883930537517769,
+      "alias": " - xnli_ru"
+    },
+    "xnli_sw": {
+      "acc,none": 0.348995983935743,
+      "acc_stderr,none": 0.009554095988300685,
+      "alias": " - xnli_sw"
+    },
+    "xnli_th": {
+      "acc,none": 0.3530120481927711,
+      "acc_stderr,none": 0.009579225840709712,
+      "alias": " - xnli_th"
+    },
+    "xnli_tr": {
+      "acc,none": 0.3690763052208835,
+      "acc_stderr,none": 0.009672395644470427,
+      "alias": " - xnli_tr"
+    },
+    "xnli_ur": {
+      "acc,none": 0.3353413654618474,
+      "acc_stderr,none": 0.009463034891512704,
+      "alias": " - xnli_ur"
+    },
+    "xnli_vi": {
+      "acc,none": 0.37028112449799194,
+      "acc_stderr,none": 0.009678915409840292,
+      "alias": " - xnli_vi"
+    },
+    "xnli_zh": {
+      "acc,none": 0.3385542168674699,
+      "acc_stderr,none": 0.00948525020851688,
+      "alias": " - xnli_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xnli_2025-03-12T16-39-24.982478.json

@@ -0,0 +1,105 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xnli"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xnli": {
+      "acc,none": 0.3767233238904627,
+      "acc_stderr,none": 0.002345702445283327,
+      "alias": "xnli"
+    },
+    "xnli_ar": {
+      "acc,none": 0.3353413654618474,
+      "acc_stderr,none": 0.009463034891512703,
+      "alias": " - xnli_ar"
+    },
+    "xnli_bg": {
+      "acc,none": 0.40522088353413654,
+      "acc_stderr,none": 0.009840367477589285,
+      "alias": " - xnli_bg"
+    },
+    "xnli_de": {
+      "acc,none": 0.4285140562248996,
+      "acc_stderr,none": 0.009919113605650934,
+      "alias": " - xnli_de"
+    },
+    "xnli_el": {
+      "acc,none": 0.3562248995983936,
+      "acc_stderr,none": 0.009598796305792166,
+      "alias": " - xnli_el"
+    },
+    "xnli_en": {
+      "acc,none": 0.4903614457831325,
+      "acc_stderr,none": 0.010020210558438297,
+      "alias": " - xnli_en"
+    },
+    "xnli_es": {
+      "acc,none": 0.36224899598393573,
+      "acc_stderr,none": 0.009634223618009006,
+      "alias": " - xnli_es"
+    },
+    "xnli_eu": {
+      "acc,none": 0.36047904191616764,
+      "acc_stderr,none": 0.006784094439482414,
+      "alias": " - xnli_eu"
+    },
+    "xnli_fr": {
+      "acc,none": 0.43253012048192774,
+      "acc_stderr,none": 0.00993040902713945,
+      "alias": " - xnli_fr"
+    },
+    "xnli_hi": {
+      "acc,none": 0.35542168674698793,
+      "acc_stderr,none": 0.009593947957927137,
+      "alias": " - xnli_hi"
+    },
+    "xnli_ru": {
+      "acc,none": 0.42048192771084336,
+      "acc_stderr,none": 0.009894519551105777,
+      "alias": " - xnli_ru"
+    },
+    "xnli_sw": {
+      "acc,none": 0.3417670682730924,
+      "acc_stderr,none": 0.009506977398287621,
+      "alias": " - xnli_sw"
+    },
+    "xnli_th": {
+      "acc,none": 0.3397590361445783,
+      "acc_stderr,none": 0.009493454925438252,
+      "alias": " - xnli_th"
+    },
+    "xnli_tr": {
+      "acc,none": 0.3755020080321285,
+      "acc_stderr,none": 0.00970642284437982,
+      "alias": " - xnli_tr"
+    },
+    "xnli_ur": {
+      "acc,none": 0.3337349397590361,
+      "acc_stderr,none": 0.009451743112667055,
+      "alias": " - xnli_ur"
+    },
+    "xnli_vi": {
+      "acc,none": 0.36947791164658633,
+      "acc_stderr,none": 0.00967457608577645,
+      "alias": " - xnli_vi"
+    },
+    "xnli_zh": {
+      "acc,none": 0.3369477911646586,
+      "acc_stderr,none": 0.009474203778757713,
+      "alias": " - xnli_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xstorycloze_2025-03-10T23-08-53.855464.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xstorycloze"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "xstorycloze": {
+      "acc,none": 0.5288490463871006,
+      "acc_stderr,none": 0.003862144546165837,
+      "alias": "xstorycloze"
+    },
+    "xstorycloze_ar": {
+      "acc,none": 0.4811383189940437,
+      "acc_stderr,none": 0.012857966762464992,
+      "alias": " - xstorycloze_ar"
+    },
+    "xstorycloze_en": {
+      "acc,none": 0.6307081403044341,
+      "acc_stderr,none": 0.012419685881273582,
+      "alias": " - xstorycloze_en"
+    },
+    "xstorycloze_es": {
+      "acc,none": 0.5373924553275976,
+      "acc_stderr,none": 0.012831093347016553,
+      "alias": " - xstorycloze_es"
+    },
+    "xstorycloze_eu": {
+      "acc,none": 0.5268034414295168,
+      "acc_stderr,none": 0.012848623899505767,
+      "alias": " - xstorycloze_eu"
+    },
+    "xstorycloze_hi": {
+      "acc,none": 0.5109199205823958,
+      "acc_stderr,none": 0.012864056278255038,
+      "alias": " - xstorycloze_hi"
+    },
+    "xstorycloze_id": {
+      "acc,none": 0.5168762409000662,
+      "acc_stderr,none": 0.012859793919977604,
+      "alias": " - xstorycloze_id"
+    },
+    "xstorycloze_my": {
+      "acc,none": 0.48974189278623426,
+      "acc_stderr,none": 0.012864417047980475,
+      "alias": " - xstorycloze_my"
+    },
+    "xstorycloze_ru": {
+      "acc,none": 0.5195234943745863,
+      "acc_stderr,none": 0.01285731253183686,
+      "alias": " - xstorycloze_ru"
+    },
+    "xstorycloze_sw": {
+      "acc,none": 0.514890800794176,
+      "acc_stderr,none": 0.012861417842074004,
+      "alias": " - xstorycloze_sw"
+    },
+    "xstorycloze_te": {
+      "acc,none": 0.5466578424884183,
+      "acc_stderr,none": 0.012810980537828157,
+      "alias": " - xstorycloze_te"
+    },
+    "xstorycloze_zh": {
+      "acc,none": 0.542686962276638,
+      "acc_stderr,none": 0.012820147204256244,
+      "alias": " - xstorycloze_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xstorycloze_2025-03-12T17-01-39.345687.json

@@ -0,0 +1,80 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xstorycloze"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xstorycloze": {
+      "acc,none": 0.5288490463871006,
+      "acc_stderr,none": 0.003861656646850473,
+      "alias": "xstorycloze"
+    },
+    "xstorycloze_ar": {
+      "acc,none": 0.4798146922567836,
+      "acc_stderr,none": 0.012856635706498292,
+      "alias": " - xstorycloze_ar"
+    },
+    "xstorycloze_en": {
+      "acc,none": 0.6333553937789543,
+      "acc_stderr,none": 0.012401034429990701,
+      "alias": " - xstorycloze_en"
+    },
+    "xstorycloze_es": {
+      "acc,none": 0.5314361350099271,
+      "acc_stderr,none": 0.012841668760976905,
+      "alias": " - xstorycloze_es"
+    },
+    "xstorycloze_eu": {
+      "acc,none": 0.5268034414295168,
+      "acc_stderr,none": 0.012848623899505767,
+      "alias": " - xstorycloze_eu"
+    },
+    "xstorycloze_hi": {
+      "acc,none": 0.513567174056916,
+      "acc_stderr,none": 0.01286238758665008,
+      "alias": " - xstorycloze_hi"
+    },
+    "xstorycloze_id": {
+      "acc,none": 0.5162144275314361,
+      "acc_stderr,none": 0.01286035780505586,
+      "alias": " - xstorycloze_id"
+    },
+    "xstorycloze_my": {
+      "acc,none": 0.48974189278623426,
+      "acc_stderr,none": 0.012864417047980475,
+      "alias": " - xstorycloze_my"
+    },
+    "xstorycloze_ru": {
+      "acc,none": 0.5215089344804765,
+      "acc_stderr,none": 0.012855214257296608,
+      "alias": " - xstorycloze_ru"
+    },
+    "xstorycloze_sw": {
+      "acc,none": 0.5122435473196558,
+      "acc_stderr,none": 0.012863267059205548,
+      "alias": " - xstorycloze_sw"
+    },
+    "xstorycloze_te": {
+      "acc,none": 0.5453342157511581,
+      "acc_stderr,none": 0.012814127367359414,
+      "alias": " - xstorycloze_te"
+    },
+    "xstorycloze_zh": {
+      "acc,none": 0.5473196558570483,
+      "acc_stderr,none": 0.012809372866181955,
+      "alias": " - xstorycloze_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xwinograd_2025-03-10T23-11-45.464852.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.6432906271072151,
+      "acc_stderr,none": 0.007109338831236582,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "acc,none": 0.6993548387096774,
+      "acc_stderr,none": 0.009511693326241081,
+      "alias": " - xwinograd_en"
+    },
+    "xwinograd_fr": {
+      "acc,none": 0.5903614457831325,
+      "acc_stderr,none": 0.05430658329539148,
+      "alias": " - xwinograd_fr"
+    },
+    "xwinograd_jp": {
+      "acc,none": 0.5526590198123045,
+      "acc_stderr,none": 0.016064426253309637,
+      "alias": " - xwinograd_jp"
+    },
+    "xwinograd_pt": {
+      "acc,none": 0.5247148288973384,
+      "acc_stderr,none": 0.030852343325490784,
+      "alias": " - xwinograd_pt"
+    },
+    "xwinograd_ru": {
+      "acc,none": 0.5619047619047619,
+      "acc_stderr,none": 0.02799953368887838,
+      "alias": " - xwinograd_ru"
+    },
+    "xwinograd_zh": {
+      "acc,none": 0.6785714285714286,
+      "acc_stderr,none": 0.02082361047648225,
+      "alias": " - xwinograd_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xwinograd_2025-03-11T21-17-31.358555.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_ids": [
+    0
+  ],
+  "stop_token_ids": [
+    0,
+    261
+  ],
+  "custom_prefix_token_id": 0,
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.6432906271072151,
+      "acc_stderr,none": 0.007109338831236581,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "acc,none": 0.6993548387096774,
+      "acc_stderr,none": 0.009511693326241081,
+      "alias": " - xwinograd_en"
+    },
+    "xwinograd_fr": {
+      "acc,none": 0.5903614457831325,
+      "acc_stderr,none": 0.05430658329539148,
+      "alias": " - xwinograd_fr"
+    },
+    "xwinograd_jp": {
+      "acc,none": 0.5526590198123045,
+      "acc_stderr,none": 0.016064426253309637,
+      "alias": " - xwinograd_jp"
+    },
+    "xwinograd_pt": {
+      "acc,none": 0.5247148288973384,
+      "acc_stderr,none": 0.030852343325490784,
+      "alias": " - xwinograd_pt"
+    },
+    "xwinograd_ru": {
+      "acc,none": 0.5619047619047619,
+      "acc_stderr,none": 0.02799953368887838,
+      "alias": " - xwinograd_ru"
+    },
+    "xwinograd_zh": {
+      "acc,none": 0.6785714285714286,
+      "acc_stderr,none": 0.02082361047648225,
+      "alias": " - xwinograd_zh"
+    }
+  }
+}

lm_eval/rwkv7-g1-0.1b-20250307-ctx4096/xwinograd_2025-03-12T17-04-51.338405.json

@@ -0,0 +1,55 @@
+{
+  "model": "/models/rwkv7-g1-0.1b-20250307-ctx4096",
+  "tasks": [
+    "xwinograd"
+  ],
+  "num_fewshot": 0,
+  "lm_eval_version": "0.4.3",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "custom_prefix_token_id": 11,
+  "pad_token_ids": [
+    11
+  ],
+  "stop_token_ids": [
+    11,
+    261
+  ],
+  "results": {
+    "xwinograd": {
+      "acc,none": 0.6419420094403236,
+      "acc_stderr,none": 0.0071176833742256414,
+      "alias": "xwinograd"
+    },
+    "xwinograd_en": {
+      "acc,none": 0.6980645161290323,
+      "acc_stderr,none": 0.009523285337477353,
+      "alias": " - xwinograd_en"
+    },
+    "xwinograd_fr": {
+      "acc,none": 0.5783132530120482,
+      "acc_stderr,none": 0.0545342848529511,
+      "alias": " - xwinograd_fr"
+    },
+    "xwinograd_jp": {
+      "acc,none": 0.5495307612095933,
+      "acc_stderr,none": 0.01607480892375643,
+      "alias": " - xwinograd_jp"
+    },
+    "xwinograd_pt": {
+      "acc,none": 0.532319391634981,
+      "acc_stderr,none": 0.030825503526303786,
+      "alias": " - xwinograd_pt"
+    },
+    "xwinograd_ru": {
+      "acc,none": 0.5682539682539682,
+      "acc_stderr,none": 0.02795249586167163,
+      "alias": " - xwinograd_ru"
+    },
+    "xwinograd_zh": {
+      "acc,none": 0.6726190476190477,
+      "acc_stderr,none": 0.02092316077596883,
+      "alias": " - xwinograd_zh"
+    }
+  }
+}