Update to 14-category MMLU-Pro model - fixes #584 #591

.downloaded

@@ -0,0 +1 @@
+2025-10-14T19:45:18Z

category_mapping.json

@@ -0,0 +1 @@
+{"category_to_idx": {"biology": 0, "business": 1, "chemistry": 2, "computer science": 3, "economics": 4, "engineering": 5, "health": 6, "history": 7, "law": 8, "math": 9, "other": 10, "philosophy": 11, "physics": 12, "psychology": 13}, "idx_to_category": {"0": "biology", "1": "business", "2": "chemistry", "3": "computer science", "4": "economics", "5": "engineering", "6": "health", "7": "history", "8": "law", "9": "math", "10": "other", "11": "philosophy", "12": "physics", "13": "psychology"}}

config.json

@@ -12,7 +12,6 @@
   "cls_token_id": 50281,
   "decoder_bias": true,
   "deterministic_flash_attn": false,
-  "dtype": "float32",
   "embedding_dropout": 0.0,
   "eos_token_id": 50282,
   "global_attn_every_n_layers": 3,
@@ -21,17 +20,39 @@
   "hidden_activation": "gelu",
   "hidden_size": 768,
   "id2label": {
-    "0": "business",
-    "1": "law",
-    "2": "psychology"
+    "0": "biology",
+    "1": "business",
+    "2": "chemistry",
+    "3": "computer science",
+    "4": "economics",
+    "5": "engineering",
+    "6": "health",
+    "7": "history",
+    "8": "law",
+    "9": "math",
+    "10": "other",
+    "11": "philosophy",
+    "12": "physics",
+    "13": "psychology"
   },
   "initializer_cutoff_factor": 2.0,
   "initializer_range": 0.02,
   "intermediate_size": 1152,
   "label2id": {
-    "business": 0,
-    "law": 1,
-    "psychology": 2
+    "biology": 0,
+    "business": 1,
+    "chemistry": 2,
+    "computer science": 3,
+    "economics": 4,
+    "engineering": 5,
+    "health": 6,
+    "history": 7,
+    "law": 8,
+    "math": 9,
+    "other": 10,
+    "philosophy": 11,
+    "physics": 12,
+    "psychology": 13
   },
   "layer_norm_eps": 1e-05,
   "local_attention": 128,
@@ -46,10 +67,12 @@
   "num_hidden_layers": 22,
   "pad_token_id": 50283,
   "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
   "repad_logits_with_grad": false,
   "sep_token_id": 50282,
   "sparse_pred_ignore_index": -100,
   "sparse_prediction": false,
-  "transformers_version": "4.56.1",
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
   "vocab_size": 50368
-}
+}

evaluation_results.json

@@ -0,0 +1,654 @@
+{
+  "validation_accuracy": 0.8526315789473684,
+  "test_accuracy": 0.8343490304709141,
+  "validation_report": {
+    "biology": {
+      "precision": 0.8425925925925926,
+      "recall": 0.8425925925925926,
+      "f1-score": 0.8425925925925926,
+      "support": 108.0
+    },
+    "business": {
+      "precision": 0.8695652173913043,
+      "recall": 0.8403361344537815,
+      "f1-score": 0.8547008547008547,
+      "support": 119.0
+    },
+    "chemistry": {
+      "precision": 0.861271676300578,
+      "recall": 0.8764705882352941,
+      "f1-score": 0.8688046647230321,
+      "support": 170.0
+    },
+    "computer science": {
+      "precision": 0.8507462686567164,
+      "recall": 0.9193548387096774,
+      "f1-score": 0.8837209302325582,
+      "support": 62.0
+    },
+    "economics": {
+      "precision": 0.7986111111111112,
+      "recall": 0.905511811023622,
+      "f1-score": 0.8487084870848709,
+      "support": 127.0
+    },
+    "engineering": {
+      "precision": 0.8776978417266187,
+      "recall": 0.8413793103448276,
+      "f1-score": 0.8591549295774648,
+      "support": 145.0
+    },
+    "health": {
+      "precision": 0.8095238095238095,
+      "recall": 0.8292682926829268,
+      "f1-score": 0.8192771084337349,
+      "support": 123.0
+    },
+    "history": {
+      "precision": 0.896551724137931,
+      "recall": 0.9122807017543859,
+      "f1-score": 0.9043478260869565,
+      "support": 57.0
+    },
+    "law": {
+      "precision": 0.9585798816568047,
+      "recall": 0.9818181818181818,
+      "f1-score": 0.9700598802395209,
+      "support": 165.0
+    },
+    "math": {
+      "precision": 0.8888888888888888,
+      "recall": 0.9108910891089109,
+      "f1-score": 0.8997555012224939,
+      "support": 202.0
+    },
+    "other": {
+      "precision": 0.7916666666666666,
+      "recall": 0.6884057971014492,
+      "f1-score": 0.7364341085271318,
+      "support": 138.0
+    },
+    "philosophy": {
+      "precision": 0.8375,
+      "recall": 0.8933333333333333,
+      "f1-score": 0.864516129032258,
+      "support": 75.0
+    },
+    "physics": {
+      "precision": 0.8157894736842105,
+      "recall": 0.7948717948717948,
+      "f1-score": 0.8051948051948052,
+      "support": 195.0
+    },
+    "psychology": {
+      "precision": 0.8073394495412844,
+      "recall": 0.7394957983193278,
+      "f1-score": 0.7719298245614035,
+      "support": 119.0
+    },
+    "accuracy": 0.8526315789473684,
+    "macro avg": {
+      "precision": 0.8504517572770369,
+      "recall": 0.8554293045964361,
+      "f1-score": 0.8520855458721199,
+      "support": 1805.0
+    },
+    "weighted avg": {
+      "precision": 0.8520451342930214,
+      "recall": 0.8526315789473684,
+      "f1-score": 0.8515396065019253,
+      "support": 1805.0
+    }
+  },
+  "test_report": {
+    "biology": {
+      "precision": 0.8461538461538461,
+      "recall": 0.822429906542056,
+      "f1-score": 0.8341232227488151,
+      "support": 107.0
+    },
+    "business": {
+      "precision": 0.8648648648648649,
+      "recall": 0.8135593220338984,
+      "f1-score": 0.8384279475982532,
+      "support": 118.0
+    },
+    "chemistry": {
+      "precision": 0.8170731707317073,
+      "recall": 0.788235294117647,
+      "f1-score": 0.8023952095808383,
+      "support": 170.0
+    },
+    "computer science": {
+      "precision": 0.8301886792452831,
+      "recall": 0.7213114754098361,
+      "f1-score": 0.7719298245614035,
+      "support": 61.0
+    },
+    "economics": {
+      "precision": 0.8688524590163934,
+      "recall": 0.8412698412698413,
+      "f1-score": 0.8548387096774194,
+      "support": 126.0
+    },
+    "engineering": {
+      "precision": 0.8394160583941606,
+      "recall": 0.7876712328767124,
+      "f1-score": 0.8127208480565371,
+      "support": 146.0
+    },
+    "health": {
+      "precision": 0.8790322580645161,
+      "recall": 0.8861788617886179,
+      "f1-score": 0.8825910931174089,
+      "support": 123.0
+    },
+    "history": {
+      "precision": 0.9433962264150944,
+      "recall": 0.8771929824561403,
+      "f1-score": 0.9090909090909091,
+      "support": 57.0
+    },
+    "law": {
+      "precision": 0.9457831325301205,
+      "recall": 0.9515151515151515,
+      "f1-score": 0.9486404833836858,
+      "support": 165.0
+    },
+    "math": {
+      "precision": 0.8440366972477065,
+      "recall": 0.9064039408866995,
+      "f1-score": 0.8741092636579573,
+      "support": 203.0
+    },
+    "other": {
+      "precision": 0.6923076923076923,
+      "recall": 0.7122302158273381,
+      "f1-score": 0.7021276595744681,
+      "support": 139.0
+    },
+    "philosophy": {
+      "precision": 0.8333333333333334,
+      "recall": 0.9333333333333333,
+      "f1-score": 0.8805031446540881,
+      "support": 75.0
+    },
+    "physics": {
+      "precision": 0.7853658536585366,
+      "recall": 0.8256410256410256,
+      "f1-score": 0.805,
+      "support": 195.0
+    },
+    "psychology": {
+      "precision": 0.768595041322314,
+      "recall": 0.775,
+      "f1-score": 0.7717842323651453,
+      "support": 120.0
+    },
+    "accuracy": 0.8343490304709141,
+    "macro avg": {
+      "precision": 0.8398856652346834,
+      "recall": 0.8315694702641643,
+      "f1-score": 0.8348773248619237,
+      "support": 1805.0
+    },
+    "weighted avg": {
+      "precision": 0.8352151540840406,
+      "recall": 0.8343490304709141,
+      "f1-score": 0.8341367555565297,
+      "support": 1805.0
+    }
+  },
+  "validation_confusion_matrix": [
+    [
+      91,
+      0,
+      2,
+      0,
+      1,
+      0,
+      9,
+      0,
+      0,
+      0,
+      2,
+      0,
+      0,
+      3
+    ],
+    [
+      0,
+      100,
+      1,
+      0,
+      10,
+      0,
+      0,
+      0,
+      0,
+      3,
+      2,
+      2,
+      0,
+      1
+    ],
+    [
+      1,
+      0,
+      149,
+      0,
+      0,
+      5,
+      1,
+      0,
+      0,
+      1,
+      0,
+      0,
+      13,
+      0
+    ],
+    [
+      0,
+      0,
+      0,
+      57,
+      0,
+      0,
+      0,
+      0,
+      0,
+      3,
+      2,
+      0,
+      0,
+      0
+    ],
+    [
+      1,
+      5,
+      0,
+      0,
+      115,
+      0,
+      0,
+      0,
+      1,
+      2,
+      1,
+      2,
+      0,
+      0
+    ],
+    [
+      0,
+      0,
+      3,
+      5,
+      0,
+      122,
+      0,
+      0,
+      0,
+      2,
+      0,
+      0,
+      13,
+      0
+    ],
+    [
+      5,
+      0,
+      1,
+      0,
+      1,
+      0,
+      102,
+      0,
+      1,
+      0,
+      5,
+      3,
+      1,
+      4
+    ],
+    [
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      1,
+      52,
+      0,
+      0,
+      2,
+      1,
+      0,
+      0
+    ],
+    [
+      0,
+      0,
+      0,
+      0,
+      1,
+      0,
+      1,
+      0,
+      162,
+      0,
+      1,
+      0,
+      0,
+      0
+    ],
+    [
+      0,
+      3,
+      0,
+      2,
+      2,
+      1,
+      1,
+      0,
+      1,
+      184,
+      1,
+      0,
+      3,
+      4
+    ],
+    [
+      2,
+      5,
+      2,
+      1,
+      10,
+      0,
+      3,
+      3,
+      2,
+      2,
+      95,
+      2,
+      4,
+      7
+    ],
+    [
+      1,
+      0,
+      0,
+      0,
+      1,
+      0,
+      1,
+      2,
+      1,
+      0,
+      1,
+      67,
+      0,
+      1
+    ],
+    [
+      1,
+      0,
+      15,
+      0,
+      0,
+      11,
+      0,
+      0,
+      0,
+      9,
+      3,
+      0,
+      155,
+      1
+    ],
+    [
+      5,
+      2,
+      0,
+      2,
+      3,
+      0,
+      7,
+      1,
+      1,
+      1,
+      5,
+      3,
+      1,
+      88
+    ]
+  ],
+  "test_confusion_matrix": [
+    [
+      88,
+      0,
+      0,
+      0,
+      1,
+      0,
+      6,
+      1,
+      2,
+      1,
+      1,
+      0,
+      1,
+      6
+    ],
+    [
+      0,
+      96,
+      0,
+      0,
+      5,
+      0,
+      0,
+      0,
+      1,
+      6,
+      3,
+      2,
+      1,
+      4
+    ],
+    [
+      2,
+      0,
+      134,
+      1,
+      0,
+      8,
+      0,
+      0,
+      0,
+      2,
+      2,
+      0,
+      21,
+      0
+    ],
+    [
+      0,
+      0,
+      1,
+      44,
+      0,
+      3,
+      1,
+      0,
+      0,
+      9,
+      3,
+      0,
+      0,
+      0
+    ],
+    [
+      1,
+      4,
+      0,
+      0,
+      106,
+      0,
+      0,
+      0,
+      1,
+      3,
+      6,
+      1,
+      0,
+      4
+    ],
+    [
+      1,
+      0,
+      13,
+      1,
+      0,
+      115,
+      0,
+      0,
+      0,
+      6,
+      1,
+      0,
+      9,
+      0
+    ],
+    [
+      6,
+      0,
+      1,
+      0,
+      1,
+      1,
+      109,
+      0,
+      0,
+      1,
+      2,
+      0,
+      0,
+      2
+    ],
+    [
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      50,
+      0,
+      0,
+      5,
+      0,
+      0,
+      1
+    ],
+    [
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      157,
+      0,
+      4,
+      4,
+      0,
+      0
+    ],
+    [
+      1,
+      2,
+      0,
+      5,
+      1,
+      0,
+      0,
+      0,
+      0,
+      184,
+      0,
+      0,
+      9,
+      1
+    ],
+    [
+      0,
+      9,
+      1,
+      2,
+      7,
+      0,
+      2,
+      1,
+      4,
+      1,
+      99,
+      4,
+      2,
+      7
+    ],
+    [
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      3,
+      70,
+      0,
+      2
+    ],
+    [
+      1,
+      0,
+      14,
+      0,
+      0,
+      10,
+      0,
+      0,
+      0,
+      4,
+      4,
+      0,
+      161,
+      1
+    ],
+    [
+      3,
+      0,
+      0,
+      0,
+      1,
+      0,
+      6,
+      1,
+      1,
+      1,
+      10,
+      3,
+      1,
+      93
+    ]
+  ]
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc0aaf61989a7eff1be0140cded181719ae059a048528153b809fd16b7dfb335
-size 598442860
+oid sha256:5ac997d635f7abf77e40553d15af381cf775aad0f20362c3e8f2e0e9d0353abe
+size 598476704

tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 512
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 50283,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
   "added_tokens": [
     {
       "id": 0,

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c469ad6d75724ea5e403a6a9e14cd2b3afbb0aebef6b5e8e129064bdaf81e4ab
+size 5777