bench: add mmlu57_token.log full 57

mmlu57_token.log

@@ -0,0 +1,1024 @@
+[MMLU57-TOKEN] start Sun Aug  9 23:10:07 UTC 2026 token=hf_oUMhDnr...
+[*] MicroLLM2 MMLU — model: /home/zeus/microllm2/microllm2-checkpoints/final_merged  shots=5  limit=20
+[*] GPT2-XL 1.5B 1024ctx  vocab=50259 (ChatML) — MMLU via direct eval (no harness needed)
+[*] lm-eval not installed — using lightweight direct MMLU eval (same logic, no harness)
+[*] Install harness for official numbers: pip install lm-eval==0.4.4
+[*] Loading tokenizer + model /home/zeus/microllm2/microllm2-checkpoints/final_merged ...
+[+] Loaded on cuda:0 dtype=torch.bfloat16 — starting MMLU
+
+============================================================
+[>] abstract_algebra  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] anatomy  (shots=5)
+
+
+
+
+
+
+
+
+
                                                        
+
+============================================================
+[>] astronomy  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
                                                          
+
+============================================================
+[>] business_ethics  (shots=5)
+
+
+
+
+
+
+
+
+
+
                                                                
+
+============================================================
+[>] clinical_knowledge  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
                                                                   
+
+============================================================
+[>] college_biology  (shots=5)
+
+
+
+
+
+
+
+
+
+
                                                                
+
+============================================================
+[>] college_chemistry  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                  
+
+============================================================
+[>] college_computer_science  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                         
+
+============================================================
+[>] college_mathematics  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                    
+
+============================================================
+[>] college_medicine  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] college_physics  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                
+
+============================================================
+[>] computer_security  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                  
+
+============================================================
+[>] conceptual_physics  (shots=5)
+
+
+
+
+
+
+
+
+
+
                                                                   
+
+============================================================
+[>] econometrics  (shots=5)
+
+
+
+
+
+
+
+
                                                             
+
+============================================================
+[>] electrical_engineering  (shots=5)
+
+
+
+
+
+
+
+
                                                                       
+
+============================================================
+[>] elementary_mathematics  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                       
+
+============================================================
+[>] formal_logic  (shots=5)
+
+
+
+
+
+
+
+
+
+
                                                             
+
+============================================================
+[>] global_facts  (shots=5)
+
+
+
+
+
+
+
                                                             
+
+============================================================
+[>] high_school_biology  (shots=5)
+
+
+
+
+
+
+
+
                                                                    
+
+============================================================
+[>] high_school_chemistry  (shots=5)
+
+
+
+
+
+
+
                                                                      
+
+============================================================
+[>] high_school_computer_science  (shots=5)
+
+
+
+
+
+
+
+
+
                                                                             
+
+============================================================
+[>] high_school_european_history  (shots=5)
+
+
+
+
+
+
+
                                                                             
+
+============================================================
+[>] high_school_geography  (shots=5)
+
+
+
+
+
+
+
                                                                      
+
+============================================================
+[>] high_school_government_and_politics  (shots=5)
+
+
+
+
+
+
+
                                                                                    
+
+============================================================
+[>] high_school_macroeconomics  (shots=5)
+
+
+
+
+
+
+
                                                                           
+
+============================================================
+[>] high_school_mathematics  (shots=5)
+
+
+
+
+
+
+
                                                                        
+
+============================================================
+[>] high_school_microeconomics  (shots=5)
+
+
+
+
+
+
+
+
                                                                           
+
+============================================================
+[>] high_school_physics  (shots=5)
+
+
+
+
+
+
+
+
                                                                    
+
+============================================================
+[>] high_school_psychology  (shots=5)
+
+
+
+
+
+
+
+
                                                                       
+
+============================================================
+[>] high_school_statistics  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
                                                                       
+
+============================================================
+[>] high_school_us_history  (shots=5)
+
+
+
+
+
+
+
                                                                       
+
+============================================================
+[>] high_school_world_history  (shots=5)
+
+
+
+
+
+
+
                                                                          
+
+============================================================
+[>] human_aging  (shots=5)
+
+
+
+
+
+
+
+
+
                                                            
+
+============================================================
+[>] human_sexuality  (shots=5)
+
+
+
+
+
+
+
+
                                                                
+
+============================================================
+[>] international_law  (shots=5)
+
+
+
+
+
+
+
+
                                                                  
+
+============================================================
+[>] jurisprudence  (shots=5)
+
+
+
+
+
+
+
                                                              
+
+============================================================
+[>] logical_fallacies  (shots=5)
+
+
+
+
+
+
+
                                                                  
+
+============================================================
+[>] machine_learning  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] management  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                           
+
+============================================================
+[>] marketing  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                          
+
+============================================================
+[>] medical_genetics  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] miscellaneous  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                              
+
+============================================================
+[>] moral_disputes  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                               
+
+============================================================
+[>] moral_scenarios  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                
+
+============================================================
+[>] nutrition  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                          
+
+============================================================
+[>] philosophy  (shots=5)
+
+
+
+
+
+
                                                           
+
+============================================================
+[>] prehistory  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                           
+
+============================================================
+[>] professional_accounting  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                        
+
+============================================================
+[>] professional_law  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] professional_medicine  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                      
+
+============================================================
+[>] professional_psychology  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                        
+
+============================================================
+[>] public_relations  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] security_studies  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                 
+
+============================================================
+[>] sociology  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                          
+
+============================================================
+[>] us_foreign_policy  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                  
+
+============================================================
+[>] virology  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                         
+
+============================================================
+[>] world_religions  (shots=5)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                
+[=] world_religions: 3/20 = 15.0%  (running avg 27.9%)
+
+============================================================
+MMLU RESULT — /home/zeus/microllm2/microllm2-checkpoints/final_merged
+Shots: 5  Subjects: 57/57
+  abstract_algebra                     30.0%  (6/20)
+  anatomy                              25.0%  (5/20)
+  astronomy                            35.0%  (7/20)
+  business_ethics                      30.0%  (6/20)
+  clinical_knowledge                   45.0%  (9/20)
+  college_biology                      45.0%  (9/20)
+  college_chemistry                    15.0%  (3/20)
+  college_computer_science             45.0%  (9/20)
+  college_mathematics                  35.0%  (7/20)
+  college_medicine                     30.0%  (6/20)
+  college_physics                      15.0%  (3/20)
+  computer_security                    30.0%  (6/20)
+  conceptual_physics                    5.0%  (1/20)
+  econometrics                         30.0%  (6/20)
+  electrical_engineering               20.0%  (4/20)
+  elementary_mathematics               30.0%  (6/20)
+  formal_logic                         10.0%  (2/20)
+  global_facts                         35.0%  (7/20)
+  high_school_biology                  45.0%  (9/20)
+  high_school_chemistry                35.0%  (7/20)
+  high_school_computer_science         35.0%  (7/20)
+  high_school_european_history         20.0%  (4/20)
+  high_school_geography                25.0%  (5/20)
+  high_school_government_and_politics  20.0%  (4/20)
+  high_school_macroeconomics            0.0%  (0/20)
+  high_school_mathematics              20.0%  (4/20)
+  high_school_microeconomics           35.0%  (7/20)
+  high_school_physics                  20.0%  (4/20)
+  high_school_psychology               25.0%  (5/20)
+  high_school_statistics               40.0%  (8/20)
+  high_school_us_history               20.0%  (4/20)
+  high_school_world_history            35.0%  (7/20)
+  human_aging                          40.0%  (8/20)
+  human_sexuality                      15.0%  (3/20)
+  international_law                    35.0%  (7/20)
+  jurisprudence                        40.0%  (8/20)
+  logical_fallacies                    35.0%  (7/20)
+  machine_learning                     50.0%  (10/20)
+  management                           20.0%  (4/20)
+  marketing                            35.0%  (7/20)
+  medical_genetics                     40.0%  (8/20)
+  miscellaneous                        30.0%  (6/20)
+  moral_disputes                       20.0%  (4/20)
+  moral_scenarios                      15.0%  (3/20)
+  nutrition                            20.0%  (4/20)
+  philosophy                           15.0%  (3/20)
+  prehistory                           25.0%  (5/20)
+  professional_accounting              30.0%  (6/20)
+  professional_law                     35.0%  (7/20)
+  professional_medicine                 5.0%  (1/20)
+  professional_psychology              45.0%  (9/20)
+  public_relations                     45.0%  (9/20)
+  security_studies                     25.0%  (5/20)
+  sociology                            20.0%  (4/20)
+  us_foreign_policy                    25.0%  (5/20)
+  virology                             25.0%  (5/20)
+  world_religions                      15.0%  (3/20)
+
+  OVERALL: 318/1140 = 27.89%
+============================================================
+[+] Saved mmlu_results.json
+
+Note: GPT2-XL base ~24-26% MMLU (random 25%). MicroLLM2 distilled should be 25-30% —
+MMLU is knowledge-heavy; GPT2 1.5B 1024ctx cannot match 7B+ models. Use as sanity check, not SOTA claim.
+[MMLU57-TOKEN] exit 0 at Sun Aug  9 23:12:32 UTC 2026
+{
+  "model": "/home/zeus/microllm2/microllm2-checkpoints/final_merged",
+  "shots": 5,
+  "limit": 20,
+  "overall": {
+    "correct": 318,
+    "total": 1140,
+    "accuracy": 0.2789473684210526
+  },
+  "subjects": {
+    "abstract_algebra": {
+      "correct": 6,
+      "total": 20,
+      "accuracy": 0.3
+    },
+    "anatomy": {
+      "correct": 5,
+      "total": 20,
+      "accuracy": 0.25
+    },
+    "astronomy": {
+      "correct": 7,
+      "total": 20,
+      "accuracy": 0.35
+    },
+    "business_ethics": {
+      "correct": 6,
+      "total": 20,
+      "accuracy": 0.3
+    },
+FINAL57 318 / 1140 27.89 57
+'mmlu_results.json' -> 'mmlu57_results.json'
+'mmlu_results.json' -> '/teamspace/studios/this_studio/microllm2/mmlu57_results.json'
+'mmlu57_token.log' -> '/teamspace/studios/this_studio/microllm2/mmlu57_token.log'