SLM-CI update

results.json

@@ -1,6 +1,30 @@
 {
-  "benchmark": "SLM-CI v0.1",
-  "updated_at": "2025-09-03",
+  "benchmark": "GODELEV/SLM-CI-Benchmark::benchmark.jsonl",
+  "num_samples": 2977,
+  "categories": [
+    "Algorithmic",
+    "Counterfactual",
+    "Spatial",
+    "State Tracking",
+    "Syllogism"
+  ],
+  "difficulties": [
+    "easy",
+    "hard",
+    "medium"
+  ],
+  "difficulty_weights": {
+    "easy": 1.0,
+    "medium": 1.5,
+    "hard": 2.25
+  },
+  "category_weights": {
+    "Syllogism": 1.35,
+    "State Tracking": 1.2,
+    "Algorithmic": 1.3,
+    "Counterfactual": 1.1,
+    "Spatial": 1.15
+  },
   "leaderboard": [
     {
       "model": "Example/Giant-360M",
@@ -16,7 +40,11 @@
         "State Tracking": 0.537,
         "Syllogism": 0.452
       },
-      "by_difficulty": { "easy": 0.721, "medium": 0.565, "hard": 0.421 }
+      "by_difficulty": {
+        "easy": 0.721,
+        "medium": 0.565,
+        "hard": 0.421
+      }
     },
     {
       "model": "CyanMonkey/Rose-1.5-Medium-A",
@@ -32,7 +60,11 @@
         "State Tracking": 0.492,
         "Syllogism": 0.415
       },
-      "by_difficulty": { "easy": 0.685, "medium": 0.522, "hard": 0.388 }
+      "by_difficulty": {
+        "easy": 0.685,
+        "medium": 0.522,
+        "hard": 0.388
+      }
     },
     {
       "model": "HuggingFaceTB/SmolLM2-135M",
@@ -48,7 +80,11 @@
         "State Tracking": 0.455,
         "Syllogism": 0.326
       },
-      "by_difficulty": { "easy": 0.592, "medium": 0.415, "hard": 0.31 }
+      "by_difficulty": {
+        "easy": 0.592,
+        "medium": 0.415,
+        "hard": 0.31
+      }
     },
     {
       "model": "HuggingFaceTB/SmolLM2-90M",
@@ -64,7 +100,49 @@
         "State Tracking": 0.372,
         "Syllogism": 0.298
       },
-      "by_difficulty": { "easy": 0.512, "medium": 0.368, "hard": 0.281 }
+      "by_difficulty": {
+        "easy": 0.512,
+        "medium": 0.368,
+        "hard": 0.281
+      }
+    },
+    {
+      "overall_acc": 0.36748404433993953,
+      "weighted_index": 0.3639346181956205,
+      "ci_index": 36.39,
+      "n": 2977,
+      "by_category": {
+        "Algorithmic": 0.26811594202898553,
+        "Counterfactual": 0.44651162790697674,
+        "Spatial": 0.3254901960784314,
+        "State Tracking": 0.47861271676300576,
+        "Syllogism": 0.3319327731092437
+      },
+      "by_difficulty": {
+        "easy": 0.3903177004538578,
+        "hard": 0.37701804368471037,
+        "medium": 0.3475851148060174
+      },
+      "model": "AxiomicLabs/GPT-X2.5-135M",
+      "time_s": 172.4,
+      "params": 135032256,
+      "params_m": 135.03,
+      "model_type": "gptx2",
+      "architectures": [
+        "GPTX2ForCausalLM"
+      ],
+      "hidden_size": 576,
+      "num_hidden_layers": 30,
+      "num_attention_heads": 9,
+      "num_key_value_heads": 3,
+      "intermediate_size": 1728,
+      "vocab_size": 32770,
+      "max_position_embeddings": 8192,
+      "tie_word_embeddings": true,
+      "rope_theta": 100000.0,
+      "tokenizer_vocab": 32770,
+      "loaded_dtype": "float16",
+      "gpu": "Tesla T4"
     },
     {
       "model": "HuggingFaceTB/SmolLM2-45M",
@@ -80,7 +158,11 @@
         "State Tracking": 0.318,
         "Syllogism": 0.271
       },
-      "by_difficulty": { "easy": 0.421, "medium": 0.315, "hard": 0.258 }
+      "by_difficulty": {
+        "easy": 0.421,
+        "medium": 0.315,
+        "hard": 0.258
+      }
     },
     {
       "model": "HuggingFaceTB/SmolLM2-17M",
@@ -96,7 +178,13 @@
         "State Tracking": 0.268,
         "Syllogism": 0.245
       },
-      "by_difficulty": { "easy": 0.315, "medium": 0.265, "hard": 0.241 }
+      "by_difficulty": {
+        "easy": 0.315,
+        "medium": 0.265,
+        "hard": 0.241
+      }
     }
-  ]
-}
+  ],
+  "run_at": "2026-09-02 23:36:56",
+  "updated_at": "2026-09-02"
+}