Upload add_sub_sorl_v6_abs30_K1_10K

add_sub_sorl_v6_abs30_K1_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v6_abs30_K1_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v6_abs30_K1_10K/metrics.json

@@ -0,0 +1,1759 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563,
+      1620,
+      1670,
+      1720,
+      1777,
+      1827,
+      1877,
+      1934,
+      1984,
+      2034,
+      2091,
+      2141,
+      2191,
+      2248,
+      2298,
+      2348,
+      2405,
+      2455,
+      2505,
+      2562,
+      2612,
+      2662,
+      2719,
+      2769,
+      2819,
+      2876,
+      2926,
+      2976,
+      3033,
+      3083,
+      3133
+    ],
+    "loss": [
+      7.265992164611816,
+      3.404426097869873,
+      1.954080581665039,
+      1.7737737894058228,
+      1.6873605251312256,
+      1.6128355264663696,
+      1.3722206354141235,
+      0.7695433497428894,
+      0.6166954040527344,
+      0.501417338848114,
+      0.4122486412525177,
+      0.395580917596817,
+      0.2903394401073456,
+      0.28911253809928894,
+      0.28732869029045105,
+      0.2673605978488922,
+      0.2304236888885498,
+      0.20156267285346985,
+      0.23798821866512299,
+      0.25514835119247437,
+      0.24532437324523926,
+      0.20568285882472992,
+      0.18954049050807953,
+      0.20266304910182953,
+      0.22727693617343903,
+      0.16868869960308075,
+      0.2077641785144806,
+      0.1665729135274887,
+      0.1537853330373764,
+      0.12550252676010132,
+      0.20871271193027496,
+      0.13248850405216217,
+      0.16434815526008606,
+      0.14673610031604767,
+      0.12699709832668304,
+      0.12676161527633667,
+      0.14729969203472137,
+      0.1481902301311493,
+      0.12077615410089493,
+      0.1248946338891983,
+      0.1077631339430809,
+      0.11444231122732162,
+      0.1087350994348526,
+      0.11233961582183838,
+      0.10330287367105484,
+      0.07320132106542587,
+      0.09157749265432358,
+      0.07470988482236862,
+      0.06538745015859604,
+      0.090658999979496,
+      0.07127472013235092,
+      0.10880061239004135,
+      0.08315103501081467,
+      0.07874953746795654,
+      0.05691386014223099,
+      0.07882998138666153,
+      0.06844855844974518,
+      0.08798480033874512,
+      0.07548613101243973,
+      0.04974472522735596
+    ],
+    "base_loss": [
+      8.48241901397705,
+      4.1414666175842285,
+      2.1049580574035645,
+      1.9186060428619385,
+      2.1715760231018066,
+      2.42810320854187,
+      2.720682382583618,
+      3.600949764251709,
+      4.264231204986572,
+      4.6700334548950195,
+      4.952982425689697,
+      5.340039253234863,
+      5.130460262298584,
+      5.388435363769531,
+      5.828049182891846,
+      5.812817096710205,
+      5.980282306671143,
+      6.1527228355407715,
+      6.522794246673584,
+      6.62658166885376,
+      6.32257604598999,
+      6.690981864929199,
+      6.888500213623047,
+      6.721907138824463,
+      7.000502586364746,
+      7.138729572296143,
+      6.728172302246094,
+      7.131743907928467,
+      6.915163516998291,
+      7.117414951324463,
+      7.412054538726807,
+      7.208516597747803,
+      7.608938694000244,
+      7.313349723815918,
+      7.931239128112793,
+      7.436122417449951,
+      7.586681842803955,
+      7.517941951751709,
+      7.97885799407959,
+      7.805827617645264,
+      8.07812786102295,
+      7.750722408294678,
+      7.977327823638916,
+      8.034120559692383,
+      8.192937850952148,
+      8.292710304260254,
+      8.43775749206543,
+      8.385404586791992,
+      8.477164268493652,
+      8.072797775268555,
+      8.61682415008545,
+      8.179729461669922,
+      8.743082046508789,
+      8.736411094665527,
+      8.595890998840332,
+      8.414719581604004,
+      8.458391189575195,
+      8.735125541687012,
+      8.983786582946777,
+      8.911020278930664
+    ],
+    "traj_loss": [
+      7.265992164611816,
+      3.404426097869873,
+      1.954080581665039,
+      1.7737737894058228,
+      1.6873605251312256,
+      1.6128355264663696,
+      1.3722206354141235,
+      0.7695433497428894,
+      0.6166954040527344,
+      0.501417338848114,
+      0.4122486412525177,
+      0.395580917596817,
+      0.2903394401073456,
+      0.28911253809928894,
+      0.28732869029045105,
+      0.2673605978488922,
+      0.2304236888885498,
+      0.20156267285346985,
+      0.23798821866512299,
+      0.25514835119247437,
+      0.24532437324523926,
+      0.20568285882472992,
+      0.18954049050807953,
+      0.20266304910182953,
+      0.22727693617343903,
+      0.16868869960308075,
+      0.2077641785144806,
+      0.1665729135274887,
+      0.1537853330373764,
+      0.12550252676010132,
+      0.20871271193027496,
+      0.13248850405216217,
+      0.16434815526008606,
+      0.14673610031604767,
+      0.12699709832668304,
+      0.12676161527633667,
+      0.14729969203472137,
+      0.1481902301311493,
+      0.12077615410089493,
+      0.1248946338891983,
+      0.1077631339430809,
+      0.11444231122732162,
+      0.1087350994348526,
+      0.11233961582183838,
+      0.10330287367105484,
+      0.07320132106542587,
+      0.09157749265432358,
+      0.07470988482236862,
+      0.06538745015859604,
+      0.090658999979496,
+      0.07127472013235092,
+      0.10880061239004135,
+      0.08315103501081467,
+      0.07874953746795654,
+      0.05691386014223099,
+      0.07882998138666153,
+      0.06844855844974518,
+      0.08798480033874512,
+      0.07548613101243973,
+      0.04974472522735596
+    ],
+    "hinge_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "masked_traj_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "abs_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "zipf_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "ortho_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "anchor_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "jacobi_loss": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "lr": [
+      3.9200000000000004e-05,
+      7.92e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      7.946710526315791e-05,
+      7.650657894736843e-05,
+      7.354605263157895e-05,
+      7.017105263157896e-05,
+      6.721052631578948e-05,
+      6.425e-05,
+      6.0875e-05,
+      5.791447368421054e-05,
+      5.495394736842105e-05,
+      5.157894736842105e-05,
+      4.861842105263157e-05,
+      4.565789473684212e-05,
+      4.2282894736842104e-05,
+      3.9322368421052625e-05,
+      3.636184210526315e-05,
+      3.2986842105263165e-05,
+      3.0026315789473686e-05,
+      2.7065789473684206e-05,
+      2.3690789473684223e-05,
+      2.0730263157894743e-05,
+      1.7769736842105264e-05,
+      1.4394736842105275e-05,
+      1.1434210526315796e-05,
+      8.473684210526318e-06
+    ]
+  },
+  "final_accuracy": 0.7083333333333334,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2396694214876033,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.11578947368421053,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.22549019607843138,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.0650887573964497,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.06451612903225806,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.22297297297297297,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.24539877300613497,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.08461538461538462,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.1724137931034483,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.2660098522167488,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.07692307692307693,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2727272727272727,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.0743801652892562,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.08163265306122448,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.25268817204301075,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.08071748878923767,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.25961538461538464,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.10377358490566038,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.08695652173913043,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.35625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.0781758957654723,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.08,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.28,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.045,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.54,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.092,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2348993288590604,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.09375,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.19642857142857142,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.19092627599243855,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.14583333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.29,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.17098445595854922,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.07476635514018691,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.35,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.1484375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.09027777777777778,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.13,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.1568627450980392,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.08247422680412371,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.24316939890710382,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.08974358974358974,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.15474209650582363,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.08080808080808081,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.2903225806451613,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.0896551724137931,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.13095238095238096,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3145539906103286,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.11504424778761062,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.058823529411764705,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.14917127071823205,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.046296296296296294,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.30726256983240224,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.11650485436893204,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.14285714285714285,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.11409395973154363,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.07981220657276995,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.345,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.1,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.13,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.54,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.1,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0775,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.25666666666666665,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.10112359550561797,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.07547169811320754,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.10022779043280182,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.07317073170731707,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.25333333333333335,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.09644670050761421,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.05825242718446602,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.315,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.12,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.07692307692307693,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.11764705882352941,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.53,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.11,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.13087248322147652,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.10396039603960396,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.0,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9933884297520661,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.81,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9815950920245399,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9769230769230769,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9540229885057471,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9507389162561576,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9829059829059829,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.66,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9669421487603306,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.8333333333333334,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9820627802690582,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.66,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9903846153846154,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.825,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8990228013029316,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.51,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.795,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.71,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.78,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.84,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.915,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9888143176733781,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.99375,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9810964083175804,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.9583333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.76,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8911917098445595,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9345794392523364,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.890625,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9305555555555556,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.79,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9379084967320261,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8865979381443299,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.79,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9453551912568307,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9829059829059829,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9950083194675541,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.985663082437276,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9920634920634921,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.77,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9859154929577465,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9734513274336283,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.9060773480662984,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.48,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9805825242718447,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.7114093959731543,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.8873239436619719,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.17,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.34,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.65,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.39,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.44,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.93,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.995,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9962546816479401,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9772209567198178,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.975609756097561,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.868020304568528,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8155339805825242,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.54,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8340080971659919,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.8169934640522876,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.42,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8288590604026845,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.8168316831683168,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.7083333333333334,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.7083333333333334,
+  "sft_overall_accuracy": 0.0
+}

add_sub_sorl_v6_abs30_K1_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1d607ac060c522a102044d02394e840ea3e9f4ccfa4c73f9273250fd706f27a
+size 650385300

add_sub_sorl_v6_abs30_K1_10K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 8e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_v6_abs30_K1_10K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 10000,
+  "mode": "sorl_v6",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162519662,
+  "run_name": "add_sub_sorl_v6_abs30_K1_10K",
+  "git_commit": "17e935f460a7f9595b705c1d614101a6b0e520f7",
+  "timestamp": "2026-04-14T10:04:32.103930+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v6",
+  "wandb_run_id": "8ispzzg0",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/8ispzzg0",
+  "final_accuracy": 0.7083333333333334,
+  "sft_accuracy": 0.0,
+  "eval_method": "ArithmeticEvaluator"
+}