Upload add_sub_sorl_v1_abs10_10K

add_sub_sorl_v1_abs10_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151654
+}

add_sub_sorl_v1_abs10_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs10_10K/metrics.json

@@ -0,0 +1,1297 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563
+    ],
+    "loss": [
+      13.724945068359375,
+      7.864343166351318,
+      5.013725280761719,
+      2.741554021835327,
+      2.641360282897949,
+      2.458710193634033,
+      2.5750374794006348,
+      2.7018885612487793,
+      2.5789809226989746,
+      2.3690404891967773,
+      1.7872730493545532,
+      1.8722014427185059,
+      1.4984050989151,
+      1.2242059707641602,
+      1.2751460075378418,
+      1.0107839107513428,
+      1.6164339780807495,
+      0.9860085248947144,
+      0.9987770915031433,
+      1.071738600730896,
+      0.49998074769973755,
+      0.004859551787376404,
+      -0.1695592701435089,
+      0.0747871994972229,
+      -0.27059921622276306,
+      -0.13733237981796265,
+      0.056721221655607224,
+      0.051335956901311874,
+      -0.45392414927482605,
+      -0.3348205089569092
+    ],
+    "base_loss": [
+      7.9022040367126465,
+      5.812614917755127,
+      3.858829975128174,
+      2.114943742752075,
+      1.96530020236969,
+      1.9036047458648682,
+      1.893314242362976,
+      1.8308947086334229,
+      1.8680362701416016,
+      1.7989755868911743,
+      1.8180350065231323,
+      1.8677423000335693,
+      1.862571120262146,
+      1.8560221195220947,
+      1.838614583015442,
+      1.7637040615081787,
+      1.8746064901351929,
+      1.8687989711761475,
+      1.8472496271133423,
+      1.8745392560958862,
+      1.8059179782867432,
+      1.755523920059204,
+      1.7115023136138916,
+      1.7793734073638916,
+      1.759653091430664,
+      1.7252968549728394,
+      1.7201368808746338,
+      1.7096773386001587,
+      1.7525498867034912,
+      1.710257887840271
+    ],
+    "info_loss": [
+      -0.2345585823059082,
+      -0.18186569213867188,
+      -0.11677074432373047,
+      -0.13591337203979492,
+      -0.12461328506469727,
+      -0.13464367389678955,
+      -0.12099599838256836,
+      -0.10128498077392578,
+      -0.11677730083465576,
+      -0.13061606884002686,
+      -0.1908276081085205,
+      -0.18726766109466553,
+      -0.2237793207168579,
+      -0.25037193298339844,
+      -0.24307358264923096,
+      -0.26141083240509033,
+      -0.2112140655517578,
+      -0.2714040279388428,
+      -0.2562897205352783,
+      -0.21821486949920654,
+      -0.1999720335006714,
+      -0.19893121719360352,
+      -0.20633161067962646,
+      -0.1859527826309204,
+      -0.21592140197753906,
+      -0.19767189025878906,
+      -0.17773842811584473,
+      -0.17674016952514648,
+      -0.2308502197265625,
+      -0.21450459957122803
+    ],
+    "abs_loss": [
+      2.2656421661376953,
+      2.060791254043579,
+      1.8972232341766357,
+      1.8288743495941162,
+      1.8518760204315186,
+      1.8478550910949707,
+      1.8548732995986938,
+      1.8174316883087158,
+      1.8104970455169678,
+      1.8124628067016602,
+      1.8303263187408447,
+      1.860059380531311,
+      1.8033771514892578,
+      1.8484278917312622,
+      1.7977077960968018,
+      1.7986536026000977,
+      1.8080449104309082,
+      1.7756034135818481,
+      1.678572177886963,
+      1.2864962816238403,
+      0.798673152923584,
+      0.611975371837616,
+      0.5902649164199829,
+      0.584165632724762,
+      0.5656200647354126,
+      0.5475164651870728,
+      0.5673137903213501,
+      0.5301952362060547,
+      0.4829615652561188,
+      0.5067222118377686
+    ],
+    "zipf_loss": [
+      7.941762447357178,
+      3.664306163787842,
+      2.132880449295044,
+      1.8028565645217896,
+      1.7370052337646484,
+      1.7167567014694214,
+      1.7061958312988281,
+      1.7021006345748901,
+      1.6976680755615234,
+      1.694979190826416,
+      1.6944814920425415,
+      1.6911298036575317,
+      1.6932895183563232,
+      1.6870603561401367,
+      1.6874964237213135,
+      1.6813228130340576,
+      1.673163652420044,
+      1.6536895036697388,
+      1.546567440032959,
+      1.250698447227478,
+      0.6139158010482788,
+      0.1774502694606781,
+      0.1232280433177948,
+      0.09652505815029144,
+      0.07239970564842224,
+      0.059338025748729706,
+      0.05723724141716957,
+      0.05604078993201256,
+      0.053732018917798996,
+      0.04929535463452339
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.5284707546234131,
+      0.21797625720500946,
+      0.12806634604930878,
+      0.09031304717063904,
+      0.08532247692346573,
+      0.07537004351615906,
+      0.07645349949598312,
+      0.07695373892784119,
+      0.07142108678817749,
+      0.08406423032283783,
+      0.08843859285116196,
+      0.07815887778997421,
+      0.07982385903596878,
+      0.07472201436758041,
+      0.07963292300701141,
+      0.07144557684659958,
+      0.07161661982536316,
+      0.07070355117321014,
+      0.06902579218149185,
+      0.06851960718631744,
+      0.06775054335594177,
+      0.0655888020992279,
+      0.059131793677806854,
+      0.05900496616959572,
+      0.056693185120821,
+      0.05708722025156021,
+      0.058303531259298325,
+      0.059081800282001495,
+      0.059238508343696594,
+      0.059114448726177216
+    ],
+    "lr": [
+      1.9600000000000002e-05,
+      3.96e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.944897959183673e-05,
+      3.638775510204082e-05,
+      3.3326530612244897e-05,
+      2.983673469387755e-05,
+      2.6775510204081637e-05,
+      2.3714285714285713e-05,
+      2.022448979591837e-05,
+      1.7163265306122454e-05,
+      1.4102040816326535e-05,
+      1.0612244897959182e-05,
+      7.551020408163262e-06,
+      4.48979591836735e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      150,
+      307,
+      464,
+      621,
+      778,
+      935,
+      1092,
+      1199,
+      1356,
+      1513
+    ],
+    "eval_accuracy": [
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.02,
+      0.01,
+      0.01
+    ]
+  },
+  "final_accuracy": 0.008333333333333333,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.24297520661157024,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.8736842105263158,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.28921568627450983,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.14792899408284024,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.8064516129032258,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.23648648648648649,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.3619631901840491,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.1,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.4827586206896552,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.33497536945812806,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.5384615384615384,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4132231404958678,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.06611570247933884,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.46938775510204084,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.34946236559139787,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.5650224215246636,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4519230769230769,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.08490566037735849,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.6086956521739131,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.40625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.4560260586319218,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.43,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.06,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.51,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.36,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.05,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.05,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.53,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.502,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.27293064876957496,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.1125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.6964285714285714,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.2665406427221172,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.5,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.31666666666666665,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.19689119170984457,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.2523364485981308,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.405,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.05,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.1484375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.2986111111111111,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.54,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.1,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.24836601307189543,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.5,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.14,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.2459016393442623,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.6709401709401709,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.23960066555740434,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.8282828282828283,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.4444444444444444,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.034482758620689655,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.875,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.1388888888888889,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6291079812206573,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.035398230088495575,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.8705882352941177,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.19337016574585636,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.1388888888888889,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7374301675977654,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.02912621359223301,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.8928571428571429,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.2214765100671141,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.13145539906103287,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.55,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.06,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.35,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.16666666666666666,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.16,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.4066666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.011235955056179775,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.7547169811320755,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.1662870159453303,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.21951219512195122,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.38,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.16243654822335024,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.20388349514563106,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.55,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17408906882591094,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.1568627450980392,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.1644295302013423,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.1188118811881188,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.004166666666666667,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.3256198347107438,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.3431372549019608,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.23076923076923078,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.8387096774193549,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.32094594594594594,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4785276073619632,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.13076923076923078,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.40229885057471265,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.4039408866995074,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.5897435897435898,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.5289256198347108,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.0743801652892562,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.4897959183673469,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.41935483870967744,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.5964125560538116,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.5480769230769231,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.08490566037735849,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.6521739130434783,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.41875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.5374592833876222,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.46,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.05,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.46,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.3175,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.12,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.48,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.448,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.36017897091722595,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.18125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.625,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.3534971644612476,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.5,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.22279792746113988,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.37383177570093457,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.53,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.04,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.19921875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.4513888888888889,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.57,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.30392156862745096,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.4948453608247423,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.09,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.31693989071038253,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.7521367521367521,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3277870216306156,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9090909090909091,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5483870967741935,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.05517241379310345,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.6666666666666666,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.2222222222222222,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7183098591549296,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.017699115044247787,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9058823529411765,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.23204419889502761,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.2037037037037037,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7932960893854749,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.02912621359223301,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9285714285714286,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.33557046979865773,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.2112676056338028,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.725,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.1,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.42,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.31,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.03,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.71,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.285,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.53,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.03745318352059925,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.7358490566037735,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.23917995444191345,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.17073170731707318,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.51,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.13,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.18274111675126903,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.39805825242718446,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.665,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.1,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.21862348178137653,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.38562091503267976,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.07,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.2953020134228188,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.33663366336633666,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.008333333333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.008333333333333333,
+  "sft_overall_accuracy": 0.004166666666666667
+}

add_sub_sorl_v1_abs10_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5539570ac41c2434a03e526e01a1a87e8aa66fcfadd6151a66949f140ded418d
+size 650303660

add_sub_sorl_v1_abs10_10K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K4_10K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 10,
+  "dataset_size": 10000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162499262,
+  "run_name": "add_sub_sorl_v1_abs10_10K",
+  "git_commit": "78d46f8665a87f4b44bd5894bd34f393f2dea51f",
+  "timestamp": "2026-04-12T08:59:09.036996+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "f2ajfc34",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/f2ajfc34",
+  "final_accuracy": 0.008333333333333333,
+  "sft_accuracy": 0.004166666666666667,
+  "eval_method": "ArithmeticEvaluator"
+}