diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1a153fd08fd69841e1fb42fe164e5aeae592be6c --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +--- +license: mit +tags: + - transformer + - interpretability + - from-scratch +library_name: pytorch +--- + +# tiny-transformer + +Trained runs for [tiny-transformer](https://github.com/coeiico/tiny-transformer): a readable, hookable GPT-style decoder-only transformer built from scratch in PyTorch, trained here on its demo task, sorting lists of 16 values. The code, the full write-up and the figures are on GitHub. + +Each folder is one run: evaluation weights (`model.safetensors`), the architecture and vocabulary (`model_config.json`), the full training record (`metrics.json`: config, seeds, data definition, code state, environment, history), a diagnostic plot, and the final analysis under `analysis/`. + +| Run | What it is | Lists sorted exactly (test, 2,048 lists) | +| --- | --- | --- | +| `default-seed0` | 4 layers, 3.2M parameters | 100% | +| `default-seed1` | same, seed 1 | 100% | +| `default-seed2` | same, seed 2 | 99.8% | +| `bigram-seed0` | baseline: no layers (`--model.n-layers 0`), sees only the previous token | 0% | + +To load one, with the [tiny-transformer](https://github.com/coeiico/tiny-transformer) package installed: + +```python +from pathlib import Path +from huggingface_hub import snapshot_download +from tiny_transformer.hub import load_model + +folder = Path(snapshot_download('coeiico/tiny-transformer', allow_patterns='default-seed0/*')) / 'default-seed0' +model, vocab = load_model(folder) +``` + +These are evaluation weights, not resumable checkpoints. The runs were trained on CUDA (an RTX 3060); a seed fixes a run on one kind of device. diff --git a/bigram-seed0/analysis/figures/figure-data/curves-bigram-seed0.csv b/bigram-seed0/analysis/figures/figure-data/curves-bigram-seed0.csv new file mode 100644 index 0000000000000000000000000000000000000000..a812ca0c230f547beb671913233f574220e4c2df --- /dev/null +++ b/bigram-seed0/analysis/figures/figure-data/curves-bigram-seed0.csv @@ -0,0 +1,17 @@ +step,train_loss,val_loss,val_token_accuracy,val_exact_match +250,2.346252202987671,2.3416833877563477,0.19580078125,0.0 +500,2.3324460983276367,2.33697247505188,0.19647216796875,0.0 +750,2.339632511138916,2.337479591369629,0.1966552734375,0.0 +1000,2.33378267288208,2.336371421813965,0.193359375,0.0 +1250,2.3311407566070557,2.3351407051086426,0.1973876953125,0.0 +1500,2.3376550674438477,2.3326027393341064,0.2003173828125,0.0 +1750,2.3303802013397217,2.3325517177581787,0.19921875,0.0 +2000,2.333252429962158,2.3325724601745605,0.1982421875,0.0 +2250,2.329752206802368,2.3292136192321777,0.19915771484375,0.0 +2500,2.325176239013672,2.329281806945801,0.19976806640625,0.0 +2750,2.3266186714172363,2.3286523818969727,0.200927734375,0.0 +3000,2.3282289505004883,2.3266916275024414,0.20263671875,0.0 +3250,2.3204801082611084,2.3267407417297363,0.20269775390625,0.0 +3500,2.3247458934783936,2.3262388706207275,0.20135498046875,0.0 +3750,2.3221328258514404,2.325857162475586,0.2020263671875,0.0 +4000,2.3267335891723633,2.324476718902588,0.20184326171875,0.0 diff --git a/bigram-seed0/analysis/figures/results.json b/bigram-seed0/analysis/figures/results.json new file mode 100644 index 0000000000000000000000000000000000000000..9ab43ea02759a207d268251cbd7cb01316db5569 --- /dev/null +++ b/bigram-seed0/analysis/figures/results.json @@ -0,0 +1,29 @@ +{ + "run": "runs/bigram-seed0", + "code": { + "commit": "42933bd7a4e2e13fd737d0d48d489f72fbeed8ac", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 2.323464870452881, + "token_accuracy": 0.20025634765625, + "exact_match": 0.0 + }, + "untrained": { + "loss": 4.254000663757324, + "token_accuracy": 0.011566162109375, + "exact_match": 0.0 + } +} diff --git a/bigram-seed0/analysis/figures/source-snapshot.tar.gz b/bigram-seed0/analysis/figures/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cc276e4403df0d58c1c3940086f7f1eadca32451 --- /dev/null +++ b/bigram-seed0/analysis/figures/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade11e9af3c8d5c2d74eaab0390b06cc6580dcebb2fa5fe3a3d64b4974b3ae70 +size 347464 diff --git a/bigram-seed0/analysis/final/results.json b/bigram-seed0/analysis/final/results.json new file mode 100644 index 0000000000000000000000000000000000000000..d988c446eefe44a2aa70c21405f3c1b170e0fa3a --- /dev/null +++ b/bigram-seed0/analysis/final/results.json @@ -0,0 +1,29 @@ +{ + "run": "runs/bigram-seed0", + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 2.323464870452881, + "token_accuracy": 0.20025634765625, + "exact_match": 0.0 + }, + "untrained": { + "loss": 4.254000663757324, + "token_accuracy": 0.011566162109375, + "exact_match": 0.0 + } +} diff --git a/bigram-seed0/analysis/final/source-snapshot.tar.gz b/bigram-seed0/analysis/final/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..befa036f4597f41b86fba3211bd75d18685b1af3 --- /dev/null +++ b/bigram-seed0/analysis/final/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc7e4cff4eccbd18081051577563726d7aa7a1642c5680295777bc9ce1c17ce5 +size 189601 diff --git a/bigram-seed0/metrics.json b/bigram-seed0/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..83831fd00f9fdbc241e958d77da7908d20fdbfce --- /dev/null +++ b/bigram-seed0/metrics.json @@ -0,0 +1,228 @@ +{ + "config": { + "run_name": "bigram-seed0", + "seed": 0, + "quick": false, + "device": "auto", + "eval_every": 250, + "task": { + "n_values": 64, + "length": 16 + }, + "splits": { + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048 + }, + "model": { + "n_layers": 0, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "optim": { + "steps": 4000, + "batch_size": 256, + "lr": 0.001, + "warmup_steps": 100, + "min_lr_ratio": 0.1, + "weight_decay": 0.1, + "grad_clip": 1.0 + } + }, + "model_config": { + "vocab_size": 65, + "n_layers": 0, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "parameters": 42240, + "data": { + "task": "sorting", + "version": 1, + "n_values": 64, + "length": 16, + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048, + "train_seed": 0 + }, + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "resumable": false, + "status": "finished", + "seconds": 141.9, + "history": [ + { + "step": 250, + "train_loss": 2.346252202987671, + "val_loss": 2.3416833877563477, + "val_token_accuracy": 0.19580078125, + "val_exact_match": 0.0, + "grad_norm": 0.4313671886920929, + "lr": 0.0009967189933441244 + }, + { + "step": 500, + "train_loss": 2.3324460983276367, + "val_loss": 2.33697247505188, + "val_token_accuracy": 0.19647216796875, + "val_exact_match": 0.0, + "grad_norm": 0.3702669143676758, + "lr": 0.0009768413988762155 + }, + { + "step": 750, + "train_loss": 2.339632511138916, + "val_loss": 2.337479591369629, + "val_token_accuracy": 0.1966552734375, + "val_exact_match": 0.0, + "grad_norm": 0.3715057969093323, + "lr": 0.0009397114317029974 + }, + { + "step": 1000, + "train_loss": 2.33378267288208, + "val_loss": 2.336371421813965, + "val_token_accuracy": 0.193359375, + "val_exact_match": 0.0, + "grad_norm": 0.35880762338638306, + "lr": 0.0008868298366769955 + }, + { + "step": 1250, + "train_loss": 2.3311407566070557, + "val_loss": 2.3351407051086426, + "val_token_accuracy": 0.1973876953125, + "val_exact_match": 0.0, + "grad_norm": 0.3279813230037689, + "lr": 0.0008203340189070906 + }, + { + "step": 1500, + "train_loss": 2.3376550674438477, + "val_loss": 2.3326027393341064, + "val_token_accuracy": 0.2003173828125, + "val_exact_match": 0.0, + "grad_norm": 0.34029656648635864, + "lr": 0.0007429116526313744 + }, + { + "step": 1750, + "train_loss": 2.3303802013397217, + "val_loss": 2.3325517177581787, + "val_token_accuracy": 0.19921875, + "val_exact_match": 0.0, + "grad_norm": 0.3435227870941162, + "lr": 0.0006576920489294011 + }, + { + "step": 2000, + "train_loss": 2.333252429962158, + "val_loss": 2.3325724601745605, + "val_token_accuracy": 0.1982421875, + "val_exact_match": 0.0, + "grad_norm": 0.31740111112594604, + "lr": 0.0005681196730492369 + }, + { + "step": 2250, + "train_loss": 2.329752206802368, + "val_loss": 2.3292136192321777, + "val_token_accuracy": 0.19915771484375, + "val_exact_match": 0.0, + "grad_norm": 0.31328055262565613, + "lr": 0.00047781492361400794 + }, + { + "step": 2500, + "train_loss": 2.325176239013672, + "val_loss": 2.329281806945801, + "val_token_accuracy": 0.19976806640625, + "val_exact_match": 0.0, + "grad_norm": 0.3233284652233124, + "lr": 0.0003904278008308589 + }, + { + "step": 2750, + "train_loss": 2.3266186714172363, + "val_loss": 2.3286523818969727, + "val_token_accuracy": 0.200927734375, + "val_exact_match": 0.0, + "grad_norm": 0.32487940788269043, + "lr": 0.0003094903782424895 + }, + { + "step": 3000, + "train_loss": 2.3282289505004883, + "val_loss": 2.3266916275024414, + "val_token_accuracy": 0.20263671875, + "val_exact_match": 0.0, + "grad_norm": 0.30471721291542053, + "lr": 0.0002382740409206803 + }, + { + "step": 3250, + "train_loss": 2.3204801082611084, + "val_loss": 2.3267407417297363, + "val_token_accuracy": 0.20269775390625, + "val_exact_match": 0.0, + "grad_norm": 0.31849977374076843, + "lr": 0.00017965726034785466 + }, + { + "step": 3500, + "train_loss": 2.3247458934783936, + "val_loss": 2.3262388706207275, + "val_token_accuracy": 0.20135498046875, + "val_exact_match": 0.0, + "grad_norm": 0.29355937242507935, + "lr": 0.00013600925035352912 + }, + { + "step": 3750, + "train_loss": 2.3221328258514404, + "val_loss": 2.325857162475586, + "val_token_accuracy": 0.2020263671875, + "val_exact_match": 0.0, + "grad_norm": 0.30890047550201416, + "lr": 0.00010909420658097956 + }, + { + "step": 4000, + "train_loss": 2.3267335891723633, + "val_loss": 2.324476718902588, + "val_token_accuracy": 0.20184326171875, + "val_exact_match": 0.0, + "grad_norm": 0.3154265582561493, + "lr": 0.0001 + } + ], + "final": { + "val_loss": 2.324476718902588, + "val_token_accuracy": 0.20184326171875, + "val_exact_match": 0.0 + } +} diff --git a/bigram-seed0/model.safetensors b/bigram-seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5309f0cd1fe2c90844cb68e00bbdf16f3817151a --- /dev/null +++ b/bigram-seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51f191672160991e25bc1cbb7e37178444fa3a3fba846188aca9d896a972215 +size 169360 diff --git a/bigram-seed0/model_config.json b/bigram-seed0/model_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9775fae012332681c3d840a135add4c2a8596b6a --- /dev/null +++ b/bigram-seed0/model_config.json @@ -0,0 +1,81 @@ +{ + "format_version": 1, + "transformer": { + "vocab_size": 65, + "n_layers": 0, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "vocab": [ + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "SEP" + ] +} diff --git a/bigram-seed0/source-snapshot.tar.gz b/bigram-seed0/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9cdbf2cae6983280a9a930e30254a87b8eddfc72 --- /dev/null +++ b/bigram-seed0/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e963742ccde0aca2462ea11e294c6b2504355b7b78eda8738885e210454496e +size 187056 diff --git a/bigram-seed0/training.png b/bigram-seed0/training.png new file mode 100644 index 0000000000000000000000000000000000000000..4dfdf832ce110f448f16cbd2017ed53bec64c1a0 Binary files /dev/null and b/bigram-seed0/training.png differ diff --git a/default-seed0/analysis/figures/critical_pattern.png b/default-seed0/analysis/figures/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..2bf99d95f050addb9392a8d0361220fa5cfbe7aa Binary files /dev/null and b/default-seed0/analysis/figures/critical_pattern.png differ diff --git a/default-seed0/analysis/figures/figure-data/curves-default-seed0.csv b/default-seed0/analysis/figures/figure-data/curves-default-seed0.csv new file mode 100644 index 0000000000000000000000000000000000000000..ae4c252a690279e28a2d740cc8cbcee4bbabf946 --- /dev/null +++ b/default-seed0/analysis/figures/figure-data/curves-default-seed0.csv @@ -0,0 +1,17 @@ +step,train_loss,val_loss,val_token_accuracy,val_exact_match +250,0.07291533797979355,0.0985519140958786,0.9683837890625,0.595703125 +500,0.014207392930984497,0.031683966517448425,0.99053955078125,0.8720703125 +750,0.02052781730890274,0.023844445124268532,0.99273681640625,0.890625 +1000,0.009082821197807789,0.01231020875275135,0.996826171875,0.951171875 +1250,0.00922011211514473,0.019146941602230072,0.994873046875,0.9208984375 +1500,0.01364622451364994,0.005576150491833687,0.998779296875,0.9814453125 +1750,0.003266311716288328,0.001853839261457324,0.9996337890625,0.994140625 +2000,0.005052485503256321,0.0006970376125536859,0.9998779296875,0.998046875 +2250,0.00020874346955679357,0.0009217980550602078,0.99981689453125,0.9970703125 +2500,0.001243826001882553,0.0005396516062319279,0.99981689453125,0.9970703125 +2750,0.00022199669911060482,0.00017582655709702522,0.99993896484375,0.9990234375 +3000,0.00021956569980829954,8.435545896645635e-05,1.0,1.0 +3250,3.6058951081940904e-05,8.97766076377593e-05,0.99993896484375,0.9990234375 +3500,4.75610613648314e-06,9.887960004562046e-06,1.0,1.0 +3750,0.00010177387594012544,0.00023778356262482703,0.99993896484375,0.9990234375 +4000,1.727779454085976e-05,1.847820749389939e-05,1.0,1.0 diff --git a/default-seed0/analysis/figures/figure-data/heads-default-seed0.csv b/default-seed0/analysis/figures/figure-data/heads-default-seed0.csv new file mode 100644 index 0000000000000000000000000000000000000000..f154946d489013f12daa97d95f08e70aeb548ccf --- /dev/null +++ b/default-seed0/analysis/figures/figure-data/heads-default-seed0.csv @@ -0,0 +1,33 @@ +layer,head,loss_rise,token_accuracy_drop,lookup_current,lookup_next +0,0,0.09456062089066108,0.021453857421875,0.58856201171875,0.196746826171875 +0,1,0.006693977446957433,0.001556396484375,0.199432373046875,0.322418212890625 +0,2,2.235693497167631,0.322906494140625,0.859954833984375,0.204315185546875 +0,3,0.15027083228687843,0.017242431640625,0.282135009765625,0.372955322265625 +0,4,0.07599250744442543,0.010345458984375,0.2462158203125,0.4195556640625 +0,5,0.00419125804091891,0.00079345703125,0.19976806640625,0.290557861328125 +0,6,0.10219017933468422,0.010589599609375,0.03662109375,0.11737060546875 +0,7,5.169036907659574,0.764434814453125,0.988250732421875,0.136383056640625 +1,0,0.00025813918091444066,6.103515625e-05,0.286376953125,0.53216552734375 +1,1,2.973878781631356e-05,3.0517578125e-05,0.254669189453125,0.451263427734375 +1,2,0.00024059577026491752,9.1552734375e-05,0.499359130859375,0.471405029296875 +1,3,0.0009068534045582055,0.0003662109375,0.36083984375,0.4954833984375 +1,4,0.003771953800423944,0.001220703125,0.21551513671875,0.45703125 +1,5,0.0006174362997626304,0.00018310546875,0.195831298828125,0.5029296875 +1,6,0.000752416023715341,0.000274658203125,0.14984130859375,0.46435546875 +1,7,0.004184644253655279,0.00128173828125,0.171142578125,0.466583251953125 +2,0,0.00014806245508225402,3.0517578125e-05,0.3846435546875,0.43756103515625 +2,1,4.894880476058461e-06,0.0,0.24420166015625,0.59722900390625 +2,2,2.7085926376457792e-05,0.0,0.2579345703125,0.504119873046875 +2,3,0.00017150774419860682,6.103515625e-05,0.267059326171875,0.58270263671875 +2,4,0.00021039024341007462,0.0001220703125,0.279937744140625,0.451690673828125 +2,5,0.0007992379169081687,0.000152587890625,0.378326416015625,0.468170166015625 +2,6,0.0008647132117403089,0.00030517578125,0.229736328125,0.674285888671875 +2,7,1.1404487850086298e-05,0.0,0.234710693359375,0.673553466796875 +3,0,1.613488984730793e-05,0.0,0.243560791015625,0.269866943359375 +3,1,8.192873792722821e-07,0.0,0.227264404296875,0.328369140625 +3,2,1.7713737179292366e-06,0.0,0.204864501953125,0.2083740234375 +3,3,2.291507462359732e-05,0.0,0.244720458984375,0.380889892578125 +3,4,0.0001252591091542854,0.0,0.27691650390625,0.379302978515625 +3,5,5.263272214506287e-06,0.0,0.246490478515625,0.265625 +3,6,4.800411898031598e-05,0.0,0.29443359375,0.322174072265625 +3,7,7.388479389192071e-06,0.0,0.200347900390625,0.201141357421875 diff --git a/default-seed0/analysis/figures/figure-data/layers-default-seed0.csv b/default-seed0/analysis/figures/figure-data/layers-default-seed0.csv new file mode 100644 index 0000000000000000000000000000000000000000..7530dad51ea97bf70549fbd37edd504100787528 --- /dev/null +++ b/default-seed0/analysis/figures/figure-data/layers-default-seed0.csv @@ -0,0 +1,5 @@ +layer,loss_rise,token_accuracy_drop +0,13.57202391098599,0.88677978515625 +1,0.19787378381352028,0.04681396484375 +2,0.04730297024707397,0.016082763671875 +3,0.0025761095375855803,0.000274658203125 diff --git a/default-seed0/analysis/figures/figure-data/pattern-default-seed0.json b/default-seed0/analysis/figures/figure-data/pattern-default-seed0.json new file mode 100644 index 0000000000000000000000000000000000000000..5c6ed80d71dd803620a7ddeac17919af3090818b --- /dev/null +++ b/default-seed0/analysis/figures/figure-data/pattern-default-seed0.json @@ -0,0 +1,1279 @@ +{ + "layer": 0, + "head": 7, + "tokens": [ + "21", + "2", + "15", + "4", + "51", + "35", + "37", + "52", + "31", + "62", + "5", + "38", + "9", + "37", + "4", + "45", + "SEP", + "2", + "4", + "4", + "5", + "9", + "15", + "21", + "31", + "35", + "37", + "37", + "38", + "45", + "51", + "52", + "62" + ], + "input_length": 16, + "pattern": [ + [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.023018356412649155, + 0.9769816994667053, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0045615979470312595, + 0.002258468186482787, + 0.9931799173355103, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.03559539467096329, + 0.06540653854608536, + 0.011447257362306118, + 0.8875508904457092, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.021527551114559174, + 0.006806362420320511, + 0.012866882607340813, + 0.010566228069365025, + 0.9482329487800598, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007833131588995457, + 0.0024875393137335777, + 0.01399846188724041, + 0.00461894366890192, + 0.0051492964848876, + 0.9659126996994019, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0009257665951736271, + 0.006245102733373642, + 0.0074912612326443195, + 0.008376035839319229, + 0.006875417195260525, + 0.055685821920633316, + 0.9144006371498108, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.003037565154954791, + 0.005996344611048698, + 0.0092690484598279, + 0.011296346783638, + 0.06189500913023949, + 0.008469926193356514, + 0.01204671896994114, + 0.8879890441894531, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0025577344931662083, + 0.0022750389762222767, + 0.0011446349089965224, + 0.0034309474285691977, + 0.006817675661295652, + 0.004609635099768639, + 0.002528931712731719, + 0.009095567278563976, + 0.9675397276878357, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.026828112080693245, + 0.011514825746417046, + 0.030997384339571, + 0.06020127609372139, + 0.12983357906341553, + 0.04715048521757126, + 0.023005908355116844, + 0.03150458261370659, + 0.005654371343553066, + 0.6333094239234924, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.01655244454741478, + 0.014901078306138515, + 0.007255906239151955, + 0.04292556270956993, + 0.011578263714909554, + 0.04288385063409805, + 0.01748061738908291, + 0.010024511255323887, + 0.029632017016410828, + 0.015783263370394707, + 0.7909825444221497, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.012589594349265099, + 0.0015876422403380275, + 0.0017474794294685125, + 0.0020861185621470213, + 0.0043601891957223415, + 0.0357985682785511, + 0.02455979399383068, + 0.001521193073131144, + 0.009579543955624104, + 0.011912292800843716, + 0.009621202014386654, + 0.8846362829208374, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.008168579079210758, + 0.003204863518476486, + 0.015219288878142834, + 0.003438628278672695, + 0.0027168861124664545, + 0.004375458229333162, + 0.0015034880489110947, + 0.004062159452587366, + 0.012900643050670624, + 0.011661041527986526, + 0.0040309918113052845, + 0.006697934120893478, + 0.9220201969146729, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0005252237897366285, + 0.0031085822265595198, + 0.004390801303088665, + 0.0035702104214578867, + 0.0035593381617218256, + 0.030331918969750404, + 0.4591764509677887, + 0.004388758447021246, + 0.0011947763850912452, + 0.0026341581251472235, + 0.0035157559905201197, + 0.019338253885507584, + 0.0003783679858315736, + 0.46388736367225647, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.024496333673596382, + 0.02895524352788925, + 0.007409717887639999, + 0.3809566795825958, + 0.025879722088575363, + 0.01931842789053917, + 0.015419346280395985, + 0.021070702001452446, + 0.019366836175322533, + 0.024991635233163834, + 0.023570610210299492, + 0.012358313426375389, + 0.007372889667749405, + 0.015129401348531246, + 0.37370413541793823, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.012154011987149715, + 0.006682737730443478, + 0.007665995042771101, + 0.028195885941386223, + 0.011137477122247219, + 0.013211286626756191, + 0.003523231018334627, + 0.02074238285422325, + 0.016506455838680267, + 0.015658063814044, + 0.0018506235210224986, + 0.0026229280047118664, + 0.011027158237993717, + 0.0032886748667806387, + 0.02927999757230282, + 0.8164531588554382, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.02281976491212845, + 0.10939843952655792, + 0.01768209971487522, + 0.0379922017455101, + 0.05958489701151848, + 0.037575434893369675, + 0.05534495413303375, + 0.07548356056213379, + 0.085883229970932, + 0.03505440801382065, + 0.03271886333823204, + 0.06485980749130249, + 0.03420936316251755, + 0.053685955703258514, + 0.03465070575475693, + 0.03298822045326233, + 0.21006812155246735, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007137803360819817, + 0.43834009766578674, + 0.007111180108040571, + 0.02079029753804207, + 0.007220623083412647, + 0.006627264432609081, + 0.01151738129556179, + 0.007246346678584814, + 0.006320110987871885, + 0.005185392685234547, + 0.009528160095214844, + 0.007866409607231617, + 0.00646289624273777, + 0.011828177608549595, + 0.019085045903921127, + 0.00794859230518341, + 0.025521352887153625, + 0.3942629396915436, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.009250459261238575, + 0.031928885728120804, + 0.003431153018027544, + 0.2892793118953705, + 0.011380745097994804, + 0.007772959768772125, + 0.008837981149554253, + 0.0097163962200284, + 0.009380745701491833, + 0.012278328649699688, + 0.016086222603917122, + 0.007512957323342562, + 0.0047597577795386314, + 0.008751295506954193, + 0.2750386893749237, + 0.02218909002840519, + 0.012468640692532063, + 0.02915336936712265, + 0.23078303039073944, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.008872050791978836, + 0.019518502056598663, + 0.003353310748934746, + 0.24201583862304688, + 0.010490399785339832, + 0.006393995601683855, + 0.006850871257483959, + 0.008608594536781311, + 0.004944298416376114, + 0.010344484820961952, + 0.015715332701802254, + 0.004991887602955103, + 0.0037644386757165194, + 0.006783084012567997, + 0.2317233681678772, + 0.01787676475942135, + 0.008699280209839344, + 0.01878373883664608, + 0.1803499013185501, + 0.18991988897323608, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006691065616905689, + 0.01196309458464384, + 0.0037312544882297516, + 0.03853219747543335, + 0.004294651560485363, + 0.018225621432065964, + 0.009463095106184483, + 0.0042219506576657295, + 0.010560271330177784, + 0.009044119156897068, + 0.3927554190158844, + 0.022279053926467896, + 0.009899655357003212, + 0.009482008405029774, + 0.039267648011446, + 0.005388212855905294, + 0.008469431661069393, + 0.012121939100325108, + 0.03598523139953613, + 0.037726208567619324, + 0.30989784002304077, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.005792838055640459, + 0.003233768744394183, + 0.010633621364831924, + 0.0039171804673969746, + 0.002460538176819682, + 0.0030334817711263895, + 0.0013890591217204928, + 0.0034110224805772305, + 0.006884894799441099, + 0.009242535568773746, + 0.004236869513988495, + 0.005316397175192833, + 0.5288781523704529, + 0.001391792669892311, + 0.0040800850838422775, + 0.006198899354785681, + 0.004258950240910053, + 0.0034238765947520733, + 0.004500585608184338, + 0.0042763021774590015, + 0.004853626247495413, + 0.37858548760414124, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.004203260876238346, + 0.002793460851535201, + 0.5633641481399536, + 0.001922506489790976, + 0.0030416715890169144, + 0.004999521188437939, + 0.0027974105905741453, + 0.003516203723847866, + 0.001284533180296421, + 0.002191777341067791, + 0.0018217077013105154, + 0.0010901825735345483, + 0.009667942300438881, + 0.002757111331447959, + 0.0019903217907994986, + 0.0031780314166098833, + 0.00159900460857898, + 0.003045066026970744, + 0.0019833394326269627, + 0.0022355637047439814, + 0.0017322205239906907, + 0.008563856594264507, + 0.37022119760513306, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.5568448305130005, + 0.0028602159582078457, + 0.008501074276864529, + 0.002472684718668461, + 0.003511592745780945, + 0.002653622068464756, + 0.0005315084126777947, + 0.0015206747921183705, + 0.0015745942946523428, + 0.0022094366140663624, + 0.0016722797881811857, + 0.0049180155619978905, + 0.0031623856630176306, + 0.0005430039600469172, + 0.002721830504015088, + 0.0032658791169524193, + 0.002483119023963809, + 0.0031409449875354767, + 0.0026567354798316956, + 0.002672189613804221, + 0.0018375213257968426, + 0.0033643117640167475, + 0.007786066737025976, + 0.3770953416824341, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0024161776527762413, + 0.0036724423989653587, + 0.0013728196499869227, + 0.004530532285571098, + 0.007108110003173351, + 0.004414150957018137, + 0.00348899164237082, + 0.00970153696835041, + 0.5030159950256348, + 0.003252868540585041, + 0.0052609797567129135, + 0.007975067012012005, + 0.01143447682261467, + 0.0035413485020399094, + 0.00464472034946084, + 0.004634521901607513, + 0.008493144065141678, + 0.00388615345582366, + 0.005385011434555054, + 0.004555848427116871, + 0.005924721714109182, + 0.011706513352692127, + 0.0016336682019755244, + 0.0026681125164031982, + 0.3752821385860443, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006361646112054586, + 0.0030678503680974245, + 0.009892310947179794, + 0.0061208149418234825, + 0.0044769528321921825, + 0.4815216064453125, + 0.01844535768032074, + 0.0061722612008452415, + 0.004645282402634621, + 0.010261544957756996, + 0.009851057082414627, + 0.02187376469373703, + 0.0017814149614423513, + 0.018494635820388794, + 0.006336814258247614, + 0.005621344316750765, + 0.005525233689695597, + 0.0028296757955104113, + 0.0059134080074727535, + 0.006024481263011694, + 0.009826972149312496, + 0.0022001052275300026, + 0.009621761739253998, + 0.006355932913720608, + 0.004840776789933443, + 0.3319369852542877, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.000425170233938843, + 0.004094802308827639, + 0.0031548335682600737, + 0.0058814301155507565, + 0.0034635127522051334, + 0.02076704613864422, + 0.32310518622398376, + 0.004715237766504288, + 0.0009895049734041095, + 0.0027204174548387527, + 0.004136086907237768, + 0.01667216047644615, + 0.0004302041488699615, + 0.32536986470222473, + 0.005149839445948601, + 0.001225978252477944, + 0.003524137893691659, + 0.0038776169531047344, + 0.005054942797869444, + 0.004758174065500498, + 0.0035773315466940403, + 0.0005115001695230603, + 0.0031328077893704176, + 0.00047373436973430216, + 0.0012087365612387657, + 0.015949063003063202, + 0.23563075065612793, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0003504737396724522, + 0.0029342903289943933, + 0.0024841963313519955, + 0.004752310924232006, + 0.0029739539604634047, + 0.01688055507838726, + 0.2607404887676239, + 0.004170580301433802, + 0.0009409718331880867, + 0.0024588305968791246, + 0.0034691242035478354, + 0.013757032342255116, + 0.0003214033495169133, + 0.26270395517349243, + 0.004193123895674944, + 0.0010010853875428438, + 0.0028727527242153883, + 0.002769015496596694, + 0.004081565886735916, + 0.003849680069833994, + 0.0029822972137480974, + 0.00038473180029541254, + 0.002470763400197029, + 0.0003893456014338881, + 0.0011314790463075042, + 0.01302724052220583, + 0.18942537903785706, + 0.19248336553573608, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007859006524085999, + 0.0016575781628489494, + 0.0011552342912182212, + 0.002312729600816965, + 0.003259645774960518, + 0.017806673422455788, + 0.01716545969247818, + 0.001212490489706397, + 0.005413018632680178, + 0.00896664708852768, + 0.00777075020596385, + 0.46369069814682007, + 0.00310750724747777, + 0.017114248126745224, + 0.0020787613466382027, + 0.0011019748635590076, + 0.004998395219445229, + 0.001606945414096117, + 0.0024974036496132612, + 0.001989792799577117, + 0.007642480544745922, + 0.003726074006408453, + 0.0012539394665509462, + 0.007231967989355326, + 0.005606942344456911, + 0.015894239768385887, + 0.0155118927359581, + 0.015764642506837845, + 0.35460278391838074, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006357785314321518, + 0.006078907288610935, + 0.0043388856574893, + 0.02823742851614952, + 0.00836637057363987, + 0.007889148779213428, + 0.002819335088133812, + 0.015533188357949257, + 0.009232904762029648, + 0.013993765227496624, + 0.0017917501972988248, + 0.002151534194126725, + 0.0071965414099395275, + 0.0026568507310003042, + 0.028629090636968613, + 0.41004717350006104, + 0.005909791216254234, + 0.006158337462693453, + 0.026087140664458275, + 0.02569008246064186, + 0.0021707997657358646, + 0.006381053477525711, + 0.0049550761468708515, + 0.006111223250627518, + 0.008736412972211838, + 0.00805989932268858, + 0.0031875991262495518, + 0.003034937893971801, + 0.0023294412530958652, + 0.3358675241470337, + 0.0, + 0.0, + 0.0 + ], + [ + 0.013543812558054924, + 0.004420229233801365, + 0.007792192045599222, + 0.005869054701179266, + 0.4126547873020172, + 0.006657087709754705, + 0.008062448352575302, + 0.03802299126982689, + 0.013350061140954494, + 0.03719658404588699, + 0.001816951553337276, + 0.0070841689594089985, + 0.001960590248927474, + 0.0080854007974267, + 0.005973675288259983, + 0.011459790170192719, + 0.014922630973160267, + 0.003987274132668972, + 0.005299489013850689, + 0.006025535985827446, + 0.0016457047313451767, + 0.0024295023176819086, + 0.007024877704679966, + 0.01224868930876255, + 0.0130916191264987, + 0.006404382176697254, + 0.008179416880011559, + 0.008356597274541855, + 0.007054286077618599, + 0.011059602722525597, + 0.30832064151763916, + 0.0, + 0.0 + ], + [ + 0.004401464015245438, + 0.005623731296509504, + 0.011734258383512497, + 0.007039395160973072, + 0.05771010369062424, + 0.010057630948722363, + 0.010448037646710873, + 0.3336871564388275, + 0.010435361415147781, + 0.02165687456727028, + 0.002097832504659891, + 0.0021464028395712376, + 0.006190245505422354, + 0.010710052214562893, + 0.007371100597083569, + 0.027940666303038597, + 0.013196882791817188, + 0.004852048121392727, + 0.006140248384326696, + 0.007695483043789864, + 0.002024211920797825, + 0.006342147011309862, + 0.010344588197767735, + 0.004595689009875059, + 0.00991485733538866, + 0.009901775978505611, + 0.011316884309053421, + 0.01131411176174879, + 0.0023516900837421417, + 0.027719907462596893, + 0.05792964622378349, + 0.2851095199584961, + 0.0 + ], + [ + 0.011347686871886253, + 0.0054034581407904625, + 0.008515670895576477, + 0.028890598565340042, + 0.05831068381667137, + 0.016622332856059074, + 0.009163033217191696, + 0.014324112795293331, + 0.002134349662810564, + 0.22702908515930176, + 0.007055801805108786, + 0.02273165062069893, + 0.030010566115379333, + 0.009357254020869732, + 0.02768339030444622, + 0.019783705472946167, + 0.012823608703911304, + 0.004220066126435995, + 0.02361401356756687, + 0.025145919993519783, + 0.006622320506721735, + 0.028714947402477264, + 0.007441896945238113, + 0.010096932761371136, + 0.002382578095421195, + 0.015767429023981094, + 0.009112813510000706, + 0.009395239874720573, + 0.02087339758872986, + 0.02110687829554081, + 0.04987165331840515, + 0.022752759978175163, + 0.2316940724849701 + ] + ], + "expected": [ + [ + 17, + 1 + ], + [ + 18, + 3 + ], + [ + 18, + 14 + ], + [ + 19, + 3 + ], + [ + 19, + 14 + ], + [ + 20, + 10 + ], + [ + 21, + 12 + ], + [ + 22, + 2 + ], + [ + 23, + 0 + ], + [ + 24, + 8 + ], + [ + 25, + 5 + ], + [ + 26, + 6 + ], + [ + 26, + 13 + ], + [ + 27, + 6 + ], + [ + 27, + 13 + ], + [ + 28, + 11 + ], + [ + 29, + 15 + ], + [ + 30, + 4 + ], + [ + 31, + 7 + ], + [ + 32, + 9 + ] + ] +} diff --git a/default-seed0/analysis/figures/head_ablation.png b/default-seed0/analysis/figures/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..bc36058bd43839ca68968033c301a5469646d0cc Binary files /dev/null and b/default-seed0/analysis/figures/head_ablation.png differ diff --git a/default-seed0/analysis/figures/lookup_current.png b/default-seed0/analysis/figures/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..60e2a64a1c81f6da5c9a4fdb239d733741b76fc4 Binary files /dev/null and b/default-seed0/analysis/figures/lookup_current.png differ diff --git a/default-seed0/analysis/figures/lookup_next.png b/default-seed0/analysis/figures/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..a612ac5f2768ae05339ac8a8f95fb8defe8c3f9a Binary files /dev/null and b/default-seed0/analysis/figures/lookup_next.png differ diff --git a/default-seed0/analysis/figures/lookup_next_pattern.png b/default-seed0/analysis/figures/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..e1dcc7b4993b42b0c55f921bcdfb7fbb0428f50f Binary files /dev/null and b/default-seed0/analysis/figures/lookup_next_pattern.png differ diff --git a/default-seed0/analysis/figures/results.json b/default-seed0/analysis/figures/results.json new file mode 100644 index 0000000000000000000000000000000000000000..6ace8e90c11ac0eac87f6d1ef38f61e9a604c412 --- /dev/null +++ b/default-seed0/analysis/figures/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed0", + "code": { + "commit": "42933bd7a4e2e13fd737d0d48d489f72fbeed8ac", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 4.249109224474523e-06, + "token_accuracy": 1.0, + "exact_match": 1.0 + }, + "untrained": { + "loss": 4.214900970458984, + "token_accuracy": 0.01812744140625, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 4.249109224474523e-06, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.09456062089066108, + 0.006693977446957433, + 2.235693497167631, + 0.15027083228687843, + 0.07599250744442543, + 0.00419125804091891, + 0.10219017933468422, + 5.169036907659574 + ], + [ + 0.00025813918091444066, + 2.973878781631356e-05, + 0.00024059577026491752, + 0.0009068534045582055, + 0.003771953800423944, + 0.0006174362997626304, + 0.000752416023715341, + 0.004184644253655279 + ], + [ + 0.00014806245508225402, + 4.894880476058461e-06, + 2.7085926376457792e-05, + 0.00017150774419860682, + 0.00021039024341007462, + 0.0007992379169081687, + 0.0008647132117403089, + 1.1404487850086298e-05 + ], + [ + 1.613488984730793e-05, + 8.192873792722821e-07, + 1.7713737179292366e-06, + 2.291507462359732e-05, + 0.0001252591091542854, + 5.263272214506287e-06, + 4.800411898031598e-05, + 7.388479389192071e-06 + ] + ], + "token_accuracy_drop": [ + [ + 0.021453857421875, + 0.001556396484375, + 0.322906494140625, + 0.017242431640625, + 0.010345458984375, + 0.00079345703125, + 0.010589599609375, + 0.764434814453125 + ], + [ + 6.103515625e-05, + 3.0517578125e-05, + 9.1552734375e-05, + 0.0003662109375, + 0.001220703125, + 0.00018310546875, + 0.000274658203125, + 0.00128173828125 + ], + [ + 3.0517578125e-05, + 0.0, + 0.0, + 6.103515625e-05, + 0.0001220703125, + 0.000152587890625, + 0.00030517578125, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "layer_loss_rise": [ + 13.57202391098599, + 0.19787378381352028, + 0.04730297024707397, + 0.0025761095375855803 + ], + "layer_token_accuracy_drop": [ + 0.88677978515625, + 0.04681396484375, + 0.016082763671875, + 0.000274658203125 + ] + }, + "lookup_next": { + "score": [ + [ + 0.196746826171875, + 0.322418212890625, + 0.204315185546875, + 0.372955322265625, + 0.4195556640625, + 0.290557861328125, + 0.11737060546875, + 0.136383056640625 + ], + [ + 0.53216552734375, + 0.451263427734375, + 0.471405029296875, + 0.4954833984375, + 0.45703125, + 0.5029296875, + 0.46435546875, + 0.466583251953125 + ], + [ + 0.43756103515625, + 0.59722900390625, + 0.504119873046875, + 0.58270263671875, + 0.451690673828125, + 0.468170166015625, + 0.674285888671875, + 0.673553466796875 + ], + [ + 0.269866943359375, + 0.328369140625, + 0.2083740234375, + 0.380889892578125, + 0.379302978515625, + 0.265625, + 0.322174072265625, + 0.201141357421875 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.58856201171875, + 0.199432373046875, + 0.859954833984375, + 0.282135009765625, + 0.2462158203125, + 0.19976806640625, + 0.03662109375, + 0.988250732421875 + ], + [ + 0.286376953125, + 0.254669189453125, + 0.499359130859375, + 0.36083984375, + 0.21551513671875, + 0.195831298828125, + 0.14984130859375, + 0.171142578125 + ], + [ + 0.3846435546875, + 0.24420166015625, + 0.2579345703125, + 0.267059326171875, + 0.279937744140625, + 0.378326416015625, + 0.229736328125, + 0.234710693359375 + ], + [ + 0.243560791015625, + 0.227264404296875, + 0.204864501953125, + 0.244720458984375, + 0.27691650390625, + 0.246490478515625, + 0.29443359375, + 0.200347900390625 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed0/analysis/figures/source-snapshot.tar.gz b/default-seed0/analysis/figures/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..fb550bbe6173384e8c06163af74b0674aa1bb3bf --- /dev/null +++ b/default-seed0/analysis/figures/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:987265c7625afba46c48ceb93bc7be63100d3e67a00f8fed1385a794487a09ab +size 347464 diff --git a/default-seed0/analysis/final/critical_pattern.png b/default-seed0/analysis/final/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..2bf99d95f050addb9392a8d0361220fa5cfbe7aa Binary files /dev/null and b/default-seed0/analysis/final/critical_pattern.png differ diff --git a/default-seed0/analysis/final/head_ablation.png b/default-seed0/analysis/final/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..bc36058bd43839ca68968033c301a5469646d0cc Binary files /dev/null and b/default-seed0/analysis/final/head_ablation.png differ diff --git a/default-seed0/analysis/final/lookup_current.png b/default-seed0/analysis/final/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..60e2a64a1c81f6da5c9a4fdb239d733741b76fc4 Binary files /dev/null and b/default-seed0/analysis/final/lookup_current.png differ diff --git a/default-seed0/analysis/final/lookup_next.png b/default-seed0/analysis/final/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..a612ac5f2768ae05339ac8a8f95fb8defe8c3f9a Binary files /dev/null and b/default-seed0/analysis/final/lookup_next.png differ diff --git a/default-seed0/analysis/final/lookup_next_pattern.png b/default-seed0/analysis/final/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..e1dcc7b4993b42b0c55f921bcdfb7fbb0428f50f Binary files /dev/null and b/default-seed0/analysis/final/lookup_next_pattern.png differ diff --git a/default-seed0/analysis/final/results.json b/default-seed0/analysis/final/results.json new file mode 100644 index 0000000000000000000000000000000000000000..1de87f2ec8c00c3a5e9d8a36cd87194ce640b61d --- /dev/null +++ b/default-seed0/analysis/final/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed0", + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 4.249109224474523e-06, + "token_accuracy": 1.0, + "exact_match": 1.0 + }, + "untrained": { + "loss": 4.214900970458984, + "token_accuracy": 0.01812744140625, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 4.249109224474523e-06, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.09456062089066108, + 0.006693977446957433, + 2.235693497167631, + 0.15027083228687843, + 0.07599250744442543, + 0.00419125804091891, + 0.10219017933468422, + 5.169036907659574 + ], + [ + 0.00025813918091444066, + 2.973878781631356e-05, + 0.00024059577026491752, + 0.0009068534045582055, + 0.003771953800423944, + 0.0006174362997626304, + 0.000752416023715341, + 0.004184644253655279 + ], + [ + 0.00014806245508225402, + 4.894880476058461e-06, + 2.7085926376457792e-05, + 0.00017150774419860682, + 0.00021039024341007462, + 0.0007992379169081687, + 0.0008647132117403089, + 1.1404487850086298e-05 + ], + [ + 1.613488984730793e-05, + 8.192873792722821e-07, + 1.7713737179292366e-06, + 2.291507462359732e-05, + 0.0001252591091542854, + 5.263272214506287e-06, + 4.800411898031598e-05, + 7.388479389192071e-06 + ] + ], + "token_accuracy_drop": [ + [ + 0.021453857421875, + 0.001556396484375, + 0.322906494140625, + 0.017242431640625, + 0.010345458984375, + 0.00079345703125, + 0.010589599609375, + 0.764434814453125 + ], + [ + 6.103515625e-05, + 3.0517578125e-05, + 9.1552734375e-05, + 0.0003662109375, + 0.001220703125, + 0.00018310546875, + 0.000274658203125, + 0.00128173828125 + ], + [ + 3.0517578125e-05, + 0.0, + 0.0, + 6.103515625e-05, + 0.0001220703125, + 0.000152587890625, + 0.00030517578125, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "layer_loss_rise": [ + 13.57202391098599, + 0.19787378381352028, + 0.04730297024707397, + 0.0025761095375855803 + ], + "layer_token_accuracy_drop": [ + 0.88677978515625, + 0.04681396484375, + 0.016082763671875, + 0.000274658203125 + ] + }, + "lookup_next": { + "score": [ + [ + 0.196746826171875, + 0.322418212890625, + 0.204315185546875, + 0.372955322265625, + 0.4195556640625, + 0.290557861328125, + 0.11737060546875, + 0.136383056640625 + ], + [ + 0.53216552734375, + 0.451263427734375, + 0.471405029296875, + 0.4954833984375, + 0.45703125, + 0.5029296875, + 0.46435546875, + 0.466583251953125 + ], + [ + 0.43756103515625, + 0.59722900390625, + 0.504119873046875, + 0.58270263671875, + 0.451690673828125, + 0.468170166015625, + 0.674285888671875, + 0.673553466796875 + ], + [ + 0.269866943359375, + 0.328369140625, + 0.2083740234375, + 0.380889892578125, + 0.379302978515625, + 0.265625, + 0.322174072265625, + 0.201141357421875 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.58856201171875, + 0.199432373046875, + 0.859954833984375, + 0.282135009765625, + 0.2462158203125, + 0.19976806640625, + 0.03662109375, + 0.988250732421875 + ], + [ + 0.286376953125, + 0.254669189453125, + 0.499359130859375, + 0.36083984375, + 0.21551513671875, + 0.195831298828125, + 0.14984130859375, + 0.171142578125 + ], + [ + 0.3846435546875, + 0.24420166015625, + 0.2579345703125, + 0.267059326171875, + 0.279937744140625, + 0.378326416015625, + 0.229736328125, + 0.234710693359375 + ], + [ + 0.243560791015625, + 0.227264404296875, + 0.204864501953125, + 0.244720458984375, + 0.27691650390625, + 0.246490478515625, + 0.29443359375, + 0.200347900390625 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed0/analysis/final/source-snapshot.tar.gz b/default-seed0/analysis/final/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f57e39642d8127b467d4123afeed8cba0aed9559 --- /dev/null +++ b/default-seed0/analysis/final/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:495dc85a840188753c5988108593ae96365fd22456f9f8e412970d33522e4f36 +size 189601 diff --git a/default-seed0/metrics.json b/default-seed0/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4143a403a4470e8c14d27e1dc868d991eff52e23 --- /dev/null +++ b/default-seed0/metrics.json @@ -0,0 +1,228 @@ +{ + "config": { + "run_name": "default-seed0", + "seed": 0, + "quick": false, + "device": "auto", + "eval_every": 250, + "task": { + "n_values": 64, + "length": 16 + }, + "splits": { + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048 + }, + "model": { + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "optim": { + "steps": 4000, + "batch_size": 256, + "lr": 0.001, + "warmup_steps": 100, + "min_lr_ratio": 0.1, + "weight_decay": 0.1, + "grad_clip": 1.0 + } + }, + "model_config": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "parameters": 3201280, + "data": { + "task": "sorting", + "version": 1, + "n_values": 64, + "length": 16, + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048, + "train_seed": 0 + }, + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "resumable": false, + "status": "finished", + "seconds": 646.7, + "history": [ + { + "step": 250, + "train_loss": 0.07291533797979355, + "val_loss": 0.0985519140958786, + "val_token_accuracy": 0.9683837890625, + "val_exact_match": 0.595703125, + "grad_norm": 2.3497650623321533, + "lr": 0.0009967189933441244 + }, + { + "step": 500, + "train_loss": 0.014207392930984497, + "val_loss": 0.031683966517448425, + "val_token_accuracy": 0.99053955078125, + "val_exact_match": 0.8720703125, + "grad_norm": 0.7331729531288147, + "lr": 0.0009768413988762155 + }, + { + "step": 750, + "train_loss": 0.02052781730890274, + "val_loss": 0.023844445124268532, + "val_token_accuracy": 0.99273681640625, + "val_exact_match": 0.890625, + "grad_norm": 1.3566714525222778, + "lr": 0.0009397114317029974 + }, + { + "step": 1000, + "train_loss": 0.009082821197807789, + "val_loss": 0.01231020875275135, + "val_token_accuracy": 0.996826171875, + "val_exact_match": 0.951171875, + "grad_norm": 0.7233428359031677, + "lr": 0.0008868298366769955 + }, + { + "step": 1250, + "train_loss": 0.00922011211514473, + "val_loss": 0.019146941602230072, + "val_token_accuracy": 0.994873046875, + "val_exact_match": 0.9208984375, + "grad_norm": 0.5756050944328308, + "lr": 0.0008203340189070906 + }, + { + "step": 1500, + "train_loss": 0.01364622451364994, + "val_loss": 0.005576150491833687, + "val_token_accuracy": 0.998779296875, + "val_exact_match": 0.9814453125, + "grad_norm": 1.3124696016311646, + "lr": 0.0007429116526313744 + }, + { + "step": 1750, + "train_loss": 0.003266311716288328, + "val_loss": 0.001853839261457324, + "val_token_accuracy": 0.9996337890625, + "val_exact_match": 0.994140625, + "grad_norm": 0.28147438168525696, + "lr": 0.0006576920489294011 + }, + { + "step": 2000, + "train_loss": 0.005052485503256321, + "val_loss": 0.0006970376125536859, + "val_token_accuracy": 0.9998779296875, + "val_exact_match": 0.998046875, + "grad_norm": 1.1249206066131592, + "lr": 0.0005681196730492369 + }, + { + "step": 2250, + "train_loss": 0.00020874346955679357, + "val_loss": 0.0009217980550602078, + "val_token_accuracy": 0.99981689453125, + "val_exact_match": 0.9970703125, + "grad_norm": 0.03951434791088104, + "lr": 0.00047781492361400794 + }, + { + "step": 2500, + "train_loss": 0.001243826001882553, + "val_loss": 0.0005396516062319279, + "val_token_accuracy": 0.99981689453125, + "val_exact_match": 0.9970703125, + "grad_norm": 0.23733462393283844, + "lr": 0.0003904278008308589 + }, + { + "step": 2750, + "train_loss": 0.00022199669911060482, + "val_loss": 0.00017582655709702522, + "val_token_accuracy": 0.99993896484375, + "val_exact_match": 0.9990234375, + "grad_norm": 0.11565989255905151, + "lr": 0.0003094903782424895 + }, + { + "step": 3000, + "train_loss": 0.00021956569980829954, + "val_loss": 8.435545896645635e-05, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.061819761991500854, + "lr": 0.0002382740409206803 + }, + { + "step": 3250, + "train_loss": 3.6058951081940904e-05, + "val_loss": 8.97766076377593e-05, + "val_token_accuracy": 0.99993896484375, + "val_exact_match": 0.9990234375, + "grad_norm": 0.004151737317442894, + "lr": 0.00017965726034785466 + }, + { + "step": 3500, + "train_loss": 4.75610613648314e-06, + "val_loss": 9.887960004562046e-06, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.0010888022370636463, + "lr": 0.00013600925035352912 + }, + { + "step": 3750, + "train_loss": 0.00010177387594012544, + "val_loss": 0.00023778356262482703, + "val_token_accuracy": 0.99993896484375, + "val_exact_match": 0.9990234375, + "grad_norm": 0.09358467906713486, + "lr": 0.00010909420658097956 + }, + { + "step": 4000, + "train_loss": 1.727779454085976e-05, + "val_loss": 1.847820749389939e-05, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.014979858882725239, + "lr": 0.0001 + } + ], + "final": { + "val_loss": 1.847820749389939e-05, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0 + } +} diff --git a/default-seed0/model.safetensors b/default-seed0/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..69b64ed431ff0b949c0ca999c294c86f3f73e05b --- /dev/null +++ b/default-seed0/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c56f9f38dbed7db80d810f7cbec73980f4303018991b46a0edb1bdf9f962fb68 +size 12811264 diff --git a/default-seed0/model_config.json b/default-seed0/model_config.json new file mode 100644 index 0000000000000000000000000000000000000000..934acf04ad61e2639626dada2f7ad2c2370bd104 --- /dev/null +++ b/default-seed0/model_config.json @@ -0,0 +1,81 @@ +{ + "format_version": 1, + "transformer": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "vocab": [ + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "SEP" + ] +} diff --git a/default-seed0/source-snapshot.tar.gz b/default-seed0/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9cdbf2cae6983280a9a930e30254a87b8eddfc72 --- /dev/null +++ b/default-seed0/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e963742ccde0aca2462ea11e294c6b2504355b7b78eda8738885e210454496e +size 187056 diff --git a/default-seed0/training.png b/default-seed0/training.png new file mode 100644 index 0000000000000000000000000000000000000000..fcf911cd21a655d41040b30a64fed1569f7182bd Binary files /dev/null and b/default-seed0/training.png differ diff --git a/default-seed1/analysis/figures/critical_pattern.png b/default-seed1/analysis/figures/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..c719ea9346411efa06c89f6ee33e2b60db24c7b3 Binary files /dev/null and b/default-seed1/analysis/figures/critical_pattern.png differ diff --git a/default-seed1/analysis/figures/figure-data/curves-default-seed1.csv b/default-seed1/analysis/figures/figure-data/curves-default-seed1.csv new file mode 100644 index 0000000000000000000000000000000000000000..1f6685baa8bbfa210eafdb9b677b2aff93548d9a --- /dev/null +++ b/default-seed1/analysis/figures/figure-data/curves-default-seed1.csv @@ -0,0 +1,17 @@ +step,train_loss,val_loss,val_token_accuracy,val_exact_match +250,0.06796592473983765,0.08403412997722626,0.97271728515625,0.650390625 +500,0.03714346885681152,0.044457193464040756,0.98626708984375,0.8046875 +750,0.03594381362199783,0.028060290962457657,0.99298095703125,0.900390625 +1000,0.008170297369360924,0.012762336991727352,0.9959716796875,0.94140625 +1250,0.006513639353215694,0.018579615280032158,0.99481201171875,0.9208984375 +1500,0.008571112528443336,0.00933537445962429,0.997802734375,0.9658203125 +1750,0.008580565452575684,0.011725695803761482,0.997314453125,0.9580078125 +2000,0.0038308706134557724,0.003425079397857189,0.9991455078125,0.986328125 +2250,0.002150578424334526,0.001471434603445232,0.9996337890625,0.994140625 +2500,0.0023180556017905474,0.0010354401310905814,0.9996337890625,0.994140625 +2750,0.0008889065939001739,0.000499909685458988,0.9998779296875,0.998046875 +3000,0.00015005430032033473,0.00048269086983054876,0.99993896484375,0.9990234375 +3250,0.0008362794178538024,0.000332412077113986,0.99993896484375,0.9990234375 +3500,0.001066122087650001,2.008096817007754e-05,1.0,1.0 +3750,1.979812077479437e-05,1.038504888128955e-05,1.0,1.0 +4000,5.5594000514247455e-06,0.00019783558673225343,0.9998779296875,0.998046875 diff --git a/default-seed1/analysis/figures/figure-data/heads-default-seed1.csv b/default-seed1/analysis/figures/figure-data/heads-default-seed1.csv new file mode 100644 index 0000000000000000000000000000000000000000..33bb7fde4e6bc6f6986af87313b6272853f5f239 --- /dev/null +++ b/default-seed1/analysis/figures/figure-data/heads-default-seed1.csv @@ -0,0 +1,33 @@ +layer,head,loss_rise,token_accuracy_drop,lookup_current,lookup_next +0,0,0.009939101075360668,0.00244140625,0.274139404296875,0.325408935546875 +0,1,5.683765084540937e-05,3.0517578125e-05,0.13470458984375,0.19158935546875 +0,2,0.22735828768236388,0.02294921875,0.109710693359375,0.24566650390625 +0,3,6.165035268773863,0.757965087890625,0.994598388671875,0.129638671875 +0,4,4.92795243189903e-05,3.0517578125e-05,0.22613525390625,0.229583740234375 +0,5,0.07720187168342818,0.01043701171875,0.29803466796875,0.35150146484375 +0,6,0.25589226793272246,0.043701171875,0.48126220703125,0.112030029296875 +0,7,0.04213370989782561,0.006103515625,0.26092529296875,0.3319091796875 +1,0,0.0015396755352412583,0.000518798828125,0.665252685546875,0.3260498046875 +1,1,0.017370990028211963,0.0048828125,0.162628173828125,0.425262451171875 +1,2,0.00043754249236371834,9.1552734375e-05,0.197509765625,0.4627685546875 +1,3,0.000748932261558366,0.000335693359375,0.362091064453125,0.472564697265625 +1,4,0.010392792073616874,0.00274658203125,0.314239501953125,0.45611572265625 +1,5,0.003005323482284439,0.001007080078125,0.163330078125,0.444305419921875 +1,6,0.003752188773432863,0.001312255859375,0.19354248046875,0.487762451171875 +1,7,0.0001520529494882794,3.0517578125e-05,0.289886474609375,0.45831298828125 +2,0,0.0002516775730327936,0.0,0.279388427734375,0.43133544921875 +2,1,8.41566961753415e-05,3.0517578125e-05,0.2215576171875,0.589935302734375 +2,2,0.0002615795692690881,9.1552734375e-05,0.33441162109375,0.513824462890625 +2,3,0.005035566182414186,0.002288818359375,0.267822265625,0.5626220703125 +2,4,0.0005736572366004111,0.00018310546875,0.23614501953125,0.60723876953125 +2,5,0.00016033766041800845,3.0517578125e-05,0.194366455078125,0.51190185546875 +2,6,0.0008616791619715514,0.000244140625,0.251312255859375,0.536346435546875 +2,7,0.0014845907153357984,0.0006103515625,0.311798095703125,0.489349365234375 +3,0,8.289516699733213e-06,0.0,0.230010986328125,0.354034423828125 +3,1,0.0002178412123612361,0.0,0.256103515625,0.410614013671875 +3,2,5.374424290494062e-06,0.0,0.21990966796875,0.4019775390625 +3,3,-1.4844681572867557e-07,0.0,0.166107177734375,0.220245361328125 +3,4,-1.2055061233695596e-06,0.0,0.149200439453125,0.181854248046875 +3,5,-4.305347829358652e-07,0.0,0.243621826171875,0.370697021484375 +3,6,3.061305505980272e-06,0.0,0.219970703125,0.2957763671875 +3,7,3.428203854127787e-07,0.0,0.25,0.246856689453125 diff --git a/default-seed1/analysis/figures/figure-data/layers-default-seed1.csv b/default-seed1/analysis/figures/figure-data/layers-default-seed1.csv new file mode 100644 index 0000000000000000000000000000000000000000..7db751e92f7e1b8b6004917fa705af37b8c00db3 --- /dev/null +++ b/default-seed1/analysis/figures/figure-data/layers-default-seed1.csv @@ -0,0 +1,5 @@ +layer,loss_rise,token_accuracy_drop +0,7.488848230352232,0.88482666015625 +1,0.20484483849031676,0.056793212890625 +2,0.17635237287504424,0.0625 +3,0.004784376842508209,0.001190185546875 diff --git a/default-seed1/analysis/figures/figure-data/pattern-default-seed1.json b/default-seed1/analysis/figures/figure-data/pattern-default-seed1.json new file mode 100644 index 0000000000000000000000000000000000000000..8aa63c16091a4dc0785121597e5d11629e339007 --- /dev/null +++ b/default-seed1/analysis/figures/figure-data/pattern-default-seed1.json @@ -0,0 +1,1279 @@ +{ + "layer": 0, + "head": 3, + "tokens": [ + "21", + "2", + "15", + "4", + "51", + "35", + "37", + "52", + "31", + "62", + "5", + "38", + "9", + "37", + "4", + "45", + "SEP", + "2", + "4", + "4", + "5", + "9", + "15", + "21", + "31", + "35", + "37", + "37", + "38", + "45", + "51", + "52", + "62" + ], + "input_length": 16, + "pattern": [ + [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0056287674233317375, + 0.994371235370636, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0337347686290741, + 0.003753614379093051, + 0.9625115990638733, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.009305569343268871, + 0.046217598021030426, + 0.00931719783693552, + 0.9351596236228943, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0029410384595394135, + 0.026337452232837677, + 0.0046421801671385765, + 0.019023653119802475, + 0.9470557570457458, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006296431180089712, + 0.0067995390854775906, + 0.0024467534385621548, + 0.010534998029470444, + 0.008756743744015694, + 0.9651655554771423, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00730488495901227, + 0.003240293124690652, + 0.022617008537054062, + 0.0014084362192079425, + 0.002127093030139804, + 0.018666155636310577, + 0.9446361660957336, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.029554974287748337, + 0.002137552248314023, + 0.008525188080966473, + 0.0029284912161529064, + 0.023835990577936172, + 0.00267363921739161, + 0.0034024526830762625, + 0.9269417524337769, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0016302927397191525, + 0.0036944225430488586, + 0.007505543064326048, + 0.0017552506178617477, + 0.00807818491011858, + 0.017201170325279236, + 0.00792906153947115, + 0.000540276407264173, + 0.9516658186912537, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.032543014734983444, + 0.003915418870747089, + 0.026386000216007233, + 0.048863545060157776, + 0.02208738960325718, + 0.012656260281801224, + 0.011211938224732876, + 0.010928686708211899, + 0.03544507548213005, + 0.7959626913070679, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.020697344094514847, + 0.01915159448981285, + 0.004889092408120632, + 0.055012255907058716, + 0.007652617059648037, + 0.01024616975337267, + 0.01014277059584856, + 0.0414687804877758, + 0.003158681793138385, + 0.007078006397932768, + 0.8205026984214783, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0038216663524508476, + 0.005144505295902491, + 0.01843036711215973, + 0.0011346811661496758, + 0.0014980246778577566, + 0.016808388754725456, + 0.0513039268553257, + 0.00504545820876956, + 0.004138512536883354, + 0.0010691492352634668, + 0.0025283447466790676, + 0.8890769481658936, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.014788861386477947, + 0.016526665538549423, + 0.006457935087382793, + 0.00567302480340004, + 0.030961129814386368, + 0.0375581756234169, + 0.008722063153982162, + 0.0053113168105483055, + 0.02551380544900894, + 0.013848965056240559, + 0.010869098827242851, + 0.032610658556222916, + 0.7911583185195923, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.003465716727077961, + 0.0013363217003643513, + 0.01127171516418457, + 0.000512742146383971, + 0.001051116967573762, + 0.008836048655211926, + 0.463044673204422, + 0.002462194301187992, + 0.007184683345258236, + 0.0022532211150974035, + 0.0023318100720643997, + 0.024335172027349472, + 0.0014328984543681145, + 0.4704817235469818, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.005401943810284138, + 0.018947286531329155, + 0.0065261246636509895, + 0.41978543996810913, + 0.01645742543041706, + 0.017205355688929558, + 0.0026961721014231443, + 0.006079786457121372, + 0.01181725412607193, + 0.029080737382173538, + 0.023446323350071907, + 0.0040831658989191055, + 0.00655824039131403, + 0.00259238388389349, + 0.42932233214378357, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.012805987149477005, + 0.013161402195692062, + 0.011426457203924656, + 0.008020521141588688, + 0.007964019663631916, + 0.014548967592418194, + 0.00676193181425333, + 0.012627669610083103, + 0.004576542880386114, + 0.007411624304950237, + 0.011817420832812786, + 0.00585333164781332, + 0.00526558980345726, + 0.006768215447664261, + 0.007559095975011587, + 0.8634310960769653, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.016494469717144966, + 0.026330171152949333, + 0.01767500676214695, + 0.024986015632748604, + 0.055334217846393585, + 0.025864768773317337, + 0.057951100170612335, + 0.07052712142467499, + 0.05733547732234001, + 0.08134312927722931, + 0.022683078423142433, + 0.03443731367588043, + 0.010515427216887474, + 0.060338545590639114, + 0.025953534990549088, + 0.126624196767807, + 0.28560641407966614, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.004350163042545319, + 0.5013648867607117, + 0.00400053383782506, + 0.01300878170877695, + 0.0232920590788126, + 0.008887429721653461, + 0.004573792684823275, + 0.004271232057362795, + 0.012335696257650852, + 0.004431537352502346, + 0.008035087957978249, + 0.008351154625415802, + 0.01756184734404087, + 0.004471444524824619, + 0.011388544924557209, + 0.012922966852784157, + 0.008924380876123905, + 0.34782838821411133, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.004142140503972769, + 0.016624469310045242, + 0.0034141079522669315, + 0.31197237968444824, + 0.012067227624356747, + 0.01185158547013998, + 0.0018036093097180128, + 0.004930904600769281, + 0.0060422481037676334, + 0.013310776092112064, + 0.0245559923350811, + 0.0026703388430178165, + 0.006114610470831394, + 0.001724808244034648, + 0.29331013560295105, + 0.011686134152114391, + 0.007224114146083593, + 0.015601199120283127, + 0.25095316767692566, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0037247471045702696, + 0.016174985095858574, + 0.0034561112988740206, + 0.2548469305038452, + 0.011354505084455013, + 0.00969721656292677, + 0.0014788563130423427, + 0.0046325745061039925, + 0.005288855172693729, + 0.011365160346031189, + 0.018620217218995094, + 0.0024125350173562765, + 0.005841365549713373, + 0.0014116688398644328, + 0.2372138947248459, + 0.008001001551747322, + 0.006045340560376644, + 0.015324318781495094, + 0.20039218664169312, + 0.18271753191947937, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.01079960074275732, + 0.011146274395287037, + 0.0022937306202948093, + 0.0337345227599144, + 0.004673227667808533, + 0.006295668426901102, + 0.005560890305787325, + 0.021529361605644226, + 0.0016887009842321277, + 0.00373083702288568, + 0.42941388487815857, + 0.0041452618315815926, + 0.008445270359516144, + 0.0054857381619513035, + 0.03315233439207077, + 0.017140835523605347, + 0.006511059124022722, + 0.010242325253784657, + 0.03184131160378456, + 0.026989096775650978, + 0.3251800835132599, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.012195952236652374, + 0.01541457325220108, + 0.004169343970716, + 0.005737371277064085, + 0.02516764961183071, + 0.021368741989135742, + 0.0050116535276174545, + 0.004626625683158636, + 0.015415117144584656, + 0.009670281782746315, + 0.008729609660804272, + 0.016900869086384773, + 0.47324156761169434, + 0.004930323921144009, + 0.0050043268129229546, + 0.008104763925075531, + 0.008036436513066292, + 0.016462786123156548, + 0.005592823028564453, + 0.005777249112725258, + 0.00816836953163147, + 0.32027363777160645, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.03327929228544235, + 0.003866764483973384, + 0.5242980122566223, + 0.003138359636068344, + 0.005307186394929886, + 0.0022026854567229748, + 0.015319602563977242, + 0.014236025512218475, + 0.012412027455866337, + 0.007968375459313393, + 0.0025641906540840864, + 0.020395701751112938, + 0.002656317315995693, + 0.01604440249502659, + 0.003501161700114608, + 0.008572499267756939, + 0.004963376093655825, + 0.003821315709501505, + 0.0032031801529228687, + 0.003379451809450984, + 0.002596514532342553, + 0.0029120221734046936, + 0.3033614754676819, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.558424174785614, + 0.0032870760187506676, + 0.017560983076691628, + 0.002726122969761491, + 0.0015964414924383163, + 0.002364212181419134, + 0.0028389273211359978, + 0.016513871029019356, + 0.0018205271335318685, + 0.004956931807100773, + 0.0098385289311409, + 0.002564018592238426, + 0.0061355289071798325, + 0.0028477362357079983, + 0.00282105035148561, + 0.006238655187189579, + 0.0025214713532477617, + 0.003119264729321003, + 0.002728755120187998, + 0.002714051166549325, + 0.008698313497006893, + 0.006394281983375549, + 0.013485396280884743, + 0.31780368089675903, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.001357833738438785, + 0.0038780684117227793, + 0.005161534994840622, + 0.0018303628312423825, + 0.008151231333613396, + 0.01706889271736145, + 0.007357828784734011, + 0.0006390373455360532, + 0.5401540994644165, + 0.004446706734597683, + 0.0006896582781337202, + 0.003252560505643487, + 0.005721498746424913, + 0.007361728232353926, + 0.0018193587893620133, + 0.002798643661662936, + 0.0036879985127598047, + 0.004257186781615019, + 0.0017483910778537393, + 0.0018491669325158, + 0.0007520219660364091, + 0.005122872069478035, + 0.005116137210279703, + 0.0016101087676361203, + 0.36416706442832947, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.005138115491718054, + 0.006377323530614376, + 0.0021314136683940887, + 0.00988619402050972, + 0.007929463870823383, + 0.5124956965446472, + 0.011656934395432472, + 0.00480135390534997, + 0.01403276901692152, + 0.003970280289649963, + 0.0036652283743023872, + 0.013282215222716331, + 0.011281702667474747, + 0.010792362503707409, + 0.0080588823184371, + 0.010196010582149029, + 0.0035960422828793526, + 0.006226282566785812, + 0.008744598366320133, + 0.00802096351981163, + 0.0038902314845472574, + 0.010061136446893215, + 0.0024058816488832235, + 0.005201243795454502, + 0.014590348117053509, + 0.3015674352645874, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.003743792185559869, + 0.001907623023726046, + 0.008844357915222645, + 0.00082426454173401, + 0.001524312887340784, + 0.01078791357576847, + 0.3340506851673126, + 0.003355358261615038, + 0.006870136596262455, + 0.002977126743644476, + 0.0025026449002325535, + 0.021317970007658005, + 0.0018526981584727764, + 0.3373992145061493, + 0.0007903872756287456, + 0.0027546691708266735, + 0.003685057396069169, + 0.001800690544769168, + 0.0009049717918969691, + 0.0008664859342388809, + 0.0025867330841720104, + 0.0017302741762250662, + 0.008128976449370384, + 0.003793260781094432, + 0.006667313165962696, + 0.008749839849770069, + 0.21958324313163757, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0031250757165253162, + 0.0014282881747931242, + 0.008465477265417576, + 0.0006142972852103412, + 0.0012052106903865933, + 0.007640608120709658, + 0.2727498710155487, + 0.0029759255703538656, + 0.005529684480279684, + 0.0025021929759532213, + 0.0020051158498972654, + 0.01989907957613468, + 0.0014877136563882232, + 0.27548494935035706, + 0.000595156685449183, + 0.0023828695993870497, + 0.003073555650189519, + 0.0013416430447250605, + 0.0006769761675968766, + 0.0006438216078095138, + 0.002058266429230571, + 0.0013912650756537914, + 0.0075393300503492355, + 0.0031569055281579494, + 0.005308047402650118, + 0.006292236037552357, + 0.17703121900558472, + 0.18339522182941437, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.003358415560796857, + 0.005256603006273508, + 0.01290899608284235, + 0.0015022206353023648, + 0.0019044167129322886, + 0.01627197302877903, + 0.03569798171520233, + 0.005330842919647694, + 0.004044719040393829, + 0.0015259361825883389, + 0.002250880701467395, + 0.45522555708885193, + 0.0064188153482973576, + 0.036091700196266174, + 0.0014627263881266117, + 0.0028452491387724876, + 0.003286269959062338, + 0.0049146125093102455, + 0.0016189708840101957, + 0.0016796275740489364, + 0.0024636464659124613, + 0.0057686069048941135, + 0.011038044467568398, + 0.0036903643049299717, + 0.004285763017833233, + 0.013809888623654842, + 0.03151492029428482, + 0.033759891986846924, + 0.29007235169410706, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.008303455077111721, + 0.009926557540893555, + 0.00761584984138608, + 0.006386931054294109, + 0.006312921643257141, + 0.009911159984767437, + 0.005230232607573271, + 0.010570872575044632, + 0.0035870082210749388, + 0.006392893847078085, + 0.008192788809537888, + 0.004686950705945492, + 0.003758795792236924, + 0.005263189319521189, + 0.006001153960824013, + 0.4830405116081238, + 0.00879328977316618, + 0.008780372329056263, + 0.006860379595309496, + 0.005471652839332819, + 0.007860271260142326, + 0.003956700675189495, + 0.007576397620141506, + 0.007419833913445473, + 0.004187455400824547, + 0.00912896916270256, + 0.0050418260507285595, + 0.0052025155164301395, + 0.004729395266622305, + 0.3298097252845764, + 0.0, + 0.0, + 0.0 + ], + [ + 0.002535423031076789, + 0.017202870920300484, + 0.0040954481810331345, + 0.011674498207867146, + 0.48751765489578247, + 0.008554724045097828, + 0.0018213633447885513, + 0.01984923705458641, + 0.014986270107328892, + 0.011790622025728226, + 0.002596684033051133, + 0.0014828713610768318, + 0.011266556568443775, + 0.0018179143080487847, + 0.011096468195319176, + 0.007654431741684675, + 0.00985523872077465, + 0.01614738442003727, + 0.010381334461271763, + 0.010833223350346088, + 0.002641882048919797, + 0.011476932093501091, + 0.004257702734321356, + 0.002431617584079504, + 0.014161685481667519, + 0.00837952084839344, + 0.002142657758668065, + 0.0019652554765343666, + 0.001792567316442728, + 0.006788833532482386, + 0.2808011770248413, + 0.0, + 0.0 + ], + [ + 0.02847876399755478, + 0.002269891556352377, + 0.01325911283493042, + 0.0062932102009654045, + 0.04032102972269058, + 0.005264019127935171, + 0.005873104091733694, + 0.4263416528701782, + 0.0018110958626493812, + 0.011429733596742153, + 0.013090435415506363, + 0.005403920542448759, + 0.00482679158449173, + 0.005964090581983328, + 0.0064321476966142654, + 0.012229817919433117, + 0.016589293256402016, + 0.0027871341444551945, + 0.006442916113883257, + 0.00659376522526145, + 0.012403004802763462, + 0.005699170287698507, + 0.012190915644168854, + 0.023476582020521164, + 0.0019740986172109842, + 0.005311937537044287, + 0.006386847700923681, + 0.0067416587844491005, + 0.005594385787844658, + 0.012196172028779984, + 0.03210291266441345, + 0.2542203664779663, + 0.0 + ], + [ + 0.0193486325442791, + 0.0027795261703431606, + 0.012100462801754475, + 0.023452261462807655, + 0.012927545234560966, + 0.006814536638557911, + 0.006893753074109554, + 0.005798648111522198, + 0.020403049886226654, + 0.35817620158195496, + 0.0025705862790346146, + 0.0016229369211941957, + 0.008698305115103722, + 0.006920742802321911, + 0.025664983317255974, + 0.010590286925435066, + 0.02775728330016136, + 0.0028971214778721333, + 0.02300073578953743, + 0.0217321515083313, + 0.0028180184308439493, + 0.00929475761950016, + 0.011649117805063725, + 0.016284432262182236, + 0.018083419650793076, + 0.008077215403318405, + 0.007342285476624966, + 0.007146804593503475, + 0.0019610587041825056, + 0.010746720246970654, + 0.013193007558584213, + 0.009442578069865704, + 0.28381088376045227 + ] + ], + "expected": [ + [ + 17, + 1 + ], + [ + 18, + 3 + ], + [ + 18, + 14 + ], + [ + 19, + 3 + ], + [ + 19, + 14 + ], + [ + 20, + 10 + ], + [ + 21, + 12 + ], + [ + 22, + 2 + ], + [ + 23, + 0 + ], + [ + 24, + 8 + ], + [ + 25, + 5 + ], + [ + 26, + 6 + ], + [ + 26, + 13 + ], + [ + 27, + 6 + ], + [ + 27, + 13 + ], + [ + 28, + 11 + ], + [ + 29, + 15 + ], + [ + 30, + 4 + ], + [ + 31, + 7 + ], + [ + 32, + 9 + ] + ] +} diff --git a/default-seed1/analysis/figures/head_ablation.png b/default-seed1/analysis/figures/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..c3bd4925d86dee92f0ac9c8cd44a0cd9880e8fe0 Binary files /dev/null and b/default-seed1/analysis/figures/head_ablation.png differ diff --git a/default-seed1/analysis/figures/lookup_current.png b/default-seed1/analysis/figures/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..d5bf92637aeee916aaaa39c7ec42734f2f5bd824 Binary files /dev/null and b/default-seed1/analysis/figures/lookup_current.png differ diff --git a/default-seed1/analysis/figures/lookup_next.png b/default-seed1/analysis/figures/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..53c583e6ce3fc9abf239a36c0217ffd3221a9b10 Binary files /dev/null and b/default-seed1/analysis/figures/lookup_next.png differ diff --git a/default-seed1/analysis/figures/lookup_next_pattern.png b/default-seed1/analysis/figures/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..90efc8b3bdeedaf15902a21e25c9d33cbee60d0b Binary files /dev/null and b/default-seed1/analysis/figures/lookup_next_pattern.png differ diff --git a/default-seed1/analysis/figures/results.json b/default-seed1/analysis/figures/results.json new file mode 100644 index 0000000000000000000000000000000000000000..4b2054672155865f50cb2412b07971bf44cb6435 --- /dev/null +++ b/default-seed1/analysis/figures/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed1", + "code": { + "commit": "42933bd7a4e2e13fd737d0d48d489f72fbeed8ac", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 1.1899957826244645e-05, + "token_accuracy": 1.0, + "exact_match": 1.0 + }, + "untrained": { + "loss": 4.231888294219971, + "token_accuracy": 0.0140380859375, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 1.1899957826244645e-05, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.009939101075360668, + 5.683765084540937e-05, + 0.22735828768236388, + 6.165035268773863, + 4.92795243189903e-05, + 0.07720187168342818, + 0.25589226793272246, + 0.04213370989782561 + ], + [ + 0.0015396755352412583, + 0.017370990028211963, + 0.00043754249236371834, + 0.000748932261558366, + 0.010392792073616874, + 0.003005323482284439, + 0.003752188773432863, + 0.0001520529494882794 + ], + [ + 0.0002516775730327936, + 8.41566961753415e-05, + 0.0002615795692690881, + 0.005035566182414186, + 0.0005736572366004111, + 0.00016033766041800845, + 0.0008616791619715514, + 0.0014845907153357984 + ], + [ + 8.289516699733213e-06, + 0.0002178412123612361, + 5.374424290494062e-06, + -1.4844681572867557e-07, + -1.2055061233695596e-06, + -4.305347829358652e-07, + 3.061305505980272e-06, + 3.428203854127787e-07 + ] + ], + "token_accuracy_drop": [ + [ + 0.00244140625, + 3.0517578125e-05, + 0.02294921875, + 0.757965087890625, + 3.0517578125e-05, + 0.01043701171875, + 0.043701171875, + 0.006103515625 + ], + [ + 0.000518798828125, + 0.0048828125, + 9.1552734375e-05, + 0.000335693359375, + 0.00274658203125, + 0.001007080078125, + 0.001312255859375, + 3.0517578125e-05 + ], + [ + 0.0, + 3.0517578125e-05, + 9.1552734375e-05, + 0.002288818359375, + 0.00018310546875, + 3.0517578125e-05, + 0.000244140625, + 0.0006103515625 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "layer_loss_rise": [ + 7.488848230352232, + 0.20484483849031676, + 0.17635237287504424, + 0.004784376842508209 + ], + "layer_token_accuracy_drop": [ + 0.88482666015625, + 0.056793212890625, + 0.0625, + 0.001190185546875 + ] + }, + "lookup_next": { + "score": [ + [ + 0.325408935546875, + 0.19158935546875, + 0.24566650390625, + 0.129638671875, + 0.229583740234375, + 0.35150146484375, + 0.112030029296875, + 0.3319091796875 + ], + [ + 0.3260498046875, + 0.425262451171875, + 0.4627685546875, + 0.472564697265625, + 0.45611572265625, + 0.444305419921875, + 0.487762451171875, + 0.45831298828125 + ], + [ + 0.43133544921875, + 0.589935302734375, + 0.513824462890625, + 0.5626220703125, + 0.60723876953125, + 0.51190185546875, + 0.536346435546875, + 0.489349365234375 + ], + [ + 0.354034423828125, + 0.410614013671875, + 0.4019775390625, + 0.220245361328125, + 0.181854248046875, + 0.370697021484375, + 0.2957763671875, + 0.246856689453125 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.274139404296875, + 0.13470458984375, + 0.109710693359375, + 0.994598388671875, + 0.22613525390625, + 0.29803466796875, + 0.48126220703125, + 0.26092529296875 + ], + [ + 0.665252685546875, + 0.162628173828125, + 0.197509765625, + 0.362091064453125, + 0.314239501953125, + 0.163330078125, + 0.19354248046875, + 0.289886474609375 + ], + [ + 0.279388427734375, + 0.2215576171875, + 0.33441162109375, + 0.267822265625, + 0.23614501953125, + 0.194366455078125, + 0.251312255859375, + 0.311798095703125 + ], + [ + 0.230010986328125, + 0.256103515625, + 0.21990966796875, + 0.166107177734375, + 0.149200439453125, + 0.243621826171875, + 0.219970703125, + 0.25 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed1/analysis/figures/source-snapshot.tar.gz b/default-seed1/analysis/figures/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..6c730fd89e6b03af8311fe823df9c42e1c63ef06 --- /dev/null +++ b/default-seed1/analysis/figures/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b33f10047810fe6d53b996f5128c29481cfa337cba04f007cfe8a49edce5bcf +size 347464 diff --git a/default-seed1/analysis/final/critical_pattern.png b/default-seed1/analysis/final/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..c719ea9346411efa06c89f6ee33e2b60db24c7b3 Binary files /dev/null and b/default-seed1/analysis/final/critical_pattern.png differ diff --git a/default-seed1/analysis/final/head_ablation.png b/default-seed1/analysis/final/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..c3bd4925d86dee92f0ac9c8cd44a0cd9880e8fe0 Binary files /dev/null and b/default-seed1/analysis/final/head_ablation.png differ diff --git a/default-seed1/analysis/final/lookup_current.png b/default-seed1/analysis/final/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..d5bf92637aeee916aaaa39c7ec42734f2f5bd824 Binary files /dev/null and b/default-seed1/analysis/final/lookup_current.png differ diff --git a/default-seed1/analysis/final/lookup_next.png b/default-seed1/analysis/final/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..53c583e6ce3fc9abf239a36c0217ffd3221a9b10 Binary files /dev/null and b/default-seed1/analysis/final/lookup_next.png differ diff --git a/default-seed1/analysis/final/lookup_next_pattern.png b/default-seed1/analysis/final/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..90efc8b3bdeedaf15902a21e25c9d33cbee60d0b Binary files /dev/null and b/default-seed1/analysis/final/lookup_next_pattern.png differ diff --git a/default-seed1/analysis/final/results.json b/default-seed1/analysis/final/results.json new file mode 100644 index 0000000000000000000000000000000000000000..9c1cedf020caf34fb08d62a314504ca2ae2bc91d --- /dev/null +++ b/default-seed1/analysis/final/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed1", + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 1.1899957826244645e-05, + "token_accuracy": 1.0, + "exact_match": 1.0 + }, + "untrained": { + "loss": 4.231888294219971, + "token_accuracy": 0.0140380859375, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 1.1899957826244645e-05, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.009939101075360668, + 5.683765084540937e-05, + 0.22735828768236388, + 6.165035268773863, + 4.92795243189903e-05, + 0.07720187168342818, + 0.25589226793272246, + 0.04213370989782561 + ], + [ + 0.0015396755352412583, + 0.017370990028211963, + 0.00043754249236371834, + 0.000748932261558366, + 0.010392792073616874, + 0.003005323482284439, + 0.003752188773432863, + 0.0001520529494882794 + ], + [ + 0.0002516775730327936, + 8.41566961753415e-05, + 0.0002615795692690881, + 0.005035566182414186, + 0.0005736572366004111, + 0.00016033766041800845, + 0.0008616791619715514, + 0.0014845907153357984 + ], + [ + 8.289516699733213e-06, + 0.0002178412123612361, + 5.374424290494062e-06, + -1.4844681572867557e-07, + -1.2055061233695596e-06, + -4.305347829358652e-07, + 3.061305505980272e-06, + 3.428203854127787e-07 + ] + ], + "token_accuracy_drop": [ + [ + 0.00244140625, + 3.0517578125e-05, + 0.02294921875, + 0.757965087890625, + 3.0517578125e-05, + 0.01043701171875, + 0.043701171875, + 0.006103515625 + ], + [ + 0.000518798828125, + 0.0048828125, + 9.1552734375e-05, + 0.000335693359375, + 0.00274658203125, + 0.001007080078125, + 0.001312255859375, + 3.0517578125e-05 + ], + [ + 0.0, + 3.0517578125e-05, + 9.1552734375e-05, + 0.002288818359375, + 0.00018310546875, + 3.0517578125e-05, + 0.000244140625, + 0.0006103515625 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "layer_loss_rise": [ + 7.488848230352232, + 0.20484483849031676, + 0.17635237287504424, + 0.004784376842508209 + ], + "layer_token_accuracy_drop": [ + 0.88482666015625, + 0.056793212890625, + 0.0625, + 0.001190185546875 + ] + }, + "lookup_next": { + "score": [ + [ + 0.325408935546875, + 0.19158935546875, + 0.24566650390625, + 0.129638671875, + 0.229583740234375, + 0.35150146484375, + 0.112030029296875, + 0.3319091796875 + ], + [ + 0.3260498046875, + 0.425262451171875, + 0.4627685546875, + 0.472564697265625, + 0.45611572265625, + 0.444305419921875, + 0.487762451171875, + 0.45831298828125 + ], + [ + 0.43133544921875, + 0.589935302734375, + 0.513824462890625, + 0.5626220703125, + 0.60723876953125, + 0.51190185546875, + 0.536346435546875, + 0.489349365234375 + ], + [ + 0.354034423828125, + 0.410614013671875, + 0.4019775390625, + 0.220245361328125, + 0.181854248046875, + 0.370697021484375, + 0.2957763671875, + 0.246856689453125 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.274139404296875, + 0.13470458984375, + 0.109710693359375, + 0.994598388671875, + 0.22613525390625, + 0.29803466796875, + 0.48126220703125, + 0.26092529296875 + ], + [ + 0.665252685546875, + 0.162628173828125, + 0.197509765625, + 0.362091064453125, + 0.314239501953125, + 0.163330078125, + 0.19354248046875, + 0.289886474609375 + ], + [ + 0.279388427734375, + 0.2215576171875, + 0.33441162109375, + 0.267822265625, + 0.23614501953125, + 0.194366455078125, + 0.251312255859375, + 0.311798095703125 + ], + [ + 0.230010986328125, + 0.256103515625, + 0.21990966796875, + 0.166107177734375, + 0.149200439453125, + 0.243621826171875, + 0.219970703125, + 0.25 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed1/analysis/final/source-snapshot.tar.gz b/default-seed1/analysis/final/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..e009c27b893b12e2319ecefa3c5f124fddd056fe --- /dev/null +++ b/default-seed1/analysis/final/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8565b984d75e3509db6cd71d4f35e8096085579a3585ad3698120f9ff78eac1f +size 189601 diff --git a/default-seed1/metrics.json b/default-seed1/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d1dbcab7d3ccccf066493bd37eabd4c339348edb --- /dev/null +++ b/default-seed1/metrics.json @@ -0,0 +1,228 @@ +{ + "config": { + "run_name": "default-seed1", + "seed": 1, + "quick": false, + "device": "auto", + "eval_every": 250, + "task": { + "n_values": 64, + "length": 16 + }, + "splits": { + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048 + }, + "model": { + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "optim": { + "steps": 4000, + "batch_size": 256, + "lr": 0.001, + "warmup_steps": 100, + "min_lr_ratio": 0.1, + "weight_decay": 0.1, + "grad_clip": 1.0 + } + }, + "model_config": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "parameters": 3201280, + "data": { + "task": "sorting", + "version": 1, + "n_values": 64, + "length": 16, + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048, + "train_seed": 1 + }, + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "resumable": false, + "status": "finished", + "seconds": 646.6, + "history": [ + { + "step": 250, + "train_loss": 0.06796592473983765, + "val_loss": 0.08403412997722626, + "val_token_accuracy": 0.97271728515625, + "val_exact_match": 0.650390625, + "grad_norm": 2.1865394115448, + "lr": 0.0009967189933441244 + }, + { + "step": 500, + "train_loss": 0.03714346885681152, + "val_loss": 0.044457193464040756, + "val_token_accuracy": 0.98626708984375, + "val_exact_match": 0.8046875, + "grad_norm": 1.7170097827911377, + "lr": 0.0009768413988762155 + }, + { + "step": 750, + "train_loss": 0.03594381362199783, + "val_loss": 0.028060290962457657, + "val_token_accuracy": 0.99298095703125, + "val_exact_match": 0.900390625, + "grad_norm": 2.0198495388031006, + "lr": 0.0009397114317029974 + }, + { + "step": 1000, + "train_loss": 0.008170297369360924, + "val_loss": 0.012762336991727352, + "val_token_accuracy": 0.9959716796875, + "val_exact_match": 0.94140625, + "grad_norm": 0.43693360686302185, + "lr": 0.0008868298366769955 + }, + { + "step": 1250, + "train_loss": 0.006513639353215694, + "val_loss": 0.018579615280032158, + "val_token_accuracy": 0.99481201171875, + "val_exact_match": 0.9208984375, + "grad_norm": 0.49892523884773254, + "lr": 0.0008203340189070906 + }, + { + "step": 1500, + "train_loss": 0.008571112528443336, + "val_loss": 0.00933537445962429, + "val_token_accuracy": 0.997802734375, + "val_exact_match": 0.9658203125, + "grad_norm": 0.7041215896606445, + "lr": 0.0007429116526313744 + }, + { + "step": 1750, + "train_loss": 0.008580565452575684, + "val_loss": 0.011725695803761482, + "val_token_accuracy": 0.997314453125, + "val_exact_match": 0.9580078125, + "grad_norm": 0.5222469568252563, + "lr": 0.0006576920489294011 + }, + { + "step": 2000, + "train_loss": 0.0038308706134557724, + "val_loss": 0.003425079397857189, + "val_token_accuracy": 0.9991455078125, + "val_exact_match": 0.986328125, + "grad_norm": 0.217371866106987, + "lr": 0.0005681196730492369 + }, + { + "step": 2250, + "train_loss": 0.002150578424334526, + "val_loss": 0.001471434603445232, + "val_token_accuracy": 0.9996337890625, + "val_exact_match": 0.994140625, + "grad_norm": 0.4117131531238556, + "lr": 0.00047781492361400794 + }, + { + "step": 2500, + "train_loss": 0.0023180556017905474, + "val_loss": 0.0010354401310905814, + "val_token_accuracy": 0.9996337890625, + "val_exact_match": 0.994140625, + "grad_norm": 0.2440403699874878, + "lr": 0.0003904278008308589 + }, + { + "step": 2750, + "train_loss": 0.0008889065939001739, + "val_loss": 0.000499909685458988, + "val_token_accuracy": 0.9998779296875, + "val_exact_match": 0.998046875, + "grad_norm": 0.18215815722942352, + "lr": 0.0003094903782424895 + }, + { + "step": 3000, + "train_loss": 0.00015005430032033473, + "val_loss": 0.00048269086983054876, + "val_token_accuracy": 0.99993896484375, + "val_exact_match": 0.9990234375, + "grad_norm": 0.08666612952947617, + "lr": 0.0002382740409206803 + }, + { + "step": 3250, + "train_loss": 0.0008362794178538024, + "val_loss": 0.000332412077113986, + "val_token_accuracy": 0.99993896484375, + "val_exact_match": 0.9990234375, + "grad_norm": 0.2885400354862213, + "lr": 0.00017965726034785466 + }, + { + "step": 3500, + "train_loss": 0.001066122087650001, + "val_loss": 2.008096817007754e-05, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.37905609607696533, + "lr": 0.00013600925035352912 + }, + { + "step": 3750, + "train_loss": 1.979812077479437e-05, + "val_loss": 1.038504888128955e-05, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.003974005114287138, + "lr": 0.00010909420658097956 + }, + { + "step": 4000, + "train_loss": 5.5594000514247455e-06, + "val_loss": 0.00019783558673225343, + "val_token_accuracy": 0.9998779296875, + "val_exact_match": 0.998046875, + "grad_norm": 0.0021852694917470217, + "lr": 0.0001 + } + ], + "final": { + "val_loss": 0.00019783558673225343, + "val_token_accuracy": 0.9998779296875, + "val_exact_match": 0.998046875 + } +} diff --git a/default-seed1/model.safetensors b/default-seed1/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c8bb19d61261f787c37f9441cab4015cd83ed1f7 --- /dev/null +++ b/default-seed1/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a7f760741bdf7dd490d4415821addc419543162fda89ed6bc45920bfa8747fa +size 12811264 diff --git a/default-seed1/model_config.json b/default-seed1/model_config.json new file mode 100644 index 0000000000000000000000000000000000000000..934acf04ad61e2639626dada2f7ad2c2370bd104 --- /dev/null +++ b/default-seed1/model_config.json @@ -0,0 +1,81 @@ +{ + "format_version": 1, + "transformer": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "vocab": [ + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "SEP" + ] +} diff --git a/default-seed1/source-snapshot.tar.gz b/default-seed1/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9cdbf2cae6983280a9a930e30254a87b8eddfc72 --- /dev/null +++ b/default-seed1/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e963742ccde0aca2462ea11e294c6b2504355b7b78eda8738885e210454496e +size 187056 diff --git a/default-seed1/training.png b/default-seed1/training.png new file mode 100644 index 0000000000000000000000000000000000000000..b9ca32f3fdd0c3f473ba2cefa315ed52be6a9875 Binary files /dev/null and b/default-seed1/training.png differ diff --git a/default-seed2/analysis/figures/critical_pattern.png b/default-seed2/analysis/figures/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..00c818b90d27e1146d68f23a2edc2e2867a0c1bd Binary files /dev/null and b/default-seed2/analysis/figures/critical_pattern.png differ diff --git a/default-seed2/analysis/figures/figure-data/curves-default-seed2.csv b/default-seed2/analysis/figures/figure-data/curves-default-seed2.csv new file mode 100644 index 0000000000000000000000000000000000000000..4941892d14f2ba79fdbe227950d27befaf716407 --- /dev/null +++ b/default-seed2/analysis/figures/figure-data/curves-default-seed2.csv @@ -0,0 +1,17 @@ +step,train_loss,val_loss,val_token_accuracy,val_exact_match +250,0.08374407142400742,0.08853542804718018,0.97259521484375,0.654296875 +500,0.03046058677136898,0.030035920441150665,0.99041748046875,0.8642578125 +750,0.019605081528425217,0.02953985333442688,0.99114990234375,0.87109375 +1000,0.010449330322444439,0.01985304430127144,0.99554443359375,0.9326171875 +1250,0.004702113103121519,0.008932323195040226,0.9974365234375,0.962890625 +1500,0.00736136082559824,0.004941404797136784,0.9989013671875,0.982421875 +1750,0.009208878502249718,0.005312320776283741,0.998779296875,0.9814453125 +2000,0.0014178939163684845,0.004395550116896629,0.9991455078125,0.986328125 +2250,0.0022070351988077164,0.0005514943040907383,0.99981689453125,0.9970703125 +2500,0.0019115328323096037,0.0011586558539420366,0.99957275390625,0.9931640625 +2750,0.003090714570134878,0.001331779290921986,0.9996337890625,0.994140625 +3000,0.0001012918510241434,0.0001072997329174541,1.0,1.0 +3250,3.054919943679124e-05,0.004428497049957514,0.99945068359375,0.9912109375 +3500,0.00010091940930578858,0.0005630630766972899,0.99981689453125,0.9970703125 +3750,1.1987899597443175e-06,4.915940280625364e-06,1.0,1.0 +4000,4.847031505050836e-06,0.0006610961863771081,0.99969482421875,0.9951171875 diff --git a/default-seed2/analysis/figures/figure-data/heads-default-seed2.csv b/default-seed2/analysis/figures/figure-data/heads-default-seed2.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4a1b3bbdddddbce098182ac3204c51955db5afb --- /dev/null +++ b/default-seed2/analysis/figures/figure-data/heads-default-seed2.csv @@ -0,0 +1,33 @@ +layer,head,loss_rise,token_accuracy_drop,lookup_current,lookup_next +0,0,0.047352743829833344,0.006805419921875,0.06396484375,0.092010498046875 +0,1,1.1891647628799547,0.140716552734375,0.776519775390625,0.19891357421875 +0,2,0.002971459209220484,0.00067138671875,0.22003173828125,0.41748046875 +0,3,6.620310669328319,0.8082275390625,0.9862060546875,0.129241943359375 +0,4,0.001162868255050853,0.000335693359375,0.122772216796875,0.2574462890625 +0,5,0.2693032673851121,0.03826904296875,0.06219482421875,0.121490478515625 +0,6,0.0006633798184338957,0.0001220703125,0.13922119140625,0.259857177734375 +0,7,0.01946811148081906,0.002471923828125,0.129913330078125,0.216217041015625 +1,0,0.012694073404418305,0.0035400390625,0.157501220703125,0.488922119140625 +1,1,0.0065985834517050534,0.001861572265625,0.299163818359375,0.602020263671875 +1,2,0.0006987463275436312,0.000274658203125,0.227294921875,0.51617431640625 +1,3,0.0009007553162518889,0.00030517578125,0.23193359375,0.54144287109375 +1,4,0.0017039284866768867,0.000762939453125,0.230072021484375,0.494171142578125 +1,5,0.0028953883156646043,0.001007080078125,0.2099609375,0.520660400390625 +1,6,0.006903988338308409,0.0018310546875,0.230255126953125,0.5172119140625 +1,7,0.0009580427140463144,0.000457763671875,0.22003173828125,0.479278564453125 +2,0,9.999852045439184e-05,3.0517578125e-05,0.187652587890625,0.54705810546875 +2,1,0.0007847294036764652,0.000152587890625,0.2132568359375,0.614532470703125 +2,2,6.677379133179784e-06,0.0,0.265167236328125,0.51446533203125 +2,3,0.00027560422313399613,0.0,0.1873779296875,0.537261962890625 +2,4,8.595394319854677e-05,0.0,0.205169677734375,0.622100830078125 +2,5,0.0008517569222021848,0.000335693359375,0.32989501953125,0.527313232421875 +2,6,6.360691622830927e-05,3.0517578125e-05,0.305511474609375,0.498138427734375 +2,7,0.0028535542369354516,0.00091552734375,0.290191650390625,0.476837158203125 +3,0,-3.0265393434092402e-05,0.0,0.23785400390625,0.34832763671875 +3,1,1.0538555216044188e-05,0.0,0.22991943359375,0.317596435546875 +3,2,2.137760748155415e-05,0.0,0.28265380859375,0.367431640625 +3,3,4.4142158003523946e-05,0.0,0.23626708984375,0.25860595703125 +3,4,1.2337521184235811e-05,0.0,0.262176513671875,0.39251708984375 +3,5,5.3422118071466684e-06,0.0,0.23712158203125,0.38818359375 +3,6,6.135689909569919e-05,0.0,0.3084716796875,0.323089599609375 +3,7,0.0001602511911187321,6.103515625e-05,0.224029541015625,0.404296875 diff --git a/default-seed2/analysis/figures/figure-data/layers-default-seed2.csv b/default-seed2/analysis/figures/figure-data/layers-default-seed2.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c8e1b28d108bcb88860c244d570d2a4efd7590d --- /dev/null +++ b/default-seed2/analysis/figures/figure-data/layers-default-seed2.csv @@ -0,0 +1,5 @@ +layer,loss_rise,token_accuracy_drop +0,10.483049278642284,0.85162353515625 +1,0.43556219854508527,0.1031494140625 +2,0.09911830493365414,0.03265380859375 +3,0.01326776811038144,0.004241943359375 diff --git a/default-seed2/analysis/figures/figure-data/pattern-default-seed2.json b/default-seed2/analysis/figures/figure-data/pattern-default-seed2.json new file mode 100644 index 0000000000000000000000000000000000000000..fac31de41e692e3a50a3448782951bae1b75cc32 --- /dev/null +++ b/default-seed2/analysis/figures/figure-data/pattern-default-seed2.json @@ -0,0 +1,1279 @@ +{ + "layer": 0, + "head": 3, + "tokens": [ + "21", + "2", + "15", + "4", + "51", + "35", + "37", + "52", + "31", + "62", + "5", + "38", + "9", + "37", + "4", + "45", + "SEP", + "2", + "4", + "4", + "5", + "9", + "15", + "21", + "31", + "35", + "37", + "37", + "38", + "45", + "51", + "52", + "62" + ], + "input_length": 16, + "pattern": [ + [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.02103869803249836, + 0.9789613485336304, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.001584514044225216, + 0.003572966903448105, + 0.9948424696922302, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007175391539931297, + 0.013961078599095345, + 0.025949055328965187, + 0.9529144763946533, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.025887785479426384, + 0.017972348257899284, + 0.009018037468194962, + 0.0031646613497287035, + 0.9439572095870972, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0004146286519244313, + 0.005030798260122538, + 0.030794603750109673, + 0.022877275943756104, + 0.0019939059857279062, + 0.9388887882232666, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.02268129773437977, + 0.0015246859984472394, + 0.00478235213086009, + 0.009695732966065407, + 0.0020760628394782543, + 0.01662675105035305, + 0.9426130652427673, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.019364947453141212, + 0.006758064031600952, + 0.04508545622229576, + 0.0075384750962257385, + 0.07095436006784439, + 0.0015190135454759002, + 0.0012125435750931501, + 0.847567081451416, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.015475370921194553, + 0.0035692693199962378, + 0.012105826288461685, + 0.006824203301221132, + 0.00290181883610785, + 0.01845148392021656, + 0.008350959978997707, + 0.0035924138501286507, + 0.9287286996841431, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00905853882431984, + 0.01204361766576767, + 0.003441830398514867, + 0.06380645185709, + 0.05361092835664749, + 0.022182554006576538, + 0.008810009807348251, + 0.05294596776366234, + 0.013502917252480984, + 0.7605971097946167, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.005306084640324116, + 0.009041777811944485, + 0.010194691829383373, + 0.053342305123806, + 0.011519726365804672, + 0.005287684965878725, + 0.01182020828127861, + 0.012499775737524033, + 0.010906824842095375, + 0.0036131443921476603, + 0.8664677739143372, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.017497682943940163, + 0.006535052787512541, + 0.010278336703777313, + 0.0320083349943161, + 0.0025849149096757174, + 0.027305737137794495, + 0.04664206877350807, + 0.011773577891290188, + 0.002364890882745385, + 0.011457731947302818, + 0.014169340021908283, + 0.8173823356628418, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006653135176748037, + 0.004730554297566414, + 0.004637997597455978, + 0.005912784021347761, + 0.01970953308045864, + 0.008885163813829422, + 0.005337592680007219, + 0.010576101019978523, + 0.0020863066893070936, + 0.01031989511102438, + 0.013760693371295929, + 0.006473017390817404, + 0.9009172320365906, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.010289459489285946, + 0.0006087846704758704, + 0.002147048944607377, + 0.0036837183870375156, + 0.0008307101088576019, + 0.006923981010913849, + 0.4728759229183197, + 0.0005958499386906624, + 0.008628464303910732, + 0.0009574713767506182, + 0.004934208933264017, + 0.017199141904711723, + 0.0022434385027736425, + 0.4680817723274231, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.005042059347033501, + 0.008757284842431545, + 0.014894523657858372, + 0.4312535524368286, + 0.006727681495249271, + 0.026009641587734222, + 0.010145221836864948, + 0.016630904749035835, + 0.010146395303308964, + 0.015836020931601524, + 0.033498410135507584, + 0.02397383190691471, + 0.005827322136610746, + 0.01089954748749733, + 0.38035768270492554, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.013708776794373989, + 0.01167106069624424, + 0.0324547253549099, + 0.0025026588700711727, + 0.014907370321452618, + 0.007911515422165394, + 0.03629322350025177, + 0.011615934781730175, + 0.010640064254403114, + 0.005626977421343327, + 0.0018340093083679676, + 0.007147335913032293, + 0.018066491931676865, + 0.03716324269771576, + 0.0027412953786551952, + 0.7857153415679932, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.03507337346673012, + 0.0689832791686058, + 0.029603952541947365, + 0.03861061483621597, + 0.06801148504018784, + 0.09042447805404663, + 0.03719828277826309, + 0.04033159464597702, + 0.06385558098554611, + 0.12230576574802399, + 0.03527894988656044, + 0.0858309343457222, + 0.032834917306900024, + 0.036878637969493866, + 0.03730073198676109, + 0.024790309369564056, + 0.15268707275390625, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.010278671979904175, + 0.5506013035774231, + 0.013715271838009357, + 0.013280510902404785, + 0.01679512858390808, + 0.007763308472931385, + 0.002985496073961258, + 0.008768368512392044, + 0.005472816992551088, + 0.007424951996654272, + 0.013059385120868683, + 0.005105638410896063, + 0.008714789524674416, + 0.002947686007246375, + 0.01366406213492155, + 0.015779783949255943, + 0.011586377397179604, + 0.29205644130706787, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.004073625430464745, + 0.014004068449139595, + 0.010871119797229767, + 0.3228211998939514, + 0.00499819777905941, + 0.022909782826900482, + 0.0077531407587230206, + 0.011964239180088043, + 0.008592464961111546, + 0.017519231885671616, + 0.03144408017396927, + 0.016104549169540405, + 0.006547039840370417, + 0.007979851216077805, + 0.2903905212879181, + 0.004881939385086298, + 0.010654064826667309, + 0.013888932764530182, + 0.1926019936800003, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0046831597574055195, + 0.009347873739898205, + 0.009188064374029636, + 0.2670532763004303, + 0.004409836605191231, + 0.018783388659358025, + 0.007813134230673313, + 0.00988030806183815, + 0.007928899489343166, + 0.01558869332075119, + 0.029612654820084572, + 0.015393096953630447, + 0.0057807122357189655, + 0.008141319267451763, + 0.2410224974155426, + 0.004781877622008324, + 0.008507851511240005, + 0.009728455916047096, + 0.15753713250160217, + 0.1648177206516266, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.004948741756379604, + 0.008985348977148533, + 0.008357335813343525, + 0.03787755221128464, + 0.009128892794251442, + 0.005946495104581118, + 0.011042079888284206, + 0.008747050538659096, + 0.011738220229744911, + 0.004419118631631136, + 0.453222393989563, + 0.006824468728154898, + 0.00797360297292471, + 0.011817879974842072, + 0.03663260489702225, + 0.003804711392149329, + 0.006030908785760403, + 0.009014645591378212, + 0.031840939074754715, + 0.03208044916391373, + 0.2895664870738983, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.006663736421614885, + 0.003479116829112172, + 0.003861436853185296, + 0.004682034254074097, + 0.013529835268855095, + 0.008267596364021301, + 0.0053383479826152325, + 0.008303462527692318, + 0.0023541320115327835, + 0.010345317423343658, + 0.009789296425879002, + 0.005401006434112787, + 0.5212092995643616, + 0.004956812132149935, + 0.0048385788686573505, + 0.007964313961565495, + 0.005271155387163162, + 0.003708701580762863, + 0.004755365662276745, + 0.004717507399618626, + 0.00840077642351389, + 0.3521621823310852, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0020906527061015368, + 0.004733143839985132, + 0.5558521151542664, + 0.003363186027854681, + 0.003578424919396639, + 0.007614694535732269, + 0.0020315845031291246, + 0.009341670200228691, + 0.004486112389713526, + 0.0014784150989726186, + 0.003662504255771637, + 0.003346491139382124, + 0.0025405841879546642, + 0.0022044198121875525, + 0.003723513102158904, + 0.007745872717350721, + 0.0031650750897824764, + 0.0046553038991987705, + 0.003301603952422738, + 0.0034593960735946894, + 0.003660234622657299, + 0.002854640129953623, + 0.3611104488372803, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.5680680274963379, + 0.004268328659236431, + 0.0016227205051109195, + 0.0012218923075124621, + 0.004104001447558403, + 0.00029271477251313627, + 0.005435158032923937, + 0.0029968987219035625, + 0.005096343345940113, + 0.0014956197701394558, + 0.0018023911397904158, + 0.003372713690623641, + 0.0019160278607159853, + 0.005747911054641008, + 0.0012715408811345696, + 0.0034704923164099455, + 0.0024200475309044123, + 0.004345678724348545, + 0.0014554394874721766, + 0.0015823255525901914, + 0.0017479347297921777, + 0.002062159823253751, + 0.00164604012388736, + 0.37255755066871643, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.010084847919642925, + 0.00283254636451602, + 0.00616444693878293, + 0.004170957952737808, + 0.0019152247114107013, + 0.013429660350084305, + 0.007179662119597197, + 0.002065215026959777, + 0.508695662021637, + 0.0035849965643137693, + 0.005455987993627787, + 0.0018950646044686437, + 0.0008975184755399823, + 0.007647697348147631, + 0.004263062030076981, + 0.004941413179039955, + 0.0045313527807593346, + 0.00340284570120275, + 0.004323442466557026, + 0.004377615172415972, + 0.005757011938840151, + 0.0010677188402041793, + 0.00600963132455945, + 0.010011492297053337, + 0.37529492378234863, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00031869480153545737, + 0.003973849583417177, + 0.016806162893772125, + 0.013593647629022598, + 0.001353137195110321, + 0.4554371237754822, + 0.006049030926078558, + 0.0014294151915237308, + 0.01977667212486267, + 0.007164720445871353, + 0.002783068921416998, + 0.009421161375939846, + 0.005596949253231287, + 0.0060126325115561485, + 0.012963874265551567, + 0.0034675991628319025, + 0.005861096549779177, + 0.004268658347427845, + 0.011158022098243237, + 0.011237646453082561, + 0.003217349760234356, + 0.005541469436138868, + 0.015553845092654228, + 0.0004343070031609386, + 0.018514243885874748, + 0.3580656051635742, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00966514740139246, + 0.0008240287424996495, + 0.0016512670554220676, + 0.003112562233582139, + 0.0008262242772616446, + 0.006150980945676565, + 0.32758915424346924, + 0.0006379647529684007, + 0.008207762613892555, + 0.0012757014483213425, + 0.004512704908847809, + 0.01668613776564598, + 0.0017828588606789708, + 0.3219505846500397, + 0.002911597490310669, + 0.007844815962016582, + 0.003493562573567033, + 0.0010628332383930683, + 0.0030754623003304005, + 0.0031144137028604746, + 0.004299969412386417, + 0.0019549117423594, + 0.0018234350718557835, + 0.007958609610795975, + 0.007523026317358017, + 0.005903665907680988, + 0.24416062235832214, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007977406494319439, + 0.000730895611923188, + 0.0014210466761142015, + 0.0028011652175337076, + 0.0007884504739195108, + 0.005231304559856653, + 0.26295486092567444, + 0.0005426090210676193, + 0.005892876069992781, + 0.001101386733353138, + 0.0036514662206172943, + 0.01359560713171959, + 0.0015227238181978464, + 0.25584256649017334, + 0.002627924084663391, + 0.006422828882932663, + 0.0030940601136535406, + 0.000932446972001344, + 0.002746666083112359, + 0.002785393735393882, + 0.003497124882414937, + 0.0016504847444593906, + 0.0015688353450968862, + 0.006548754405230284, + 0.005473894998431206, + 0.005020079202950001, + 0.1932240128517151, + 0.20035310089588165, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007011083420366049, + 0.004123412072658539, + 0.004012262914329767, + 0.018878787755966187, + 0.0015969033120200038, + 0.01585872657597065, + 0.021582115441560745, + 0.00656988937407732, + 0.0011303380597382784, + 0.008409169502556324, + 0.008099283091723919, + 0.39500585198402405, + 0.008711021393537521, + 0.022568630054593086, + 0.016759052872657776, + 0.003329850034788251, + 0.008418659679591656, + 0.004542597569525242, + 0.015655577182769775, + 0.01613236777484417, + 0.00841239932924509, + 0.0081669632345438, + 0.004240439739078283, + 0.0066054160706698895, + 0.001299525611102581, + 0.013708256185054779, + 0.01945258304476738, + 0.020693844184279442, + 0.3290250301361084, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.007410213351249695, + 0.009788033552467823, + 0.014487621374428272, + 0.0019636047072708607, + 0.011935103684663773, + 0.006313051097095013, + 0.020669851452112198, + 0.00798715464770794, + 0.00656835176050663, + 0.005873838905245066, + 0.0014697627630084753, + 0.004803499206900597, + 0.011863230727612972, + 0.020676545798778534, + 0.002118593780323863, + 0.4252171814441681, + 0.006146552506834269, + 0.008993386290967464, + 0.0020322289783507586, + 0.0021645028609782457, + 0.0016547301784157753, + 0.011473353020846844, + 0.013433309271931648, + 0.00707376841455698, + 0.006409593392163515, + 0.005978371016681194, + 0.019444704055786133, + 0.01891355775296688, + 0.004204237833619118, + 0.33293214440345764, + 0.0, + 0.0, + 0.0 + ], + [ + 0.009955355897545815, + 0.009748119860887527, + 0.003725842572748661, + 0.0019740762654691935, + 0.44065582752227783, + 0.0014040765818208456, + 0.0015789575409144163, + 0.03220970183610916, + 0.0028995866887271404, + 0.014803172089159489, + 0.006875703576952219, + 0.0014641617890447378, + 0.010935857892036438, + 0.001469078822992742, + 0.0021334851626306772, + 0.011742248199880123, + 0.007882769219577312, + 0.008419481106102467, + 0.0022890360560268164, + 0.002442779717966914, + 0.00708719901740551, + 0.009775285609066486, + 0.003887902246788144, + 0.009308230131864548, + 0.002869158051908016, + 0.0014811842702329159, + 0.0016314064851030707, + 0.0016226922161877155, + 0.0016022950876504183, + 0.012305674143135548, + 0.37381964921951294, + 0.0, + 0.0 + ], + [ + 0.03277520090341568, + 0.01364891417324543, + 0.056880827993154526, + 0.01201050728559494, + 0.05386797711253166, + 0.004176339600235224, + 0.00364101049490273, + 0.2150014340877533, + 0.004031006712466478, + 0.020199721679091454, + 0.015337759628891945, + 0.016372648999094963, + 0.05278164520859718, + 0.0037919634487479925, + 0.012592457234859467, + 0.02643291838467121, + 0.01616365648806095, + 0.013655982911586761, + 0.012421208433806896, + 0.0125851109623909, + 0.014520397409796715, + 0.0505409874022007, + 0.04854936525225639, + 0.030934927985072136, + 0.0044258045963943005, + 0.004211958032101393, + 0.004205095116049051, + 0.003988170996308327, + 0.015366748906672001, + 0.02451629377901554, + 0.048078808933496475, + 0.1522931158542633, + 0.0 + ], + [ + 0.0041123442351818085, + 0.008301986381411552, + 0.0015794412465766072, + 0.027510425075888634, + 0.021376341581344604, + 0.013647555373609066, + 0.003686603158712387, + 0.020753832533955574, + 0.007263286039233208, + 0.3446098268032074, + 0.006845963187515736, + 0.016572441905736923, + 0.016647983342409134, + 0.003469004761427641, + 0.02440200187265873, + 0.0077178338542580605, + 0.017743604257702827, + 0.00802538450807333, + 0.02294943295419216, + 0.02340482361614704, + 0.0077221207320690155, + 0.015057295560836792, + 0.0017457695212215185, + 0.0040049063973128796, + 0.007263910956680775, + 0.012116590514779091, + 0.003786713583394885, + 0.003710875054821372, + 0.016932573169469833, + 0.008843764662742615, + 0.020229745656251907, + 0.015186822041869164, + 0.2827787697315216 + ] + ], + "expected": [ + [ + 17, + 1 + ], + [ + 18, + 3 + ], + [ + 18, + 14 + ], + [ + 19, + 3 + ], + [ + 19, + 14 + ], + [ + 20, + 10 + ], + [ + 21, + 12 + ], + [ + 22, + 2 + ], + [ + 23, + 0 + ], + [ + 24, + 8 + ], + [ + 25, + 5 + ], + [ + 26, + 6 + ], + [ + 26, + 13 + ], + [ + 27, + 6 + ], + [ + 27, + 13 + ], + [ + 28, + 11 + ], + [ + 29, + 15 + ], + [ + 30, + 4 + ], + [ + 31, + 7 + ], + [ + 32, + 9 + ] + ] +} diff --git a/default-seed2/analysis/figures/head_ablation.png b/default-seed2/analysis/figures/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..18f00a916acfad28b353a2c98647ac22be731261 Binary files /dev/null and b/default-seed2/analysis/figures/head_ablation.png differ diff --git a/default-seed2/analysis/figures/lookup_current.png b/default-seed2/analysis/figures/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..9134b3698120ab58ab904fe88e1889d4110e509d Binary files /dev/null and b/default-seed2/analysis/figures/lookup_current.png differ diff --git a/default-seed2/analysis/figures/lookup_next.png b/default-seed2/analysis/figures/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..79f22b217f2f343c3be10f96a2090c4870bee92c Binary files /dev/null and b/default-seed2/analysis/figures/lookup_next.png differ diff --git a/default-seed2/analysis/figures/lookup_next_pattern.png b/default-seed2/analysis/figures/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..286f5b4f1def3997fb3847977daa8157847e2dec Binary files /dev/null and b/default-seed2/analysis/figures/lookup_next_pattern.png differ diff --git a/default-seed2/analysis/figures/results.json b/default-seed2/analysis/figures/results.json new file mode 100644 index 0000000000000000000000000000000000000000..a407dba46228303e7f7e6419444181801c80294d --- /dev/null +++ b/default-seed2/analysis/figures/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed2", + "code": { + "commit": "42933bd7a4e2e13fd737d0d48d489f72fbeed8ac", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 0.00046359977568499744, + "token_accuracy": 0.999847412109375, + "exact_match": 0.99755859375 + }, + "untrained": { + "loss": 4.222298622131348, + "token_accuracy": 0.0142822265625, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 0.00046359977568499744, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.047352743829833344, + 1.1891647628799547, + 0.002971459209220484, + 6.620310669328319, + 0.001162868255050853, + 0.2693032673851121, + 0.0006633798184338957, + 0.01946811148081906 + ], + [ + 0.012694073404418305, + 0.0065985834517050534, + 0.0006987463275436312, + 0.0009007553162518889, + 0.0017039284866768867, + 0.0028953883156646043, + 0.006903988338308409, + 0.0009580427140463144 + ], + [ + 9.999852045439184e-05, + 0.0007847294036764652, + 6.677379133179784e-06, + 0.00027560422313399613, + 8.595394319854677e-05, + 0.0008517569222021848, + 6.360691622830927e-05, + 0.0028535542369354516 + ], + [ + -3.0265393434092402e-05, + 1.0538555216044188e-05, + 2.137760748155415e-05, + 4.4142158003523946e-05, + 1.2337521184235811e-05, + 5.3422118071466684e-06, + 6.135689909569919e-05, + 0.0001602511911187321 + ] + ], + "token_accuracy_drop": [ + [ + 0.006805419921875, + 0.140716552734375, + 0.00067138671875, + 0.8082275390625, + 0.000335693359375, + 0.03826904296875, + 0.0001220703125, + 0.002471923828125 + ], + [ + 0.0035400390625, + 0.001861572265625, + 0.000274658203125, + 0.00030517578125, + 0.000762939453125, + 0.001007080078125, + 0.0018310546875, + 0.000457763671875 + ], + [ + 3.0517578125e-05, + 0.000152587890625, + 0.0, + 0.0, + 0.0, + 0.000335693359375, + 3.0517578125e-05, + 0.00091552734375 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 6.103515625e-05 + ] + ], + "layer_loss_rise": [ + 10.483049278642284, + 0.43556219854508527, + 0.09911830493365414, + 0.01326776811038144 + ], + "layer_token_accuracy_drop": [ + 0.85162353515625, + 0.1031494140625, + 0.03265380859375, + 0.004241943359375 + ] + }, + "lookup_next": { + "score": [ + [ + 0.092010498046875, + 0.19891357421875, + 0.41748046875, + 0.129241943359375, + 0.2574462890625, + 0.121490478515625, + 0.259857177734375, + 0.216217041015625 + ], + [ + 0.488922119140625, + 0.602020263671875, + 0.51617431640625, + 0.54144287109375, + 0.494171142578125, + 0.520660400390625, + 0.5172119140625, + 0.479278564453125 + ], + [ + 0.54705810546875, + 0.614532470703125, + 0.51446533203125, + 0.537261962890625, + 0.622100830078125, + 0.527313232421875, + 0.498138427734375, + 0.476837158203125 + ], + [ + 0.34832763671875, + 0.317596435546875, + 0.367431640625, + 0.25860595703125, + 0.39251708984375, + 0.38818359375, + 0.323089599609375, + 0.404296875 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.06396484375, + 0.776519775390625, + 0.22003173828125, + 0.9862060546875, + 0.122772216796875, + 0.06219482421875, + 0.13922119140625, + 0.129913330078125 + ], + [ + 0.157501220703125, + 0.299163818359375, + 0.227294921875, + 0.23193359375, + 0.230072021484375, + 0.2099609375, + 0.230255126953125, + 0.22003173828125 + ], + [ + 0.187652587890625, + 0.2132568359375, + 0.265167236328125, + 0.1873779296875, + 0.205169677734375, + 0.32989501953125, + 0.305511474609375, + 0.290191650390625 + ], + [ + 0.23785400390625, + 0.22991943359375, + 0.28265380859375, + 0.23626708984375, + 0.262176513671875, + 0.23712158203125, + 0.3084716796875, + 0.224029541015625 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed2/analysis/figures/source-snapshot.tar.gz b/default-seed2/analysis/figures/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..53c54430ef781001eb51f25b497dbeaff162bcbf --- /dev/null +++ b/default-seed2/analysis/figures/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17289914b31ffdd29ca16523f3e26df66f4e2edb58f6dd650c6f22d3a10ddbdf +size 347464 diff --git a/default-seed2/analysis/final/critical_pattern.png b/default-seed2/analysis/final/critical_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..00c818b90d27e1146d68f23a2edc2e2867a0c1bd Binary files /dev/null and b/default-seed2/analysis/final/critical_pattern.png differ diff --git a/default-seed2/analysis/final/head_ablation.png b/default-seed2/analysis/final/head_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..18f00a916acfad28b353a2c98647ac22be731261 Binary files /dev/null and b/default-seed2/analysis/final/head_ablation.png differ diff --git a/default-seed2/analysis/final/lookup_current.png b/default-seed2/analysis/final/lookup_current.png new file mode 100644 index 0000000000000000000000000000000000000000..9134b3698120ab58ab904fe88e1889d4110e509d Binary files /dev/null and b/default-seed2/analysis/final/lookup_current.png differ diff --git a/default-seed2/analysis/final/lookup_next.png b/default-seed2/analysis/final/lookup_next.png new file mode 100644 index 0000000000000000000000000000000000000000..79f22b217f2f343c3be10f96a2090c4870bee92c Binary files /dev/null and b/default-seed2/analysis/final/lookup_next.png differ diff --git a/default-seed2/analysis/final/lookup_next_pattern.png b/default-seed2/analysis/final/lookup_next_pattern.png new file mode 100644 index 0000000000000000000000000000000000000000..286f5b4f1def3997fb3847977daa8157847e2dec Binary files /dev/null and b/default-seed2/analysis/final/lookup_next_pattern.png differ diff --git a/default-seed2/analysis/final/results.json b/default-seed2/analysis/final/results.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc203e8afe5953ea851ee326d3b22bfc8d86a02 --- /dev/null +++ b/default-seed2/analysis/final/results.json @@ -0,0 +1,219 @@ +{ + "run": "runs/default-seed2", + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "test_size": 2048, + "chance_token_accuracy": 0.015625, + "trained": { + "loss": 0.00046359977568499744, + "token_accuracy": 0.999847412109375, + "exact_match": 0.99755859375 + }, + "untrained": { + "loss": 4.222298622131348, + "token_accuracy": 0.0142822265625, + "exact_match": 0.0 + }, + "head_ablation": { + "clean_loss": 0.00046359977568499744, + "identity_patch_max_logit_change": 0.0, + "loss_rise": [ + [ + 0.047352743829833344, + 1.1891647628799547, + 0.002971459209220484, + 6.620310669328319, + 0.001162868255050853, + 0.2693032673851121, + 0.0006633798184338957, + 0.01946811148081906 + ], + [ + 0.012694073404418305, + 0.0065985834517050534, + 0.0006987463275436312, + 0.0009007553162518889, + 0.0017039284866768867, + 0.0028953883156646043, + 0.006903988338308409, + 0.0009580427140463144 + ], + [ + 9.999852045439184e-05, + 0.0007847294036764652, + 6.677379133179784e-06, + 0.00027560422313399613, + 8.595394319854677e-05, + 0.0008517569222021848, + 6.360691622830927e-05, + 0.0028535542369354516 + ], + [ + -3.0265393434092402e-05, + 1.0538555216044188e-05, + 2.137760748155415e-05, + 4.4142158003523946e-05, + 1.2337521184235811e-05, + 5.3422118071466684e-06, + 6.135689909569919e-05, + 0.0001602511911187321 + ] + ], + "token_accuracy_drop": [ + [ + 0.006805419921875, + 0.140716552734375, + 0.00067138671875, + 0.8082275390625, + 0.000335693359375, + 0.03826904296875, + 0.0001220703125, + 0.002471923828125 + ], + [ + 0.0035400390625, + 0.001861572265625, + 0.000274658203125, + 0.00030517578125, + 0.000762939453125, + 0.001007080078125, + 0.0018310546875, + 0.000457763671875 + ], + [ + 3.0517578125e-05, + 0.000152587890625, + 0.0, + 0.0, + 0.0, + 0.000335693359375, + 3.0517578125e-05, + 0.00091552734375 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 6.103515625e-05 + ] + ], + "layer_loss_rise": [ + 10.483049278642284, + 0.43556219854508527, + 0.09911830493365414, + 0.01326776811038144 + ], + "layer_token_accuracy_drop": [ + 0.85162353515625, + 0.1031494140625, + 0.03265380859375, + 0.004241943359375 + ] + }, + "lookup_next": { + "score": [ + [ + 0.092010498046875, + 0.19891357421875, + 0.41748046875, + 0.129241943359375, + 0.2574462890625, + 0.121490478515625, + 0.259857177734375, + 0.216217041015625 + ], + [ + 0.488922119140625, + 0.602020263671875, + 0.51617431640625, + 0.54144287109375, + 0.494171142578125, + 0.520660400390625, + 0.5172119140625, + 0.479278564453125 + ], + [ + 0.54705810546875, + 0.614532470703125, + 0.51446533203125, + 0.537261962890625, + 0.622100830078125, + 0.527313232421875, + 0.498138427734375, + 0.476837158203125 + ], + [ + 0.34832763671875, + 0.317596435546875, + 0.367431640625, + 0.25860595703125, + 0.39251708984375, + 0.38818359375, + 0.323089599609375, + 0.404296875 + ] + ], + "chance": 0.07715225219726562 + }, + "lookup_current": { + "score": [ + [ + 0.06396484375, + 0.776519775390625, + 0.22003173828125, + 0.9862060546875, + 0.122772216796875, + 0.06219482421875, + 0.13922119140625, + 0.129913330078125 + ], + [ + 0.157501220703125, + 0.299163818359375, + 0.227294921875, + 0.23193359375, + 0.230072021484375, + 0.2099609375, + 0.230255126953125, + 0.22003173828125 + ], + [ + 0.187652587890625, + 0.2132568359375, + 0.265167236328125, + 0.1873779296875, + 0.205169677734375, + 0.32989501953125, + 0.305511474609375, + 0.290191650390625 + ], + [ + 0.23785400390625, + 0.22991943359375, + 0.28265380859375, + 0.23626708984375, + 0.262176513671875, + 0.23712158203125, + 0.3084716796875, + 0.224029541015625 + ] + ], + "chance": 0.07715225219726562 + } +} diff --git a/default-seed2/analysis/final/source-snapshot.tar.gz b/default-seed2/analysis/final/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..a49d9e44fe543b14bc0fb320f9768c5929d3ea65 --- /dev/null +++ b/default-seed2/analysis/final/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17968d1f3ae77d01cfc40a2aac72d1700f588e8b26ec7c55c6edd78e76352e74 +size 189601 diff --git a/default-seed2/metrics.json b/default-seed2/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f730782dd176cebc827924150b2a2f5dbccb8a6a --- /dev/null +++ b/default-seed2/metrics.json @@ -0,0 +1,228 @@ +{ + "config": { + "run_name": "default-seed2", + "seed": 2, + "quick": false, + "device": "auto", + "eval_every": 250, + "task": { + "n_values": 64, + "length": 16 + }, + "splits": { + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048 + }, + "model": { + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "optim": { + "steps": 4000, + "batch_size": 256, + "lr": 0.001, + "warmup_steps": 100, + "min_lr_ratio": 0.1, + "weight_decay": 0.1, + "grad_clip": 1.0 + } + }, + "model_config": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "parameters": 3201280, + "data": { + "task": "sorting", + "version": 1, + "n_values": 64, + "length": 16, + "val_seed": 1000003, + "val_size": 1024, + "test_seed": 2000003, + "test_size": 2048, + "train_seed": 2 + }, + "code": { + "commit": "19276f75392bee9d5f6f61820395002a0746e349", + "dirty": true, + "snapshot": "source-snapshot.tar.gz" + }, + "environment": { + "python": "3.12.3", + "torch": "2.15.0+cu132", + "device": "cuda", + "precision": "torch.float32", + "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2", + "gpu": "NVIDIA GeForce RTX 3060", + "cuda": "13.2" + }, + "resumable": false, + "status": "finished", + "seconds": 646.7, + "history": [ + { + "step": 250, + "train_loss": 0.08374407142400742, + "val_loss": 0.08853542804718018, + "val_token_accuracy": 0.97259521484375, + "val_exact_match": 0.654296875, + "grad_norm": 2.7647416591644287, + "lr": 0.0009967189933441244 + }, + { + "step": 500, + "train_loss": 0.03046058677136898, + "val_loss": 0.030035920441150665, + "val_token_accuracy": 0.99041748046875, + "val_exact_match": 0.8642578125, + "grad_norm": 1.855226993560791, + "lr": 0.0009768413988762155 + }, + { + "step": 750, + "train_loss": 0.019605081528425217, + "val_loss": 0.02953985333442688, + "val_token_accuracy": 0.99114990234375, + "val_exact_match": 0.87109375, + "grad_norm": 0.8727502226829529, + "lr": 0.0009397114317029974 + }, + { + "step": 1000, + "train_loss": 0.010449330322444439, + "val_loss": 0.01985304430127144, + "val_token_accuracy": 0.99554443359375, + "val_exact_match": 0.9326171875, + "grad_norm": 0.843723714351654, + "lr": 0.0008868298366769955 + }, + { + "step": 1250, + "train_loss": 0.004702113103121519, + "val_loss": 0.008932323195040226, + "val_token_accuracy": 0.9974365234375, + "val_exact_match": 0.962890625, + "grad_norm": 0.41519755125045776, + "lr": 0.0008203340189070906 + }, + { + "step": 1500, + "train_loss": 0.00736136082559824, + "val_loss": 0.004941404797136784, + "val_token_accuracy": 0.9989013671875, + "val_exact_match": 0.982421875, + "grad_norm": 1.172151803970337, + "lr": 0.0007429116526313744 + }, + { + "step": 1750, + "train_loss": 0.009208878502249718, + "val_loss": 0.005312320776283741, + "val_token_accuracy": 0.998779296875, + "val_exact_match": 0.9814453125, + "grad_norm": 1.4689639806747437, + "lr": 0.0006576920489294011 + }, + { + "step": 2000, + "train_loss": 0.0014178939163684845, + "val_loss": 0.004395550116896629, + "val_token_accuracy": 0.9991455078125, + "val_exact_match": 0.986328125, + "grad_norm": 0.2403556853532791, + "lr": 0.0005681196730492369 + }, + { + "step": 2250, + "train_loss": 0.0022070351988077164, + "val_loss": 0.0005514943040907383, + "val_token_accuracy": 0.99981689453125, + "val_exact_match": 0.9970703125, + "grad_norm": 0.19152404367923737, + "lr": 0.00047781492361400794 + }, + { + "step": 2500, + "train_loss": 0.0019115328323096037, + "val_loss": 0.0011586558539420366, + "val_token_accuracy": 0.99957275390625, + "val_exact_match": 0.9931640625, + "grad_norm": 0.22928588092327118, + "lr": 0.0003904278008308589 + }, + { + "step": 2750, + "train_loss": 0.003090714570134878, + "val_loss": 0.001331779290921986, + "val_token_accuracy": 0.9996337890625, + "val_exact_match": 0.994140625, + "grad_norm": 0.6994138956069946, + "lr": 0.0003094903782424895 + }, + { + "step": 3000, + "train_loss": 0.0001012918510241434, + "val_loss": 0.0001072997329174541, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 0.010654399171471596, + "lr": 0.0002382740409206803 + }, + { + "step": 3250, + "train_loss": 3.054919943679124e-05, + "val_loss": 0.004428497049957514, + "val_token_accuracy": 0.99945068359375, + "val_exact_match": 0.9912109375, + "grad_norm": 0.004973436240106821, + "lr": 0.00017965726034785466 + }, + { + "step": 3500, + "train_loss": 0.00010091940930578858, + "val_loss": 0.0005630630766972899, + "val_token_accuracy": 0.99981689453125, + "val_exact_match": 0.9970703125, + "grad_norm": 0.17502489686012268, + "lr": 0.00013600925035352912 + }, + { + "step": 3750, + "train_loss": 1.1987899597443175e-06, + "val_loss": 4.915940280625364e-06, + "val_token_accuracy": 1.0, + "val_exact_match": 1.0, + "grad_norm": 6.335216312436387e-05, + "lr": 0.00010909420658097956 + }, + { + "step": 4000, + "train_loss": 4.847031505050836e-06, + "val_loss": 0.0006610961863771081, + "val_token_accuracy": 0.99969482421875, + "val_exact_match": 0.9951171875, + "grad_norm": 0.0012014994863420725, + "lr": 0.0001 + } + ], + "final": { + "val_loss": 0.0006610961863771081, + "val_token_accuracy": 0.99969482421875, + "val_exact_match": 0.9951171875 + } +} diff --git a/default-seed2/model.safetensors b/default-seed2/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..24debb7bcb25976b97f834a71206e631b703c346 --- /dev/null +++ b/default-seed2/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60e7b77001fa7d1d1f7b681dfe3cacfb8ea4d341a2e2ecb042631842fe10e46 +size 12811264 diff --git a/default-seed2/model_config.json b/default-seed2/model_config.json new file mode 100644 index 0000000000000000000000000000000000000000..934acf04ad61e2639626dada2f7ad2c2370bd104 --- /dev/null +++ b/default-seed2/model_config.json @@ -0,0 +1,81 @@ +{ + "format_version": 1, + "transformer": { + "vocab_size": 65, + "n_layers": 4, + "d_model": 256, + "n_heads": 8, + "d_mlp": 1024, + "max_seq_len": 33, + "mlp": "gelu", + "norm": "layernorm", + "dropout": 0.0 + }, + "vocab": [ + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "SEP" + ] +} diff --git a/default-seed2/source-snapshot.tar.gz b/default-seed2/source-snapshot.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9cdbf2cae6983280a9a930e30254a87b8eddfc72 --- /dev/null +++ b/default-seed2/source-snapshot.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e963742ccde0aca2462ea11e294c6b2504355b7b78eda8738885e210454496e +size 187056 diff --git a/default-seed2/training.png b/default-seed2/training.png new file mode 100644 index 0000000000000000000000000000000000000000..94861533705099b7040744da9bac47d37b0d7916 Binary files /dev/null and b/default-seed2/training.png differ