Hugging Face's logo

coeiico
/
tiny-transformer

PyTorch
Safetensors
transformer
interpretability
from-scratch
Model card Files
xet
 Community
tiny-transformer
File size: 6,048 Bytes
9cdf072
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
{
  "config": {
    "run_name": "default-seed0",
    "seed": 0,
    "quick": false,
    "device": "auto",
    "eval_every": 250,
    "task": {
      "n_values": 64,
      "length": 16
    },
    "splits": {
      "val_seed": 1000003,
      "val_size": 1024,
      "test_seed": 2000003,
      "test_size": 2048
    },
    "model": {
      "n_layers": 4,
      "d_model": 256,
      "n_heads": 8,
      "d_mlp": 1024,
      "mlp": "gelu",
      "norm": "layernorm",
      "dropout": 0.0
    },
    "optim": {
      "steps": 4000,
      "batch_size": 256,
      "lr": 0.001,
      "warmup_steps": 100,
      "min_lr_ratio": 0.1,
      "weight_decay": 0.1,
      "grad_clip": 1.0
    }
  },
  "model_config": {
    "vocab_size": 65,
    "n_layers": 4,
    "d_model": 256,
    "n_heads": 8,
    "d_mlp": 1024,
    "max_seq_len": 33,
    "mlp": "gelu",
    "norm": "layernorm",
    "dropout": 0.0
  },
  "parameters": 3201280,
  "data": {
    "task": "sorting",
    "version": 1,
    "n_values": 64,
    "length": 16,
    "val_seed": 1000003,
    "val_size": 1024,
    "test_seed": 2000003,
    "test_size": 2048,
    "train_seed": 0
  },
  "code": {
    "commit": "19276f75392bee9d5f6f61820395002a0746e349",
    "dirty": true,
    "snapshot": "source-snapshot.tar.gz"
  },
  "environment": {
    "python": "3.12.3",
    "torch": "2.15.0+cu132",
    "device": "cuda",
    "precision": "torch.float32",
    "uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2",
    "gpu": "NVIDIA GeForce RTX 3060",
    "cuda": "13.2"
  },
  "resumable": false,
  "status": "finished",
  "seconds": 646.7,
  "history": [
    {
      "step": 250,
      "train_loss": 0.07291533797979355,
      "val_loss": 0.0985519140958786,
      "val_token_accuracy": 0.9683837890625,
      "val_exact_match": 0.595703125,
      "grad_norm": 2.3497650623321533,
      "lr": 0.0009967189933441244
    },
    {
      "step": 500,
      "train_loss": 0.014207392930984497,
      "val_loss": 0.031683966517448425,
      "val_token_accuracy": 0.99053955078125,
      "val_exact_match": 0.8720703125,
      "grad_norm": 0.7331729531288147,
      "lr": 0.0009768413988762155
    },
    {
      "step": 750,
      "train_loss": 0.02052781730890274,
      "val_loss": 0.023844445124268532,
      "val_token_accuracy": 0.99273681640625,
      "val_exact_match": 0.890625,
      "grad_norm": 1.3566714525222778,
      "lr": 0.0009397114317029974
    },
    {
      "step": 1000,
      "train_loss": 0.009082821197807789,
      "val_loss": 0.01231020875275135,
      "val_token_accuracy": 0.996826171875,
      "val_exact_match": 0.951171875,
      "grad_norm": 0.7233428359031677,
      "lr": 0.0008868298366769955
    },
    {
      "step": 1250,
      "train_loss": 0.00922011211514473,
      "val_loss": 0.019146941602230072,
      "val_token_accuracy": 0.994873046875,
      "val_exact_match": 0.9208984375,
      "grad_norm": 0.5756050944328308,
      "lr": 0.0008203340189070906
    },
    {
      "step": 1500,
      "train_loss": 0.01364622451364994,
      "val_loss": 0.005576150491833687,
      "val_token_accuracy": 0.998779296875,
      "val_exact_match": 0.9814453125,
      "grad_norm": 1.3124696016311646,
      "lr": 0.0007429116526313744
    },
    {
      "step": 1750,
      "train_loss": 0.003266311716288328,
      "val_loss": 0.001853839261457324,
      "val_token_accuracy": 0.9996337890625,
      "val_exact_match": 0.994140625,
      "grad_norm": 0.28147438168525696,
      "lr": 0.0006576920489294011
    },
    {
      "step": 2000,
      "train_loss": 0.005052485503256321,
      "val_loss": 0.0006970376125536859,
      "val_token_accuracy": 0.9998779296875,
      "val_exact_match": 0.998046875,
      "grad_norm": 1.1249206066131592,
      "lr": 0.0005681196730492369
    },
    {
      "step": 2250,
      "train_loss": 0.00020874346955679357,
      "val_loss": 0.0009217980550602078,
      "val_token_accuracy": 0.99981689453125,
      "val_exact_match": 0.9970703125,
      "grad_norm": 0.03951434791088104,
      "lr": 0.00047781492361400794
    },
    {
      "step": 2500,
      "train_loss": 0.001243826001882553,
      "val_loss": 0.0005396516062319279,
      "val_token_accuracy": 0.99981689453125,
      "val_exact_match": 0.9970703125,
      "grad_norm": 0.23733462393283844,
      "lr": 0.0003904278008308589
    },
    {
      "step": 2750,
      "train_loss": 0.00022199669911060482,
      "val_loss": 0.00017582655709702522,
      "val_token_accuracy": 0.99993896484375,
      "val_exact_match": 0.9990234375,
      "grad_norm": 0.11565989255905151,
      "lr": 0.0003094903782424895
    },
    {
      "step": 3000,
      "train_loss": 0.00021956569980829954,
      "val_loss": 8.435545896645635e-05,
      "val_token_accuracy": 1.0,
      "val_exact_match": 1.0,
      "grad_norm": 0.061819761991500854,
      "lr": 0.0002382740409206803
    },
    {
      "step": 3250,
      "train_loss": 3.6058951081940904e-05,
      "val_loss": 8.97766076377593e-05,
      "val_token_accuracy": 0.99993896484375,
      "val_exact_match": 0.9990234375,
      "grad_norm": 0.004151737317442894,
      "lr": 0.00017965726034785466
    },
    {
      "step": 3500,
      "train_loss": 4.75610613648314e-06,
      "val_loss": 9.887960004562046e-06,
      "val_token_accuracy": 1.0,
      "val_exact_match": 1.0,
      "grad_norm": 0.0010888022370636463,
      "lr": 0.00013600925035352912
    },
    {
      "step": 3750,
      "train_loss": 0.00010177387594012544,
      "val_loss": 0.00023778356262482703,
      "val_token_accuracy": 0.99993896484375,
      "val_exact_match": 0.9990234375,
      "grad_norm": 0.09358467906713486,
      "lr": 0.00010909420658097956
    },
    {
      "step": 4000,
      "train_loss": 1.727779454085976e-05,
      "val_loss": 1.847820749389939e-05,
      "val_token_accuracy": 1.0,
      "val_exact_match": 1.0,
      "grad_norm": 0.014979858882725239,
      "lr": 0.0001
    }
  ],
  "final": {
    "val_loss": 1.847820749389939e-05,
    "val_token_accuracy": 1.0,
    "val_exact_match": 1.0
  }
}