Hugging Face's logo

Compactbot
/
recurrent-context-causal-lab

Model card Files
xet
 Community
2
Compactbot commited on
Commit
e4fc867
·
verified ·
1 Parent(s): 031ae95

Add measured experimental results

Browse files
Files changed (1) hide show
  1. results.json +226 -0
results.json ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment": "recurrent_ctx_interaction",
3
+ "question": "Does recurrent depth amplify the benefit of longer context at tiny scale?",
4
+ "timestamp": "2026-10-09T22:44:59Z",
5
+ "config": {
6
+ "d_model": 256,
7
+ "n_heads": 4,
8
+ "vocab": 256,
9
+ "batch": 32,
10
+ "steps": 2000,
11
+ "lr": 0.0003,
12
+ "device": "cuda"
13
+ },
14
+ "results": {
15
+ "recurrent_ctx64": {
16
+ "final_train_loss": 1.853339433670044,
17
+ "avg_train_loss_last_100": 1.8315880990028381,
18
+ "best_val_loss": 1.80549016436752,
19
+ "final_val_loss": 1.80549016436752,
20
+ "final_val_ppl": 6.08295236438963,
21
+ "val_curve": [
22
+ [
23
+ 200,
24
+ 2.67759696902061
25
+ ],
26
+ [
27
+ 400,
28
+ 2.4178282270626146
29
+ ],
30
+ [
31
+ 600,
32
+ 2.2530713908526363
33
+ ],
34
+ [
35
+ 800,
36
+ 2.111119897024972
37
+ ],
38
+ [
39
+ 1000,
40
+ 2.0204621748048432
41
+ ],
42
+ [
43
+ 1200,
44
+ 1.9230032847852123
45
+ ],
46
+ [
47
+ 1400,
48
+ 1.8634966042577004
49
+ ],
50
+ [
51
+ 1600,
52
+ 1.827961488645904
53
+ ],
54
+ [
55
+ 1800,
56
+ 1.8110538823263986
57
+ ],
58
+ [
59
+ 2000,
60
+ 1.80549016436752
61
+ ]
62
+ ],
63
+ "params": 1397504,
64
+ "train_time_s": 10.449508428573608
65
+ },
66
+ "recurrent_ctx256": {
67
+ "final_train_loss": 2.2531955242156982,
68
+ "avg_train_loss_last_100": 2.245353183746338,
69
+ "best_val_loss": 2.235594819516552,
70
+ "final_val_loss": 2.235594819516552,
71
+ "final_val_ppl": 9.352042975781874,
72
+ "val_curve": [
73
+ [
74
+ 200,
75
+ 2.5640992836076384
76
+ ],
77
+ [
78
+ 400,
79
+ 2.4392180549855134
80
+ ],
81
+ [
82
+ 600,
83
+ 2.3669187321954843
84
+ ],
85
+ [
86
+ 800,
87
+ 2.323085509514322
88
+ ],
89
+ [
90
+ 1000,
91
+ 2.2937028310736833
92
+ ],
93
+ [
94
+ 1200,
95
+ 2.2744714941297257
96
+ ],
97
+ [
98
+ 1400,
99
+ 2.2546324447709685
100
+ ],
101
+ [
102
+ 1600,
103
+ 2.245326196904085
104
+ ],
105
+ [
106
+ 1800,
107
+ 2.2375385070333675
108
+ ],
109
+ [
110
+ 2000,
111
+ 2.235594819516552
112
+ ]
113
+ ],
114
+ "params": 1446656,
115
+ "train_time_s": 25.86048126220703
116
+ },
117
+ "stacked_ctx64": {
118
+ "final_train_loss": 1.8972289562225342,
119
+ "avg_train_loss_last_100": 1.9378939366340637,
120
+ "best_val_loss": 1.9216734706139078,
121
+ "final_val_loss": 1.9216734706139078,
122
+ "final_val_ppl": 6.832382699238212,
123
+ "val_curve": [
124
+ [
125
+ 200,
126
+ 2.790465888198541
127
+ ],
128
+ [
129
+ 400,
130
+ 2.4131838263297567
131
+ ],
132
+ [
133
+ 600,
134
+ 2.278608127516143
135
+ ],
136
+ [
137
+ 800,
138
+ 2.14184259103269
139
+ ],
140
+ [
141
+ 1000,
142
+ 2.0658977236066547
143
+ ],
144
+ [
145
+ 1200,
146
+ 2.006630532595576
147
+ ],
148
+ [
149
+ 1400,
150
+ 1.9629625875122694
151
+ ],
152
+ [
153
+ 1600,
154
+ 1.9364350644909605
155
+ ],
156
+ [
157
+ 1800,
158
+ 1.9249796546235376
159
+ ],
160
+ [
161
+ 2000,
162
+ 1.9216734706139078
163
+ ]
164
+ ],
165
+ "params": 1402880,
166
+ "train_time_s": 7.927839279174805
167
+ },
168
+ "stacked_ctx256": {
169
+ "final_train_loss": 2.2880020141601562,
170
+ "avg_train_loss_last_100": 2.284539570808411,
171
+ "best_val_loss": 2.2725286454570535,
172
+ "final_val_loss": 2.2725286454570535,
173
+ "final_val_ppl": 9.703907558344023,
174
+ "val_curve": [
175
+ [
176
+ 200,
177
+ 2.7255869145296057
178
+ ],
179
+ [
180
+ 400,
181
+ 2.4116121185069184
182
+ ],
183
+ [
184
+ 600,
185
+ 2.3523950440543038
186
+ ],
187
+ [
188
+ 800,
189
+ 2.3312676225389755
190
+ ],
191
+ [
192
+ 1000,
193
+ 2.3142523259532695
194
+ ],
195
+ [
196
+ 1200,
197
+ 2.298062850991074
198
+ ],
199
+ [
200
+ 1400,
201
+ 2.284544699532645
202
+ ],
203
+ [
204
+ 1600,
205
+ 2.278283355673965
206
+ ],
207
+ [
208
+ 1800,
209
+ 2.2737569964661892
210
+ ],
211
+ [
212
+ 2000,
213
+ 2.2725286454570535
214
+ ]
215
+ ],
216
+ "params": 1452032,
217
+ "train_time_s": 13.692972898483276
218
+ }
219
+ },
220
+ "analysis": {
221
+ "recurrent_ctx_gain": -3.2690906113922438,
222
+ "stacked_ctx_gain": -2.8715248591058105,
223
+ "recurrent_advantage_ctx64": 0.749430334848582,
224
+ "recurrent_advantage_ctx256": 0.3518645825621487
225
+ }
226
+ }