Hugging Face's logo

SlayerLab
/
bdh-scaling

PyTorch
English
Polish
Czech
bdh
linear-attention
scaling-laws
replication
Model card Files
xet
 Community
bdh-scaling
File size: 2,316 Bytes
2f957bf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
[
  {
    "step": 0,
    "val_loss": 5.619070100784302,
    "tok_per_s": 0.0
  },
  {
    "step": 0,
    "train_loss": 5.616189956665039,
    "lr": 1e-06,
    "grad_norm": 29.273247877751935
  },
  {
    "step": 400,
    "train_loss": 2.6516408920288086,
    "lr": 0.00040100000000000004,
    "grad_norm": 1.352886640052276
  },
  {
    "step": 500,
    "val_loss": 3.729518210887909,
    "tok_per_s": 9780.942111616952
  },
  {
    "step": 800,
    "train_loss": 2.1480326652526855,
    "lr": 0.0008010000000000001,
    "grad_norm": 1.4094370777928795
  },
  {
    "step": 1000,
    "val_loss": 3.7294750928878786,
    "tok_per_s": 12756.35488794051
  },
  {
    "step": 1200,
    "train_loss": 2.240513563156128,
    "lr": 0.000988964669915361,
    "grad_norm": 0.8976523252261448
  },
  {
    "step": 1500,
    "val_loss": 3.4839155077934265,
    "tok_per_s": 14198.011686047617
  },
  {
    "step": 1600,
    "train_loss": 1.931168556213379,
    "lr": 0.0009042005118087267,
    "grad_norm": 0.7024651656325952
  },
  {
    "step": 2000,
    "val_loss": 3.584827196598053,
    "tok_per_s": 15103.588029890892
  },
  {
    "step": 2000,
    "train_loss": 1.7497514486312866,
    "lr": 0.0007495464131639923,
    "grad_norm": 0.7033821207986488
  },
  {
    "step": 2400,
    "train_loss": 1.6890695095062256,
    "lr": 0.0005517434726252113,
    "grad_norm": 0.5302752272105545
  },
  {
    "step": 2500,
    "val_loss": 3.5327754378318788,
    "tok_per_s": 15689.552746711572
  },
  {
    "step": 2800,
    "train_loss": 1.6101696491241455,
    "lr": 0.00034499361559483973,
    "grad_norm": 0.5084148331381609
  },
  {
    "step": 3000,
    "val_loss": 3.376665234565735,
    "tok_per_s": 16068.29102371795
  },
  {
    "step": 3200,
    "train_loss": 1.9343514442443848,
    "lr": 0.00016504577061691466,
    "grad_norm": 0.6363576595902085
  },
  {
    "step": 3500,
    "val_loss": 3.4783490896224976,
    "tok_per_s": 16377.076453772866
  },
  {
    "step": 3600,
    "train_loss": 1.604689121246338,
    "lr": 4.301455486296946e-05,
    "grad_norm": 0.5709548462524181
  },
  {
    "step": 4000,
    "val_loss": 3.490683543682098,
    "tok_per_s": 16631.861465038073
  },
  {
    "step": 4000,
    "train_loss": 1.6441144943237305,
    "lr": 2.741556527352529e-10,
    "grad_norm": 0.5403057249666863
  }
]