Hugging Face's logo

SlayerLab
/
bdh-scaling

PyTorch
English
Polish
Czech
bdh
linear-attention
scaling-laws
replication
Model card Files
xet
 Community
bdh-scaling
File size: 2,316 Bytes
2f957bf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
[
  {
    "step": 0,
    "val_loss": 5.653509521484375,
    "tok_per_s": 0.0
  },
  {
    "step": 0,
    "train_loss": 5.5814666748046875,
    "lr": 1e-06,
    "grad_norm": 13.262894322262873
  },
  {
    "step": 400,
    "train_loss": 2.416318416595459,
    "lr": 0.00040100000000000004,
    "grad_norm": 0.6385184898864017
  },
  {
    "step": 500,
    "val_loss": 3.082964277267456,
    "tok_per_s": 7353.034710926345
  },
  {
    "step": 800,
    "train_loss": 1.6120072603225708,
    "lr": 0.0008010000000000001,
    "grad_norm": 0.7084504426978477
  },
  {
    "step": 1000,
    "val_loss": 3.0517220973968504,
    "tok_per_s": 7422.438347837392
  },
  {
    "step": 1200,
    "train_loss": 1.5758367776870728,
    "lr": 0.000988964669915361,
    "grad_norm": 0.606163267395569
  },
  {
    "step": 1500,
    "val_loss": 2.738486385345459,
    "tok_per_s": 7446.004210597483
  },
  {
    "step": 1600,
    "train_loss": 1.3207018375396729,
    "lr": 0.0009042005118087267,
    "grad_norm": 0.5098235841908811
  },
  {
    "step": 2000,
    "val_loss": 2.8913249254226683,
    "tok_per_s": 7457.7868950539005
  },
  {
    "step": 2000,
    "train_loss": 1.0848292112350464,
    "lr": 0.0007495464131639923,
    "grad_norm": 0.43657336874306774
  },
  {
    "step": 2400,
    "train_loss": 1.086487889289856,
    "lr": 0.0005517434726252113,
    "grad_norm": 0.41061234194201285
  },
  {
    "step": 2500,
    "val_loss": 2.8504832863807676,
    "tok_per_s": 7464.902119770951
  },
  {
    "step": 2800,
    "train_loss": 1.0458492040634155,
    "lr": 0.00034499361559483973,
    "grad_norm": 0.38063624390350104
  },
  {
    "step": 3000,
    "val_loss": 2.5975316882133486,
    "tok_per_s": 7469.624347383343
  },
  {
    "step": 3200,
    "train_loss": 1.2963083982467651,
    "lr": 0.00016504577061691466,
    "grad_norm": 0.554473536775391
  },
  {
    "step": 3500,
    "val_loss": 2.753005313873291,
    "tok_per_s": 7472.654569796267
  },
  {
    "step": 3600,
    "train_loss": 1.0354111194610596,
    "lr": 4.301455486296946e-05,
    "grad_norm": 0.4133181698759571
  },
  {
    "step": 4000,
    "val_loss": 2.771369421482086,
    "tok_per_s": 7475.274380762689
  },
  {
    "step": 4000,
    "train_loss": 1.1188071966171265,
    "lr": 2.741556527352529e-10,
    "grad_norm": 0.4136691179164048
  }
]