Hugging Face's logo

SlayerLab
/
bdh-scaling

PyTorch
English
Polish
Czech
bdh
linear-attention
scaling-laws
replication
Model card Files
xet
 Community
bdh-scaling
File size: 2,321 Bytes
2f957bf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
[
  {
    "step": 0,
    "val_loss": 5.574858093261719,
    "tok_per_s": 0.0
  },
  {
    "step": 0,
    "train_loss": 5.5917205810546875,
    "lr": 1e-06,
    "grad_norm": 12.717687525126948
  },
  {
    "step": 400,
    "train_loss": 2.40862774848938,
    "lr": 0.00040100000000000004,
    "grad_norm": 0.6444747412529727
  },
  {
    "step": 500,
    "val_loss": 2.9989272117614747,
    "tok_per_s": 3718.2248728935924
  },
  {
    "step": 800,
    "train_loss": 1.5925631523132324,
    "lr": 0.0008010000000000001,
    "grad_norm": 0.55659908126474
  },
  {
    "step": 1000,
    "val_loss": 2.9269163846969604,
    "tok_per_s": 3750.517035850887
  },
  {
    "step": 1200,
    "train_loss": 1.52901291847229,
    "lr": 0.000988964669915361,
    "grad_norm": 0.495081136965696
  },
  {
    "step": 1500,
    "val_loss": 2.7253864288330076,
    "tok_per_s": 3761.4818781802555
  },
  {
    "step": 1600,
    "train_loss": 1.3101574182510376,
    "lr": 0.0009042005118087267,
    "grad_norm": 0.5471434485914135
  },
  {
    "step": 2000,
    "val_loss": 2.8566264510154724,
    "tok_per_s": 3766.8687186069046
  },
  {
    "step": 2000,
    "train_loss": 1.0662428140640259,
    "lr": 0.0007495464131639923,
    "grad_norm": 0.40414115602629586
  },
  {
    "step": 2400,
    "train_loss": 1.0698206424713135,
    "lr": 0.0005517434726252113,
    "grad_norm": 0.3638077828799298
  },
  {
    "step": 2500,
    "val_loss": 2.8301284432411196,
    "tok_per_s": 3770.1451769209293
  },
  {
    "step": 2800,
    "train_loss": 1.0058163404464722,
    "lr": 0.00034499361559483973,
    "grad_norm": 0.35350228406513934
  },
  {
    "step": 3000,
    "val_loss": 2.6090492963790894,
    "tok_per_s": 3772.2713155786982
  },
  {
    "step": 3200,
    "train_loss": 1.2697056531906128,
    "lr": 0.00016504577061691466,
    "grad_norm": 0.4867203515384807
  },
  {
    "step": 3500,
    "val_loss": 2.7727127075195312,
    "tok_per_s": 3773.412703899254
  },
  {
    "step": 3600,
    "train_loss": 1.0147331953048706,
    "lr": 4.301455486296946e-05,
    "grad_norm": 0.3598032306041595
  },
  {
    "step": 4000,
    "val_loss": 2.7373061776161194,
    "tok_per_s": 3774.2970880066314
  },
  {
    "step": 4000,
    "train_loss": 1.102695345878601,
    "lr": 2.741556527352529e-10,
    "grad_norm": 0.36774689673228605
  }
]