Hugging Face's logo

CodeIsAbstract
/
HybridModelScratch

Transformers
Safetensors
hybrid_fourier_lm
Generated from Trainer
Model card Files
xet
 Community
CodeIsAbstract commited on
Commit
c7977d4
·
verified ·
1 Parent(s): 50f0041

Training in progress, step 300, checkpoint

Browse files
Files changed (8) hide show
  1. last-checkpoint/config.json +2 -2
  2. last-checkpoint/model.safetensors +2 -2
  3. last-checkpoint/optimizer.pt +2 -2
  4. last-checkpoint/rng_state_0.pth +3 -0
  5. last-checkpoint/rng_state_1.pth +3 -0
  6. last-checkpoint/scheduler.pt +1 -1
  7. last-checkpoint/trainer_state.json +105 -24
  8. last-checkpoint/training_args.bin +1 -1
last-checkpoint/config.json CHANGED
@@ -23,9 +23,9 @@
23
  ],
24
  "model_type": "hybrid_fourier_lm",
25
  "num_layers": 12,
26
- "num_modes": 128,
27
  "pad_token_id": 0,
28
- "time_scale": 32.0,
29
  "transformers_version": "4.56.0",
30
  "vocab_size": 50277
31
  }
 
23
  ],
24
  "model_type": "hybrid_fourier_lm",
25
  "num_layers": 12,
26
+ "num_modes": 64,
27
  "pad_token_id": 0,
28
+ "time_scale": 128.0,
29
  "transformers_version": "4.56.0",
30
  "vocab_size": 50277
31
  }
last-checkpoint/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ab7d60b908941ca0da8c0d5c61e9391a1b78eb40ba1aaba26858fa2b02261b2
3
- size 693148584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bae43b52627ec635e8138665a3a76fb67c1c6817b62a6b5533adc30466d1216
3
+ size 579748776
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:def443ffb3e45820eed5edbac96d7c8d3742a6a0b60d8a70e42c81253b2ea241
3
- size 1386414411
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4caadb553679cfc495bfa78dbf95efbd0038757efc70a680411a31a089ef9238
3
+ size 1159627083
last-checkpoint/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02b94dfa6d7836c5c68f2d2549fc9be186c2691c198e846c097e4aae3ccf90e6
3
+ size 14917
last-checkpoint/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a903f2c82e7b21fcfdd55b988513d1f4b97daf80c782473ccecc104f23b4dd7c
3
+ size 14917
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b2c20f79557a74480933950d77095f63537d2e565ad7970afd529b23fbf5946
3
  size 1465
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81bef68395366893911697d5372e508b6e708198d9017941b12c0581628d2a93
3
  size 1465
last-checkpoint/trainer_state.json CHANGED
@@ -4,47 +4,128 @@
4
  "best_model_checkpoint": null,
5
  "epoch": 0.15,
6
  "eval_steps": 100,
7
- "global_step": 150,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "epoch": 0.05,
14
- "grad_norm": 13.136443138122559,
15
- "learning_rate": 9.799999999999998e-05,
16
- "loss": 83.7759,
 
 
 
 
 
 
 
17
  "step": 50
18
  },
19
  {
20
- "epoch": 0.1,
21
- "grad_norm": 4.371773719787598,
22
- "learning_rate": 0.000198,
23
- "loss": 60.3835,
 
 
 
 
 
 
 
24
  "step": 100
25
  },
26
  {
27
- "epoch": 0.1,
28
- "eval_accuracy": 0.12107013087626775,
29
- "eval_loss": 6.948241233825684,
30
- "eval_runtime": 10.1735,
31
- "eval_samples_per_second": 95.345,
32
- "eval_steps_per_second": 3.047,
33
  "step": 100
34
  },
35
  {
36
- "epoch": 0.15,
37
- "grad_norm": 4.350087642669678,
38
- "learning_rate": 0.000298,
39
- "loss": 53.8921,
 
 
 
 
 
 
 
40
  "step": 150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  }
42
  ],
43
- "logging_steps": 50,
44
- "max_steps": 1000,
45
  "num_input_tokens_seen": 0,
46
  "num_train_epochs": 9223372036854775807,
47
- "save_steps": 150,
48
  "stateful_callbacks": {
49
  "TrainerControl": {
50
  "args": {
@@ -57,8 +138,8 @@
57
  "attributes": {}
58
  }
59
  },
60
- "total_flos": 3.17716313407488e+16,
61
- "train_batch_size": 32,
62
  "trial_name": null,
63
  "trial_params": null
64
  }
 
4
  "best_model_checkpoint": null,
5
  "epoch": 0.15,
6
  "eval_steps": 100,
7
+ "global_step": 300,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "epoch": 0.0125,
14
+ "grad_norm": 7352.2265625,
15
+ "learning_rate": 1.6000000000000003e-05,
16
+ "loss": 175.5381,
17
+ "step": 25
18
+ },
19
+ {
20
+ "epoch": 0.025,
21
+ "grad_norm": 2155.41015625,
22
+ "learning_rate": 3.266666666666667e-05,
23
+ "loss": 174.6277,
24
  "step": 50
25
  },
26
  {
27
+ "epoch": 0.0375,
28
+ "grad_norm": 29.340621948242188,
29
+ "learning_rate": 4.933333333333334e-05,
30
+ "loss": 157.9852,
31
+ "step": 75
32
+ },
33
+ {
34
+ "epoch": 0.05,
35
+ "grad_norm": 11.753185272216797,
36
+ "learning_rate": 6.6e-05,
37
+ "loss": 133.1257,
38
  "step": 100
39
  },
40
  {
41
+ "epoch": 0.05,
42
+ "eval_accuracy": 0.08756986554511534,
43
+ "eval_loss": 7.706512928009033,
44
+ "eval_runtime": 7.3236,
45
+ "eval_samples_per_second": 66.224,
46
+ "eval_steps_per_second": 2.185,
47
  "step": 100
48
  },
49
  {
50
+ "epoch": 0.0625,
51
+ "grad_norm": 11.1610746383667,
52
+ "learning_rate": 8.266666666666667e-05,
53
+ "loss": 119.0136,
54
+ "step": 125
55
+ },
56
+ {
57
+ "epoch": 0.075,
58
+ "grad_norm": 7.337241172790527,
59
+ "learning_rate": 9.933333333333334e-05,
60
+ "loss": 113.2304,
61
  "step": 150
62
+ },
63
+ {
64
+ "epoch": 0.0875,
65
+ "grad_norm": 9.895074844360352,
66
+ "learning_rate": 9.995847987378953e-05,
67
+ "loss": 109.5068,
68
+ "step": 175
69
+ },
70
+ {
71
+ "epoch": 0.1,
72
+ "grad_norm": 8.935585975646973,
73
+ "learning_rate": 9.982700328363471e-05,
74
+ "loss": 107.0186,
75
+ "step": 200
76
+ },
77
+ {
78
+ "epoch": 0.1,
79
+ "eval_accuracy": 0.13644318107230674,
80
+ "eval_loss": 6.609853267669678,
81
+ "eval_runtime": 7.4218,
82
+ "eval_samples_per_second": 65.348,
83
+ "eval_steps_per_second": 2.156,
84
+ "step": 200
85
+ },
86
+ {
87
+ "epoch": 0.1125,
88
+ "grad_norm": 17.207761764526367,
89
+ "learning_rate": 9.96057350657239e-05,
90
+ "loss": 105.3798,
91
+ "step": 225
92
+ },
93
+ {
94
+ "epoch": 0.125,
95
+ "grad_norm": 37.589744567871094,
96
+ "learning_rate": 9.929507396034141e-05,
97
+ "loss": 103.9933,
98
+ "step": 250
99
+ },
100
+ {
101
+ "epoch": 0.1375,
102
+ "grad_norm": 48.09913635253906,
103
+ "learning_rate": 9.889557979979694e-05,
104
+ "loss": 103.0857,
105
+ "step": 275
106
+ },
107
+ {
108
+ "epoch": 0.15,
109
+ "grad_norm": 18.950061798095703,
110
+ "learning_rate": 9.840797249956985e-05,
111
+ "loss": 102.3462,
112
+ "step": 300
113
+ },
114
+ {
115
+ "epoch": 0.15,
116
+ "eval_accuracy": 0.14476797317970908,
117
+ "eval_loss": 6.367699146270752,
118
+ "eval_runtime": 7.5111,
119
+ "eval_samples_per_second": 64.571,
120
+ "eval_steps_per_second": 2.13,
121
+ "step": 300
122
  }
123
  ],
124
+ "logging_steps": 25,
125
+ "max_steps": 2000,
126
  "num_input_tokens_seen": 0,
127
  "num_train_epochs": 9223372036854775807,
128
+ "save_steps": 300,
129
  "stateful_callbacks": {
130
  "TrainerControl": {
131
  "args": {
 
138
  "attributes": {}
139
  }
140
  },
141
+ "total_flos": 2.006672071458816e+17,
142
+ "train_batch_size": 64,
143
  "trial_name": null,
144
  "trial_params": null
145
  }
last-checkpoint/training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d216d38e0e5957b2090f5129648bebdd9025a8fa1a2649f1ed7e9e8705c0034c
3
  size 5841
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62f6f9e5edba54231bd7b997d23bedb3d9444419419088dc30d8c2eaddbccf29
3
  size 5841