Hugging Face's logo

kavanmevada
/
eng-word-model

Text Generation
Transformers
TensorBoard
Safetensors
smollm3
Model card Files
xet
Metrics Community
kavanmevada commited on
Commit
fd6fe50
·
verified ·
1 Parent(s): 1286ce7

Training in progress, step 10, checkpoint

Browse files
Files changed (8) hide show
  1. last-checkpoint/config.json +108 -0
  2. last-checkpoint/generation_config.json +8 -0
  3. last-checkpoint/model.safetensors +3 -0
  4. last-checkpoint/optimizer.pt +3 -0
  5. last-checkpoint/rng_state.pth +3 -0
  6. last-checkpoint/scheduler.pt +3 -0
  7. last-checkpoint/trainer_state.json +104 -0
  8. last-checkpoint/training_args.bin +3 -0
last-checkpoint/config.json ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "SmolLM3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 128000,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 128012,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 1024,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 4096,
14
+ "layer_types": [
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention",
44
+ "full_attention",
45
+ "full_attention",
46
+ "full_attention",
47
+ "full_attention",
48
+ "full_attention",
49
+ "full_attention",
50
+ "full_attention"
51
+ ],
52
+ "max_position_embeddings": 2048,
53
+ "max_window_layers": 28,
54
+ "mlp_bias": false,
55
+ "model_type": "smollm3",
56
+ "no_rope_layer_interval": 4,
57
+ "no_rope_layers": [
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 0,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 0,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 0,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 0,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 0,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 0,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 0,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 0,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 0
94
+ ],
95
+ "num_attention_heads": 16,
96
+ "num_hidden_layers": 24,
97
+ "num_key_value_heads": 16,
98
+ "pad_token_id": 0,
99
+ "pretraining_tp": 2,
100
+ "rms_norm_eps": 1e-06,
101
+ "rope_scaling": null,
102
+ "rope_theta": 5000000.0,
103
+ "sliding_window": null,
104
+ "transformers_version": "4.56.1",
105
+ "use_cache": false,
106
+ "use_sliding_window": false,
107
+ "vocab_size": 64000
108
+ }
last-checkpoint/generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "eos_token_id": 128012,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.56.1",
7
+ "use_cache": false
8
+ }
last-checkpoint/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c3670be1af3539ae8a67428522133af1e4e7e33ee61d9ee329e80d5b593414d
3
+ size 936503576
last-checkpoint/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc74174eecd2ca55845b181f90e9c90c13ec6a9387895824dc6139d4dff1a17d
3
+ size 936544523
last-checkpoint/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
3
+ size 14645
last-checkpoint/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbc31b3a35ee70b3d0c3cae9e6f20b54d89521282a10384e6af274c001e3e3ab
3
+ size 1401
last-checkpoint/trainer_state.json ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 4.445442199249165e-05,
6
+ "eval_steps": 500,
7
+ "global_step": 10,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 4.445442199249165e-06,
14
+ "grad_norm": 39.5,
15
+ "learning_rate": 2e-05,
16
+ "loss": 11.1663,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 8.89088439849833e-06,
21
+ "grad_norm": 24.0,
22
+ "learning_rate": 1.9999999999984765e-05,
23
+ "loss": 10.3397,
24
+ "step": 2
25
+ },
26
+ {
27
+ "epoch": 1.3336326597747494e-05,
28
+ "grad_norm": 18.5,
29
+ "learning_rate": 1.999999999993905e-05,
30
+ "loss": 9.3351,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 1.778176879699666e-05,
35
+ "grad_norm": 17.75,
36
+ "learning_rate": 1.9999999999862864e-05,
37
+ "loss": 9.2201,
38
+ "step": 4
39
+ },
40
+ {
41
+ "epoch": 2.2227210996245824e-05,
42
+ "grad_norm": 21.375,
43
+ "learning_rate": 1.9999999999756195e-05,
44
+ "loss": 9.3185,
45
+ "step": 5
46
+ },
47
+ {
48
+ "epoch": 2.667265319549499e-05,
49
+ "grad_norm": 14.25,
50
+ "learning_rate": 1.9999999999619057e-05,
51
+ "loss": 8.8933,
52
+ "step": 6
53
+ },
54
+ {
55
+ "epoch": 3.1118095394744156e-05,
56
+ "grad_norm": 16.75,
57
+ "learning_rate": 1.9999999999451446e-05,
58
+ "loss": 9.0228,
59
+ "step": 7
60
+ },
61
+ {
62
+ "epoch": 3.556353759399332e-05,
63
+ "grad_norm": 13.3125,
64
+ "learning_rate": 1.9999999999253356e-05,
65
+ "loss": 8.7269,
66
+ "step": 8
67
+ },
68
+ {
69
+ "epoch": 4.0008979793242484e-05,
70
+ "grad_norm": 15.5,
71
+ "learning_rate": 1.999999999902479e-05,
72
+ "loss": 9.1175,
73
+ "step": 9
74
+ },
75
+ {
76
+ "epoch": 4.445442199249165e-05,
77
+ "grad_norm": 15.8125,
78
+ "learning_rate": 1.9999999998765746e-05,
79
+ "loss": 9.1452,
80
+ "step": 10
81
+ }
82
+ ],
83
+ "logging_steps": 1,
84
+ "max_steps": 1799596,
85
+ "num_input_tokens_seen": 0,
86
+ "num_train_epochs": 8,
87
+ "save_steps": 10,
88
+ "stateful_callbacks": {
89
+ "TrainerControl": {
90
+ "args": {
91
+ "should_epoch_stop": false,
92
+ "should_evaluate": false,
93
+ "should_log": false,
94
+ "should_save": true,
95
+ "should_training_stop": false
96
+ },
97
+ "attributes": {}
98
+ }
99
+ },
100
+ "total_flos": 395873511014400.0,
101
+ "train_batch_size": 1,
102
+ "trial_name": null,
103
+ "trial_params": null
104
+ }
last-checkpoint/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0227229208d45c03cd0a5e81b699cfeee25e60733e5b5af8f25f9b48729fc0f0
3
+ size 6033