Hugging Face's logo

hauser458original
/
lfm2.5-350m-code-math

Text Generation
Safetensors
English
lfm2
lfm2.5
liquid
code
math
fine-tune
conversational
Model card Files
xet
 Community
1
hauser458original commited on
Commit
ec55f05
·
verified ·
1 Parent(s): b88fc56

Upload folder using huggingface_hub

Browse files
Files changed (41) hide show
  1. README.md +58 -0
  2. chat_template.jinja +64 -0
  3. checkpoint-4600/chat_template.jinja +64 -0
  4. checkpoint-4600/config.json +60 -0
  5. checkpoint-4600/generation_config.json +9 -0
  6. checkpoint-4600/model.safetensors +3 -0
  7. checkpoint-4600/optimizer.pt +3 -0
  8. checkpoint-4600/rng_state.pth +3 -0
  9. checkpoint-4600/scheduler.pt +3 -0
  10. checkpoint-4600/tokenizer.json +0 -0
  11. checkpoint-4600/tokenizer_config.json +21 -0
  12. checkpoint-4600/trainer_state.json +2127 -0
  13. checkpoint-4600/training_args.bin +3 -0
  14. checkpoint-5600/chat_template.jinja +64 -0
  15. checkpoint-5600/config.json +60 -0
  16. checkpoint-5600/generation_config.json +9 -0
  17. checkpoint-5600/model.safetensors +3 -0
  18. checkpoint-5600/optimizer.pt +3 -0
  19. checkpoint-5600/rng_state.pth +3 -0
  20. checkpoint-5600/scheduler.pt +3 -0
  21. checkpoint-5600/tokenizer.json +0 -0
  22. checkpoint-5600/tokenizer_config.json +21 -0
  23. checkpoint-5600/trainer_state.json +2582 -0
  24. checkpoint-5600/training_args.bin +3 -0
  25. checkpoint-5778/chat_template.jinja +64 -0
  26. checkpoint-5778/config.json +60 -0
  27. checkpoint-5778/generation_config.json +9 -0
  28. checkpoint-5778/model.safetensors +3 -0
  29. checkpoint-5778/optimizer.pt +3 -0
  30. checkpoint-5778/rng_state.pth +3 -0
  31. checkpoint-5778/scheduler.pt +3 -0
  32. checkpoint-5778/tokenizer.json +0 -0
  33. checkpoint-5778/tokenizer_config.json +21 -0
  34. checkpoint-5778/trainer_state.json +2663 -0
  35. checkpoint-5778/training_args.bin +3 -0
  36. config.json +60 -0
  37. generation_config.json +9 -0
  38. model.safetensors +3 -0
  39. tokenizer.json +0 -0
  40. tokenizer_config.json +21 -0
  41. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: LiquidAI/LFM2.5-350M
3
+ library_name: transformers
4
+ model_name: lfm2.5-350m-multilang-code-math
5
+ tags:
6
+ - generated_from_trainer
7
+ - trl
8
+ - sft
9
+ licence: license
10
+ ---
11
+
12
+ # Model Card for lfm2.5-350m-multilang-code-math
13
+
14
+ This model is a fine-tuned version of [LiquidAI/LFM2.5-350M](https://huggingface.co/LiquidAI/LFM2.5-350M).
15
+ It has been trained using [TRL](https://github.com/huggingface/trl).
16
+
17
+ ## Quick start
18
+
19
+ ```python
20
+ from transformers import pipeline
21
+
22
+ question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
23
+ generator = pipeline("text-generation", model="None", device="cuda")
24
+ output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
25
+ print(output["generated_text"])
26
+ ```
27
+
28
+ ## Training procedure
29
+
30
+
31
+
32
+
33
+ This model was trained with SFT.
34
+
35
+ ### Framework versions
36
+
37
+ - TRL: 0.24.0
38
+ - Transformers: 5.5.0
39
+ - Pytorch: 2.10.0
40
+ - Datasets: 4.3.0
41
+ - Tokenizers: 0.22.2
42
+
43
+ ## Citations
44
+
45
+
46
+
47
+ Cite TRL as:
48
+
49
+ ```bibtex
50
+ @misc{vonwerra2022trl,
51
+ title = {{TRL: Transformer Reinforcement Learning}},
52
+ author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
53
+ year = 2020,
54
+ journal = {GitHub repository},
55
+ publisher = {GitHub},
56
+ howpublished = {\url{https://github.com/huggingface/trl}}
57
+ }
58
+ ```
chat_template.jinja ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set keep_past_thinking = keep_past_thinking | default(false) -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set sys_content = messages[0]["content"] -%}
6
+ {%- if sys_content is not string -%}
7
+ {%- for item in sys_content -%}
8
+ {%- if item["type"] == "text" -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + item["text"] -%}
10
+ {%- endif -%}
11
+ {%- endfor -%}
12
+ {%- else -%}
13
+ {%- set ns.system_prompt = sys_content -%}
14
+ {%- endif -%}
15
+ {%- set messages = messages[1:] -%}
16
+ {%- endif -%}
17
+ {%- if tools -%}
18
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
19
+ {%- for tool in tools -%}
20
+ {%- if tool is not string -%}
21
+ {%- set tool = tool | tojson -%}
22
+ {%- endif -%}
23
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
24
+ {%- if not loop.last -%}
25
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- set ns.system_prompt = ns.system_prompt + "]" -%}
29
+ {%- endif -%}
30
+ {%- if ns.system_prompt -%}
31
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
32
+ {%- endif -%}
33
+ {%- set ns.last_assistant_index = -1 -%}
34
+ {%- for message in messages -%}
35
+ {%- if message["role"] == "assistant" -%}
36
+ {%- set ns.last_assistant_index = loop.index0 -%}
37
+ {%- endif -%}
38
+ {%- endfor -%}
39
+ {%- for message in messages -%}
40
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
41
+ {%- set content = message["content"] -%}
42
+ {%- if content is not string -%}
43
+ {%- set ns.content = "" -%}
44
+ {%- for item in content -%}
45
+ {%- if item["type"] == "image" -%}
46
+ {%- set ns.content = ns.content + "<image>" -%}
47
+ {%- elif item["type"] == "text" -%}
48
+ {%- set ns.content = ns.content + item["text"] -%}
49
+ {%- else -%}
50
+ {%- set ns.content = ns.content + item | tojson -%}
51
+ {%- endif -%}
52
+ {%- endfor -%}
53
+ {%- set content = ns.content -%}
54
+ {%- endif -%}
55
+ {%- if message["role"] == "assistant" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}
56
+ {%- if "</think>" in content -%}
57
+ {%- set content = content.split("</think>")[-1] | trim -%}
58
+ {%- endif -%}
59
+ {%- endif -%}
60
+ {{- content + "<|im_end|>\n" -}}
61
+ {%- endfor -%}
62
+ {%- if add_generation_prompt -%}
63
+ {{- "<|im_start|>assistant\n" -}}
64
+ {%- endif -%}
checkpoint-4600/chat_template.jinja ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set keep_past_thinking = keep_past_thinking | default(false) -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set sys_content = messages[0]["content"] -%}
6
+ {%- if sys_content is not string -%}
7
+ {%- for item in sys_content -%}
8
+ {%- if item["type"] == "text" -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + item["text"] -%}
10
+ {%- endif -%}
11
+ {%- endfor -%}
12
+ {%- else -%}
13
+ {%- set ns.system_prompt = sys_content -%}
14
+ {%- endif -%}
15
+ {%- set messages = messages[1:] -%}
16
+ {%- endif -%}
17
+ {%- if tools -%}
18
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
19
+ {%- for tool in tools -%}
20
+ {%- if tool is not string -%}
21
+ {%- set tool = tool | tojson -%}
22
+ {%- endif -%}
23
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
24
+ {%- if not loop.last -%}
25
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- set ns.system_prompt = ns.system_prompt + "]" -%}
29
+ {%- endif -%}
30
+ {%- if ns.system_prompt -%}
31
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
32
+ {%- endif -%}
33
+ {%- set ns.last_assistant_index = -1 -%}
34
+ {%- for message in messages -%}
35
+ {%- if message["role"] == "assistant" -%}
36
+ {%- set ns.last_assistant_index = loop.index0 -%}
37
+ {%- endif -%}
38
+ {%- endfor -%}
39
+ {%- for message in messages -%}
40
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
41
+ {%- set content = message["content"] -%}
42
+ {%- if content is not string -%}
43
+ {%- set ns.content = "" -%}
44
+ {%- for item in content -%}
45
+ {%- if item["type"] == "image" -%}
46
+ {%- set ns.content = ns.content + "<image>" -%}
47
+ {%- elif item["type"] == "text" -%}
48
+ {%- set ns.content = ns.content + item["text"] -%}
49
+ {%- else -%}
50
+ {%- set ns.content = ns.content + item | tojson -%}
51
+ {%- endif -%}
52
+ {%- endfor -%}
53
+ {%- set content = ns.content -%}
54
+ {%- endif -%}
55
+ {%- if message["role"] == "assistant" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}
56
+ {%- if "</think>" in content -%}
57
+ {%- set content = content.split("</think>")[-1] | trim -%}
58
+ {%- endif -%}
59
+ {%- endif -%}
60
+ {{- content + "<|im_end|>\n" -}}
61
+ {%- endfor -%}
62
+ {%- if add_generation_prompt -%}
63
+ {{- "<|im_start|>assistant\n" -}}
64
+ {%- endif -%}
checkpoint-4600/config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2ForCausalLM"
4
+ ],
5
+ "block_auto_adjust_ff_dim": true,
6
+ "block_dim": 1024,
7
+ "block_ffn_dim_multiplier": 1.0,
8
+ "block_mlp_init_scale": 1.0,
9
+ "block_multiple_of": 256,
10
+ "block_norm_eps": 1e-05,
11
+ "block_out_init_scale": 1.0,
12
+ "block_use_swiglu": true,
13
+ "block_use_xavier_init": true,
14
+ "bos_token_id": 1,
15
+ "conv_L_cache": 3,
16
+ "conv_bias": false,
17
+ "conv_dim": 1024,
18
+ "conv_use_xavier_init": true,
19
+ "dtype": "bfloat16",
20
+ "eos_token_id": 7,
21
+ "full_attn_idxs": null,
22
+ "hidden_size": 1024,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 6656,
25
+ "layer_types": [
26
+ "conv",
27
+ "conv",
28
+ "full_attention",
29
+ "conv",
30
+ "conv",
31
+ "full_attention",
32
+ "conv",
33
+ "conv",
34
+ "full_attention",
35
+ "conv",
36
+ "full_attention",
37
+ "conv",
38
+ "full_attention",
39
+ "conv",
40
+ "full_attention",
41
+ "conv"
42
+ ],
43
+ "max_position_embeddings": 128000,
44
+ "model_type": "lfm2",
45
+ "norm_eps": 1e-05,
46
+ "num_attention_heads": 16,
47
+ "num_heads": 16,
48
+ "num_hidden_layers": 16,
49
+ "num_key_value_heads": 8,
50
+ "pad_token_id": 0,
51
+ "rope_parameters": {
52
+ "rope_theta": 1000000.0,
53
+ "rope_type": "default"
54
+ },
55
+ "tie_word_embeddings": true,
56
+ "transformers_version": "5.5.0",
57
+ "use_cache": false,
58
+ "use_pos_enc": true,
59
+ "vocab_size": 65536
60
+ }
checkpoint-4600/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 7
6
+ ],
7
+ "pad_token_id": 0,
8
+ "transformers_version": "5.5.0"
9
+ }
checkpoint-4600/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3eaee205a112143b29e0fc07a362b9bd5068d377cd2d96bf537ab645b8e19098
3
+ size 708984464
checkpoint-4600/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a94f1548d649ac43815598fdfaba0a52354db6e67ed0e288421bdc110fb0c28
3
+ size 1418061003
checkpoint-4600/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:032ccdc994a6cd5743c0d83b7429c3d5708fc0f4abbefd313e29e5f9d7ab7672
3
+ size 14645
checkpoint-4600/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d178b4640ba62cf85f1c99484a7599f40ea0f03ef2bf8ee9499fcff38f2f38c
3
+ size 1465
checkpoint-4600/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-4600/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|im_end|>",
6
+ "extra_special_tokens": [],
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "model_input_names": [
10
+ "input_ids",
11
+ "attention_mask"
12
+ ],
13
+ "model_max_length": 1000000000000000019884624838656,
14
+ "model_specific_special_tokens": {},
15
+ "pad_token": "<|pad|>",
16
+ "sp_model_kwargs": {},
17
+ "spaces_between_special_tokens": false,
18
+ "tokenizer_class": "TokenizersBackend",
19
+ "use_default_system_prompt": false,
20
+ "use_fast": true
21
+ }
checkpoint-4600/trainer_state.json ADDED
@@ -0,0 +1,2127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 4600,
3
+ "best_metric": 0.7991446852684021,
4
+ "best_model_checkpoint": "lfm2.5-350m-multilang-code-math/checkpoint-4600",
5
+ "epoch": 1.592348970053661,
6
+ "eval_steps": 200,
7
+ "global_step": 4600,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2043220043182372,
14
+ "epoch": 0.008655011251514627,
15
+ "grad_norm": 23.5,
16
+ "learning_rate": 2.7586206896551725e-06,
17
+ "loss": 3.0389138793945314,
18
+ "mean_token_accuracy": 0.5589780753850937,
19
+ "num_tokens": 151155.0,
20
+ "step": 25
21
+ },
22
+ {
23
+ "entropy": 1.11516796708107,
24
+ "epoch": 0.017310022503029255,
25
+ "grad_norm": 7.34375,
26
+ "learning_rate": 5.6321839080459775e-06,
27
+ "loss": 1.8928765869140625,
28
+ "mean_token_accuracy": 0.6762128543853759,
29
+ "num_tokens": 310957.0,
30
+ "step": 50
31
+ },
32
+ {
33
+ "entropy": 1.2031074380874633,
34
+ "epoch": 0.02596503375454388,
35
+ "grad_norm": 3.78125,
36
+ "learning_rate": 8.505747126436782e-06,
37
+ "loss": 1.2897039794921874,
38
+ "mean_token_accuracy": 0.7282541489601135,
39
+ "num_tokens": 461068.0,
40
+ "step": 75
41
+ },
42
+ {
43
+ "entropy": 1.0323714792728425,
44
+ "epoch": 0.03462004500605851,
45
+ "grad_norm": 3.71875,
46
+ "learning_rate": 1.1379310344827587e-05,
47
+ "loss": 1.0183975982666016,
48
+ "mean_token_accuracy": 0.757768098115921,
49
+ "num_tokens": 615118.0,
50
+ "step": 100
51
+ },
52
+ {
53
+ "entropy": 0.9247142732143402,
54
+ "epoch": 0.043275056257573136,
55
+ "grad_norm": 2.578125,
56
+ "learning_rate": 1.4252873563218392e-05,
57
+ "loss": 0.8757442474365235,
58
+ "mean_token_accuracy": 0.7792941844463348,
59
+ "num_tokens": 762906.0,
60
+ "step": 125
61
+ },
62
+ {
63
+ "entropy": 0.9190187740325928,
64
+ "epoch": 0.05193006750908776,
65
+ "grad_norm": 2.296875,
66
+ "learning_rate": 1.7126436781609197e-05,
67
+ "loss": 0.891803970336914,
68
+ "mean_token_accuracy": 0.7753661525249481,
69
+ "num_tokens": 921356.0,
70
+ "step": 150
71
+ },
72
+ {
73
+ "entropy": 0.886636551618576,
74
+ "epoch": 0.06058507876060239,
75
+ "grad_norm": 2.296875,
76
+ "learning_rate": 2e-05,
77
+ "loss": 0.861456298828125,
78
+ "mean_token_accuracy": 0.7826532816886902,
79
+ "num_tokens": 1076183.0,
80
+ "step": 175
81
+ },
82
+ {
83
+ "entropy": 0.9015009069442749,
84
+ "epoch": 0.06924009001211702,
85
+ "grad_norm": 2.5625,
86
+ "learning_rate": 1.9999017921043748e-05,
87
+ "loss": 0.8748048400878906,
88
+ "mean_token_accuracy": 0.7792416322231293,
89
+ "num_tokens": 1229170.0,
90
+ "step": 200
91
+ },
92
+ {
93
+ "epoch": 0.06924009001211702,
94
+ "eval_entropy": 0.8920288547140653,
95
+ "eval_loss": 0.8666635155677795,
96
+ "eval_mean_token_accuracy": 0.7797926357535065,
97
+ "eval_num_tokens": 1229170.0,
98
+ "eval_runtime": 16.3134,
99
+ "eval_samples_per_second": 298.222,
100
+ "eval_steps_per_second": 18.696,
101
+ "step": 200
102
+ },
103
+ {
104
+ "entropy": 0.9062747704982758,
105
+ "epoch": 0.07789510126363164,
106
+ "grad_norm": 2.40625,
107
+ "learning_rate": 1.999607187707079e-05,
108
+ "loss": 0.8789938354492187,
109
+ "mean_token_accuracy": 0.7778702569007874,
110
+ "num_tokens": 1381380.0,
111
+ "step": 225
112
+ },
113
+ {
114
+ "entropy": 0.8776320171356201,
115
+ "epoch": 0.08655011251514627,
116
+ "grad_norm": 2.46875,
117
+ "learning_rate": 1.9991162446730694e-05,
118
+ "loss": 0.8517158508300782,
119
+ "mean_token_accuracy": 0.7819743084907532,
120
+ "num_tokens": 1538186.0,
121
+ "step": 250
122
+ },
123
+ {
124
+ "entropy": 0.8657179689407348,
125
+ "epoch": 0.09520512376666089,
126
+ "grad_norm": 2.25,
127
+ "learning_rate": 1.9984290594313102e-05,
128
+ "loss": 0.8408647155761719,
129
+ "mean_token_accuracy": 0.7840179252624512,
130
+ "num_tokens": 1690067.0,
131
+ "step": 275
132
+ },
133
+ {
134
+ "entropy": 0.8955290937423706,
135
+ "epoch": 0.10386013501817552,
136
+ "grad_norm": 2.328125,
137
+ "learning_rate": 1.9975457669558346e-05,
138
+ "loss": 0.8683735656738282,
139
+ "mean_token_accuracy": 0.7763688516616821,
140
+ "num_tokens": 1845254.0,
141
+ "step": 300
142
+ },
143
+ {
144
+ "entropy": 0.8744900381565094,
145
+ "epoch": 0.11251514626969016,
146
+ "grad_norm": 2.34375,
147
+ "learning_rate": 1.996466540739233e-05,
148
+ "loss": 0.8470248413085938,
149
+ "mean_token_accuracy": 0.7828968441486359,
150
+ "num_tokens": 2000855.0,
151
+ "step": 325
152
+ },
153
+ {
154
+ "entropy": 0.8875207018852234,
155
+ "epoch": 0.12117015752120477,
156
+ "grad_norm": 2.34375,
157
+ "learning_rate": 1.9951915927585762e-05,
158
+ "loss": 0.8633122253417969,
159
+ "mean_token_accuracy": 0.7807920384407043,
160
+ "num_tokens": 2159255.0,
161
+ "step": 350
162
+ },
163
+ {
164
+ "entropy": 0.8727393555641174,
165
+ "epoch": 0.1298251687727194,
166
+ "grad_norm": 2.40625,
167
+ "learning_rate": 1.9937211734337812e-05,
168
+ "loss": 0.8484575653076172,
169
+ "mean_token_accuracy": 0.7856597316265106,
170
+ "num_tokens": 2313980.0,
171
+ "step": 375
172
+ },
173
+ {
174
+ "entropy": 0.8450514304637909,
175
+ "epoch": 0.13848018002423404,
176
+ "grad_norm": 2.390625,
177
+ "learning_rate": 1.992055571578423e-05,
178
+ "loss": 0.8231178283691406,
179
+ "mean_token_accuracy": 0.7884540009498596,
180
+ "num_tokens": 2468392.0,
181
+ "step": 400
182
+ },
183
+ {
184
+ "epoch": 0.13848018002423404,
185
+ "eval_entropy": 0.8820633643963298,
186
+ "eval_loss": 0.8462674021720886,
187
+ "eval_mean_token_accuracy": 0.7830698802822926,
188
+ "eval_num_tokens": 2468392.0,
189
+ "eval_runtime": 16.2902,
190
+ "eval_samples_per_second": 298.645,
191
+ "eval_steps_per_second": 18.723,
192
+ "step": 400
193
+ },
194
+ {
195
+ "entropy": 0.8852161896228791,
196
+ "epoch": 0.14713519127574864,
197
+ "grad_norm": 2.25,
198
+ "learning_rate": 1.990195114343008e-05,
199
+ "loss": 0.8604666137695313,
200
+ "mean_token_accuracy": 0.7796429479122162,
201
+ "num_tokens": 2626504.0,
202
+ "step": 425
203
+ },
204
+ {
205
+ "entropy": 0.8595891773700715,
206
+ "epoch": 0.15579020252726328,
207
+ "grad_norm": 2.109375,
208
+ "learning_rate": 1.988140167150716e-05,
209
+ "loss": 0.8375725555419922,
210
+ "mean_token_accuracy": 0.7837563192844391,
211
+ "num_tokens": 2785992.0,
212
+ "step": 450
213
+ },
214
+ {
215
+ "entropy": 0.8787437427043915,
216
+ "epoch": 0.1644452137787779,
217
+ "grad_norm": 2.21875,
218
+ "learning_rate": 1.9858911336256258e-05,
219
+ "loss": 0.8602984619140625,
220
+ "mean_token_accuracy": 0.7801311421394348,
221
+ "num_tokens": 2943389.0,
222
+ "step": 475
223
+ },
224
+ {
225
+ "entropy": 0.8887673962116241,
226
+ "epoch": 0.17310022503029254,
227
+ "grad_norm": 2.34375,
228
+ "learning_rate": 1.983448455513437e-05,
229
+ "loss": 0.8559275054931641,
230
+ "mean_token_accuracy": 0.7799410057067871,
231
+ "num_tokens": 3098910.0,
232
+ "step": 500
233
+ },
234
+ {
235
+ "entropy": 0.8632562351226807,
236
+ "epoch": 0.18175523628180718,
237
+ "grad_norm": 2.359375,
238
+ "learning_rate": 1.9808126125947035e-05,
239
+ "loss": 0.8414566040039062,
240
+ "mean_token_accuracy": 0.7838216722011566,
241
+ "num_tokens": 3259790.0,
242
+ "step": 525
243
+ },
244
+ {
245
+ "entropy": 0.8468562078475952,
246
+ "epoch": 0.19041024753332178,
247
+ "grad_norm": 2.921875,
248
+ "learning_rate": 1.977984122590598e-05,
249
+ "loss": 0.8214312744140625,
250
+ "mean_token_accuracy": 0.7904518485069275,
251
+ "num_tokens": 3408921.0,
252
+ "step": 550
253
+ },
254
+ {
255
+ "entropy": 0.862267234325409,
256
+ "epoch": 0.1990652587848364,
257
+ "grad_norm": 2.21875,
258
+ "learning_rate": 1.9749635410612222e-05,
259
+ "loss": 0.8381607818603516,
260
+ "mean_token_accuracy": 0.7850737714767456,
261
+ "num_tokens": 3564021.0,
262
+ "step": 575
263
+ },
264
+ {
265
+ "entropy": 0.8673117184638977,
266
+ "epoch": 0.20772027003635105,
267
+ "grad_norm": 2.28125,
268
+ "learning_rate": 1.971751461296488e-05,
269
+ "loss": 0.8422865295410156,
270
+ "mean_token_accuracy": 0.7839160299301148,
271
+ "num_tokens": 3720222.0,
272
+ "step": 600
273
+ },
274
+ {
275
+ "epoch": 0.20772027003635105,
276
+ "eval_entropy": 0.8581681211463741,
277
+ "eval_loss": 0.8380036950111389,
278
+ "eval_mean_token_accuracy": 0.7849426593936858,
279
+ "eval_num_tokens": 3720222.0,
280
+ "eval_runtime": 16.2659,
281
+ "eval_samples_per_second": 299.091,
282
+ "eval_steps_per_second": 18.751,
283
+ "step": 600
284
+ },
285
+ {
286
+ "entropy": 0.896509393453598,
287
+ "epoch": 0.21637528128786568,
288
+ "grad_norm": 2.25,
289
+ "learning_rate": 1.9683485141995832e-05,
290
+ "loss": 0.8751831817626953,
291
+ "mean_token_accuracy": 0.778223477602005,
292
+ "num_tokens": 3868288.0,
293
+ "step": 625
294
+ },
295
+ {
296
+ "entropy": 0.8858666527271271,
297
+ "epoch": 0.2250302925393803,
298
+ "grad_norm": 2.4375,
299
+ "learning_rate": 1.964755368163055e-05,
300
+ "loss": 0.8597821807861328,
301
+ "mean_token_accuracy": 0.7796040976047516,
302
+ "num_tokens": 4020387.0,
303
+ "step": 650
304
+ },
305
+ {
306
+ "entropy": 0.8457015335559845,
307
+ "epoch": 0.23368530379089492,
308
+ "grad_norm": 2.0625,
309
+ "learning_rate": 1.9609727289375254e-05,
310
+ "loss": 0.8197212982177734,
311
+ "mean_token_accuracy": 0.7878750622272491,
312
+ "num_tokens": 4175769.0,
313
+ "step": 675
314
+ },
315
+ {
316
+ "entropy": 0.8535351765155792,
317
+ "epoch": 0.24234031504240955,
318
+ "grad_norm": 2.328125,
319
+ "learning_rate": 1.9570013394930704e-05,
320
+ "loss": 0.8300620269775391,
321
+ "mean_token_accuracy": 0.786250706911087,
322
+ "num_tokens": 4331114.0,
323
+ "step": 700
324
+ },
325
+ {
326
+ "entropy": 0.8614680600166321,
327
+ "epoch": 0.2509953262939242,
328
+ "grad_norm": 2.0625,
329
+ "learning_rate": 1.9528419798732902e-05,
330
+ "loss": 0.8407713317871094,
331
+ "mean_token_accuracy": 0.7833027255535125,
332
+ "num_tokens": 4488237.0,
333
+ "step": 725
334
+ },
335
+ {
336
+ "entropy": 0.8501963889598847,
337
+ "epoch": 0.2596503375454388,
338
+ "grad_norm": 2.328125,
339
+ "learning_rate": 1.948495467042095e-05,
340
+ "loss": 0.8218775177001953,
341
+ "mean_token_accuracy": 0.788744968175888,
342
+ "num_tokens": 4642068.0,
343
+ "step": 750
344
+ },
345
+ {
346
+ "entropy": 0.8383999621868133,
347
+ "epoch": 0.26830534879695345,
348
+ "grad_norm": 2.296875,
349
+ "learning_rate": 1.9439626547232433e-05,
350
+ "loss": 0.80978515625,
351
+ "mean_token_accuracy": 0.7903480684757233,
352
+ "num_tokens": 4788314.0,
353
+ "step": 775
354
+ },
355
+ {
356
+ "entropy": 0.8568655729293824,
357
+ "epoch": 0.2769603600484681,
358
+ "grad_norm": 2.53125,
359
+ "learning_rate": 1.939244433232652e-05,
360
+ "loss": 0.8343601226806641,
361
+ "mean_token_accuracy": 0.7855056190490722,
362
+ "num_tokens": 4941131.0,
363
+ "step": 800
364
+ },
365
+ {
366
+ "epoch": 0.2769603600484681,
367
+ "eval_entropy": 0.859573404906226,
368
+ "eval_loss": 0.8307726383209229,
369
+ "eval_mean_token_accuracy": 0.7864908138259512,
370
+ "eval_num_tokens": 4941131.0,
371
+ "eval_runtime": 16.2516,
372
+ "eval_samples_per_second": 299.355,
373
+ "eval_steps_per_second": 18.767,
374
+ "step": 800
375
+ },
376
+ {
377
+ "entropy": 0.8670335674285888,
378
+ "epoch": 0.2856153712999827,
379
+ "grad_norm": 1.9765625,
380
+ "learning_rate": 1.934341729303529e-05,
381
+ "loss": 0.8394245910644531,
382
+ "mean_token_accuracy": 0.7841019070148468,
383
+ "num_tokens": 5096940.0,
384
+ "step": 825
385
+ },
386
+ {
387
+ "entropy": 0.867160736322403,
388
+ "epoch": 0.2942703825514973,
389
+ "grad_norm": 2.21875,
390
+ "learning_rate": 1.9292555059043457e-05,
391
+ "loss": 0.8409418487548828,
392
+ "mean_token_accuracy": 0.7830280113220215,
393
+ "num_tokens": 5250616.0,
394
+ "step": 850
395
+ },
396
+ {
397
+ "entropy": 0.8582142984867096,
398
+ "epoch": 0.3029253938030119,
399
+ "grad_norm": 2.15625,
400
+ "learning_rate": 1.9239867620496952e-05,
401
+ "loss": 0.8346880340576172,
402
+ "mean_token_accuracy": 0.783586630821228,
403
+ "num_tokens": 5408253.0,
404
+ "step": 875
405
+ },
406
+ {
407
+ "entropy": 0.84959406375885,
408
+ "epoch": 0.31158040505452655,
409
+ "grad_norm": 2.1875,
410
+ "learning_rate": 1.9185365326040713e-05,
411
+ "loss": 0.826163330078125,
412
+ "mean_token_accuracy": 0.7881129395961761,
413
+ "num_tokens": 5564921.0,
414
+ "step": 900
415
+ },
416
+ {
417
+ "entropy": 0.8895779585838318,
418
+ "epoch": 0.3202354163060412,
419
+ "grad_norm": 2.1875,
420
+ "learning_rate": 1.9129058880786026e-05,
421
+ "loss": 0.866309585571289,
422
+ "mean_token_accuracy": 0.7782393181324005,
423
+ "num_tokens": 5722704.0,
424
+ "step": 925
425
+ },
426
+ {
427
+ "entropy": 0.8372803354263305,
428
+ "epoch": 0.3288904275575558,
429
+ "grad_norm": 1.890625,
430
+ "learning_rate": 1.907095934420789e-05,
431
+ "loss": 0.8083054351806641,
432
+ "mean_token_accuracy": 0.7903852713108063,
433
+ "num_tokens": 5881153.0,
434
+ "step": 950
435
+ },
436
+ {
437
+ "entropy": 0.8762626206874847,
438
+ "epoch": 0.33754543880907045,
439
+ "grad_norm": 1.96875,
440
+ "learning_rate": 1.9011078127972748e-05,
441
+ "loss": 0.8495908355712891,
442
+ "mean_token_accuracy": 0.7841223049163818,
443
+ "num_tokens": 6039323.0,
444
+ "step": 975
445
+ },
446
+ {
447
+ "entropy": 0.8670505321025849,
448
+ "epoch": 0.3462004500605851,
449
+ "grad_norm": 2.25,
450
+ "learning_rate": 1.8949426993697074e-05,
451
+ "loss": 0.8448488616943359,
452
+ "mean_token_accuracy": 0.785959769487381,
453
+ "num_tokens": 6197683.0,
454
+ "step": 1000
455
+ },
456
+ {
457
+ "epoch": 0.3462004500605851,
458
+ "eval_entropy": 0.8536441159052927,
459
+ "eval_loss": 0.8251546025276184,
460
+ "eval_mean_token_accuracy": 0.7873384729760592,
461
+ "eval_num_tokens": 6197683.0,
462
+ "eval_runtime": 16.2591,
463
+ "eval_samples_per_second": 299.217,
464
+ "eval_steps_per_second": 18.759,
465
+ "step": 1000
466
+ },
467
+ {
468
+ "entropy": 0.8527129173278809,
469
+ "epoch": 0.3548554613120997,
470
+ "grad_norm": 2.171875,
471
+ "learning_rate": 1.888601805063719e-05,
472
+ "loss": 0.8221394348144532,
473
+ "mean_token_accuracy": 0.7878592002391815,
474
+ "num_tokens": 6353591.0,
475
+ "step": 1025
476
+ },
477
+ {
478
+ "entropy": 0.8309027600288391,
479
+ "epoch": 0.36351047256361435,
480
+ "grad_norm": 2.1875,
481
+ "learning_rate": 1.8820863753310812e-05,
482
+ "loss": 0.804949951171875,
483
+ "mean_token_accuracy": 0.7907659840583802,
484
+ "num_tokens": 6503522.0,
485
+ "step": 1050
486
+ },
487
+ {
488
+ "entropy": 0.8664905667304993,
489
+ "epoch": 0.372165483815129,
490
+ "grad_norm": 2.234375,
491
+ "learning_rate": 1.875397689905081e-05,
492
+ "loss": 0.8386592102050782,
493
+ "mean_token_accuracy": 0.783461629152298,
494
+ "num_tokens": 6657038.0,
495
+ "step": 1075
496
+ },
497
+ {
498
+ "entropy": 0.8521995890140533,
499
+ "epoch": 0.38082049506664356,
500
+ "grad_norm": 2.140625,
501
+ "learning_rate": 1.8685370625491582e-05,
502
+ "loss": 0.8274839782714843,
503
+ "mean_token_accuracy": 0.7857646262645721,
504
+ "num_tokens": 6813403.0,
505
+ "step": 1100
506
+ },
507
+ {
508
+ "entropy": 0.85324840426445,
509
+ "epoch": 0.3894755063181582,
510
+ "grad_norm": 2.03125,
511
+ "learning_rate": 1.861505840798864e-05,
512
+ "loss": 0.8251450347900391,
513
+ "mean_token_accuracy": 0.7872388279438018,
514
+ "num_tokens": 6969337.0,
515
+ "step": 1125
516
+ },
517
+ {
518
+ "entropy": 0.8578063642978668,
519
+ "epoch": 0.3981305175696728,
520
+ "grad_norm": 2.09375,
521
+ "learning_rate": 1.8543054056971815e-05,
522
+ "loss": 0.834039077758789,
523
+ "mean_token_accuracy": 0.7848770570755005,
524
+ "num_tokens": 7124334.0,
525
+ "step": 1150
526
+ },
527
+ {
528
+ "entropy": 0.8491312241554261,
529
+ "epoch": 0.40678552882118746,
530
+ "grad_norm": 2.0625,
531
+ "learning_rate": 1.8469371715232686e-05,
532
+ "loss": 0.8220222473144532,
533
+ "mean_token_accuracy": 0.7871887254714965,
534
+ "num_tokens": 7277475.0,
535
+ "step": 1175
536
+ },
537
+ {
538
+ "entropy": 0.8443890881538391,
539
+ "epoch": 0.4154405400727021,
540
+ "grad_norm": 2.203125,
541
+ "learning_rate": 1.839402585514671e-05,
542
+ "loss": 0.8166568756103516,
543
+ "mean_token_accuracy": 0.7897883796691895,
544
+ "num_tokens": 7428616.0,
545
+ "step": 1200
546
+ },
547
+ {
548
+ "epoch": 0.4154405400727021,
549
+ "eval_entropy": 0.8290919256014903,
550
+ "eval_loss": 0.8211944103240967,
551
+ "eval_mean_token_accuracy": 0.788363546621604,
552
+ "eval_num_tokens": 7428616.0,
553
+ "eval_runtime": 16.2451,
554
+ "eval_samples_per_second": 299.474,
555
+ "eval_steps_per_second": 18.775,
556
+ "step": 1200
557
+ },
558
+ {
559
+ "entropy": 0.8553554141521453,
560
+ "epoch": 0.4240955513242167,
561
+ "grad_norm": 2.140625,
562
+ "learning_rate": 1.8317031275830607e-05,
563
+ "loss": 0.8335649108886719,
564
+ "mean_token_accuracy": 0.7848791778087616,
565
+ "num_tokens": 7586653.0,
566
+ "step": 1225
567
+ },
568
+ {
569
+ "entropy": 0.8513217878341675,
570
+ "epoch": 0.43275056257573136,
571
+ "grad_norm": 2.078125,
572
+ "learning_rate": 1.8238403100235602e-05,
573
+ "loss": 0.8226930999755859,
574
+ "mean_token_accuracy": 0.7892848730087281,
575
+ "num_tokens": 7736619.0,
576
+ "step": 1250
577
+ },
578
+ {
579
+ "entropy": 0.8567298305034637,
580
+ "epoch": 0.441405573827246,
581
+ "grad_norm": 2.25,
582
+ "learning_rate": 1.8158156772177015e-05,
583
+ "loss": 0.8301739501953125,
584
+ "mean_token_accuracy": 0.7857687485218048,
585
+ "num_tokens": 7895674.0,
586
+ "step": 1275
587
+ },
588
+ {
589
+ "entropy": 0.8383160126209259,
590
+ "epoch": 0.4500605850787606,
591
+ "grad_norm": 2.28125,
592
+ "learning_rate": 1.807630805330087e-05,
593
+ "loss": 0.8143462371826172,
594
+ "mean_token_accuracy": 0.7904347920417786,
595
+ "num_tokens": 8052368.0,
596
+ "step": 1300
597
+ },
598
+ {
599
+ "entropy": 0.858016027212143,
600
+ "epoch": 0.45871559633027525,
601
+ "grad_norm": 2.328125,
602
+ "learning_rate": 1.7992873019988044e-05,
603
+ "loss": 0.8361105346679687,
604
+ "mean_token_accuracy": 0.7837494373321533,
605
+ "num_tokens": 8209037.0,
606
+ "step": 1325
607
+ },
608
+ {
609
+ "entropy": 0.8485116994380951,
610
+ "epoch": 0.46737060758178983,
611
+ "grad_norm": 2.078125,
612
+ "learning_rate": 1.7907868060196627e-05,
613
+ "loss": 0.8189002990722656,
614
+ "mean_token_accuracy": 0.7860724258422852,
615
+ "num_tokens": 8366536.0,
616
+ "step": 1350
617
+ },
618
+ {
619
+ "entropy": 0.8689926111698151,
620
+ "epoch": 0.47602561883330446,
621
+ "grad_norm": 2.28125,
622
+ "learning_rate": 1.7821309870243056e-05,
623
+ "loss": 0.8450764465332031,
624
+ "mean_token_accuracy": 0.7817527639865876,
625
+ "num_tokens": 8524698.0,
626
+ "step": 1375
627
+ },
628
+ {
629
+ "entropy": 0.8510251712799072,
630
+ "epoch": 0.4846806300848191,
631
+ "grad_norm": 2.03125,
632
+ "learning_rate": 1.77332154515227e-05,
633
+ "loss": 0.8352996063232422,
634
+ "mean_token_accuracy": 0.786122510433197,
635
+ "num_tokens": 8680719.0,
636
+ "step": 1400
637
+ },
638
+ {
639
+ "epoch": 0.4846806300848191,
640
+ "eval_entropy": 0.8555903234442727,
641
+ "eval_loss": 0.8170819878578186,
642
+ "eval_mean_token_accuracy": 0.7890570513537672,
643
+ "eval_num_tokens": 8680719.0,
644
+ "eval_runtime": 16.2534,
645
+ "eval_samples_per_second": 299.322,
646
+ "eval_steps_per_second": 18.765,
647
+ "step": 1400
648
+ },
649
+ {
650
+ "entropy": 0.8443090248107911,
651
+ "epoch": 0.49333564133633373,
652
+ "grad_norm": 2.34375,
653
+ "learning_rate": 1.764360210717051e-05,
654
+ "loss": 0.8160585784912109,
655
+ "mean_token_accuracy": 0.7888865935802459,
656
+ "num_tokens": 8839183.0,
657
+ "step": 1425
658
+ },
659
+ {
660
+ "entropy": 0.8397502684593201,
661
+ "epoch": 0.5019906525878484,
662
+ "grad_norm": 2.015625,
663
+ "learning_rate": 1.7552487438662432e-05,
664
+ "loss": 0.8148053741455078,
665
+ "mean_token_accuracy": 0.7885977244377136,
666
+ "num_tokens": 8995586.0,
667
+ "step": 1450
668
+ },
669
+ {
670
+ "entropy": 0.8441075325012207,
671
+ "epoch": 0.5106456638393629,
672
+ "grad_norm": 2.34375,
673
+ "learning_rate": 1.7459889342358176e-05,
674
+ "loss": 0.8217481994628906,
675
+ "mean_token_accuracy": 0.7888561117649079,
676
+ "num_tokens": 9146595.0,
677
+ "step": 1475
678
+ },
679
+ {
680
+ "entropy": 0.8394048309326172,
681
+ "epoch": 0.5193006750908776,
682
+ "grad_norm": 2.296875,
683
+ "learning_rate": 1.736582600598609e-05,
684
+ "loss": 0.8105857849121094,
685
+ "mean_token_accuracy": 0.7910154783725738,
686
+ "num_tokens": 9297789.0,
687
+ "step": 1500
688
+ },
689
+ {
690
+ "entropy": 0.8256420779228211,
691
+ "epoch": 0.5279556863423922,
692
+ "grad_norm": 2.140625,
693
+ "learning_rate": 1.7270315905070822e-05,
694
+ "loss": 0.7967173004150391,
695
+ "mean_token_accuracy": 0.7917508590221405,
696
+ "num_tokens": 9450532.0,
697
+ "step": 1525
698
+ },
699
+ {
700
+ "entropy": 0.817123122215271,
701
+ "epoch": 0.5366106975939069,
702
+ "grad_norm": 1.8828125,
703
+ "learning_rate": 1.717337779930441e-05,
704
+ "loss": 0.7964045715332031,
705
+ "mean_token_accuracy": 0.793145546913147,
706
+ "num_tokens": 9605286.0,
707
+ "step": 1550
708
+ },
709
+ {
710
+ "entropy": 0.8218163585662842,
711
+ "epoch": 0.5452657088454215,
712
+ "grad_norm": 2.25,
713
+ "learning_rate": 1.7075030728861602e-05,
714
+ "loss": 0.7938800048828125,
715
+ "mean_token_accuracy": 0.7939867222309113,
716
+ "num_tokens": 9755402.0,
717
+ "step": 1575
718
+ },
719
+ {
720
+ "entropy": 0.8473324692249298,
721
+ "epoch": 0.5539207200969362,
722
+ "grad_norm": 2.171875,
723
+ "learning_rate": 1.697529401066006e-05,
724
+ "loss": 0.8210481262207031,
725
+ "mean_token_accuracy": 0.7879379904270172,
726
+ "num_tokens": 9915397.0,
727
+ "step": 1600
728
+ },
729
+ {
730
+ "epoch": 0.5539207200969362,
731
+ "eval_entropy": 0.8279588664164309,
732
+ "eval_loss": 0.8136925101280212,
733
+ "eval_mean_token_accuracy": 0.7897968792524495,
734
+ "eval_num_tokens": 9915397.0,
735
+ "eval_runtime": 16.2401,
736
+ "eval_samples_per_second": 299.567,
737
+ "eval_steps_per_second": 18.781,
738
+ "step": 1600
739
+ },
740
+ {
741
+ "entropy": 0.83288947224617,
742
+ "epoch": 0.5625757313484507,
743
+ "grad_norm": 2.125,
744
+ "learning_rate": 1.6874187234566198e-05,
745
+ "loss": 0.8104652404785156,
746
+ "mean_token_accuracy": 0.7881759464740753,
747
+ "num_tokens": 10066176.0,
748
+ "step": 1625
749
+ },
750
+ {
751
+ "entropy": 0.8406240797042847,
752
+ "epoch": 0.5712307425999654,
753
+ "grad_norm": 2.21875,
754
+ "learning_rate": 1.677173025954745e-05,
755
+ "loss": 0.8155978393554687,
756
+ "mean_token_accuracy": 0.7885338914394379,
757
+ "num_tokens": 10216630.0,
758
+ "step": 1650
759
+ },
760
+ {
761
+ "entropy": 0.8361802196502686,
762
+ "epoch": 0.57988575385148,
763
+ "grad_norm": 2.078125,
764
+ "learning_rate": 1.6667943209771634e-05,
765
+ "loss": 0.8124593353271484,
766
+ "mean_token_accuracy": 0.7909375095367431,
767
+ "num_tokens": 10368775.0,
768
+ "step": 1675
769
+ },
770
+ {
771
+ "entropy": 0.8546354687213897,
772
+ "epoch": 0.5885407651029946,
773
+ "grad_norm": 2.390625,
774
+ "learning_rate": 1.6562846470654253e-05,
775
+ "loss": 0.8320878601074219,
776
+ "mean_token_accuracy": 0.7862278306484223,
777
+ "num_tokens": 10522306.0,
778
+ "step": 1700
779
+ },
780
+ {
781
+ "entropy": 0.8271597623825073,
782
+ "epoch": 0.5971957763545093,
783
+ "grad_norm": 2.359375,
784
+ "learning_rate": 1.6456460684854474e-05,
785
+ "loss": 0.802950210571289,
786
+ "mean_token_accuracy": 0.7914960932731628,
787
+ "num_tokens": 10677278.0,
788
+ "step": 1725
789
+ },
790
+ {
791
+ "entropy": 0.8680684280395508,
792
+ "epoch": 0.6058507876060238,
793
+ "grad_norm": 2.21875,
794
+ "learning_rate": 1.6348806748220596e-05,
795
+ "loss": 0.8468040466308594,
796
+ "mean_token_accuracy": 0.7825022864341736,
797
+ "num_tokens": 10832759.0,
798
+ "step": 1750
799
+ },
800
+ {
801
+ "entropy": 0.8460379076004029,
802
+ "epoch": 0.6145057988575385,
803
+ "grad_norm": 1.90625,
804
+ "learning_rate": 1.6239905805685766e-05,
805
+ "loss": 0.81728515625,
806
+ "mean_token_accuracy": 0.7885628819465638,
807
+ "num_tokens": 10987982.0,
808
+ "step": 1775
809
+ },
810
+ {
811
+ "entropy": 0.8258041095733643,
812
+ "epoch": 0.6231608101090531,
813
+ "grad_norm": 1.9609375,
814
+ "learning_rate": 1.6129779247114776e-05,
815
+ "loss": 0.8052913665771484,
816
+ "mean_token_accuracy": 0.7923011314868927,
817
+ "num_tokens": 11142409.0,
818
+ "step": 1800
819
+ },
820
+ {
821
+ "epoch": 0.6231608101090531,
822
+ "eval_entropy": 0.8428774887421092,
823
+ "eval_loss": 0.8103926181793213,
824
+ "eval_mean_token_accuracy": 0.790385250967057,
825
+ "eval_num_tokens": 11142409.0,
826
+ "eval_runtime": 16.2616,
827
+ "eval_samples_per_second": 299.17,
828
+ "eval_steps_per_second": 18.756,
829
+ "step": 1800
830
+ },
831
+ {
832
+ "entropy": 0.8314215159416198,
833
+ "epoch": 0.6318158213605678,
834
+ "grad_norm": 2.09375,
835
+ "learning_rate": 1.601844870310277e-05,
836
+ "loss": 0.8011739349365234,
837
+ "mean_token_accuracy": 0.7905190980434418,
838
+ "num_tokens": 11295629.0,
839
+ "step": 1825
840
+ },
841
+ {
842
+ "entropy": 0.8174971842765808,
843
+ "epoch": 0.6404708326120824,
844
+ "grad_norm": 2.03125,
845
+ "learning_rate": 1.5905936040726636e-05,
846
+ "loss": 0.7869554138183594,
847
+ "mean_token_accuracy": 0.7940086150169372,
848
+ "num_tokens": 11449328.0,
849
+ "step": 1850
850
+ },
851
+ {
852
+ "entropy": 0.8386989748477935,
853
+ "epoch": 0.6491258438635971,
854
+ "grad_norm": 2.03125,
855
+ "learning_rate": 1.579226335924999e-05,
856
+ "loss": 0.8210865783691407,
857
+ "mean_token_accuracy": 0.7893707096576691,
858
+ "num_tokens": 11605530.0,
859
+ "step": 1875
860
+ },
861
+ {
862
+ "entropy": 0.8110369992256165,
863
+ "epoch": 0.6577808551151116,
864
+ "grad_norm": 2.3125,
865
+ "learning_rate": 1.5677452985782493e-05,
866
+ "loss": 0.7894587707519531,
867
+ "mean_token_accuracy": 0.7973484849929809,
868
+ "num_tokens": 11760213.0,
869
+ "step": 1900
870
+ },
871
+ {
872
+ "entropy": 0.8280953919887543,
873
+ "epoch": 0.6664358663666263,
874
+ "grad_norm": 2.203125,
875
+ "learning_rate": 1.5561527470894503e-05,
876
+ "loss": 0.7997794342041016,
877
+ "mean_token_accuracy": 0.7926359629631042,
878
+ "num_tokens": 11913380.0,
879
+ "step": 1925
880
+ },
881
+ {
882
+ "entropy": 0.8196328341960907,
883
+ "epoch": 0.6750908776181409,
884
+ "grad_norm": 2.171875,
885
+ "learning_rate": 1.544450958418775e-05,
886
+ "loss": 0.7956825256347656,
887
+ "mean_token_accuracy": 0.7946672177314759,
888
+ "num_tokens": 12065793.0,
889
+ "step": 1950
890
+ },
891
+ {
892
+ "entropy": 0.8513234782218934,
893
+ "epoch": 0.6837458888696555,
894
+ "grad_norm": 1.8671875,
895
+ "learning_rate": 1.5326422309823046e-05,
896
+ "loss": 0.8301435089111329,
897
+ "mean_token_accuracy": 0.7851076364517212,
898
+ "num_tokens": 12229815.0,
899
+ "step": 1975
900
+ },
901
+ {
902
+ "entropy": 0.8259239780902863,
903
+ "epoch": 0.6924009001211702,
904
+ "grad_norm": 2.078125,
905
+ "learning_rate": 1.5207288842005813e-05,
906
+ "loss": 0.8005783081054687,
907
+ "mean_token_accuracy": 0.79151895403862,
908
+ "num_tokens": 12391391.0,
909
+ "step": 2000
910
+ },
911
+ {
912
+ "epoch": 0.6924009001211702,
913
+ "eval_entropy": 0.828711828149733,
914
+ "eval_loss": 0.8071080446243286,
915
+ "eval_mean_token_accuracy": 0.7911858433582744,
916
+ "eval_num_tokens": 12391391.0,
917
+ "eval_runtime": 16.2568,
918
+ "eval_samples_per_second": 299.259,
919
+ "eval_steps_per_second": 18.761,
920
+ "step": 2000
921
+ },
922
+ {
923
+ "entropy": 0.8361215782165528,
924
+ "epoch": 0.7010559113726847,
925
+ "grad_norm": 2.109375,
926
+ "learning_rate": 1.508713258043041e-05,
927
+ "loss": 0.8149121856689453,
928
+ "mean_token_accuracy": 0.7890836715698242,
929
+ "num_tokens": 12548530.0,
930
+ "step": 2025
931
+ },
932
+ {
933
+ "entropy": 0.8404046738147736,
934
+ "epoch": 0.7097109226241994,
935
+ "grad_norm": 2.0,
936
+ "learning_rate": 1.4965977125684018e-05,
937
+ "loss": 0.814455337524414,
938
+ "mean_token_accuracy": 0.7903144693374634,
939
+ "num_tokens": 12703495.0,
940
+ "step": 2050
941
+ },
942
+ {
943
+ "entropy": 0.8286141097545624,
944
+ "epoch": 0.718365933875714,
945
+ "grad_norm": 2.296875,
946
+ "learning_rate": 1.484384627461115e-05,
947
+ "loss": 0.8082135009765625,
948
+ "mean_token_accuracy": 0.7908019495010375,
949
+ "num_tokens": 12856585.0,
950
+ "step": 2075
951
+ },
952
+ {
953
+ "entropy": 0.8540136122703552,
954
+ "epoch": 0.7270209451272287,
955
+ "grad_norm": 2.09375,
956
+ "learning_rate": 1.4720764015639555e-05,
957
+ "loss": 0.8297102355957031,
958
+ "mean_token_accuracy": 0.7875827658176422,
959
+ "num_tokens": 13011641.0,
960
+ "step": 2100
961
+ },
962
+ {
963
+ "entropy": 0.8180252170562744,
964
+ "epoch": 0.7356759563787433,
965
+ "grad_norm": 2.171875,
966
+ "learning_rate": 1.4596754524068515e-05,
967
+ "loss": 0.791978988647461,
968
+ "mean_token_accuracy": 0.7945426023006439,
969
+ "num_tokens": 13167258.0,
970
+ "step": 2125
971
+ },
972
+ {
973
+ "entropy": 0.8130170261859894,
974
+ "epoch": 0.744330967630258,
975
+ "grad_norm": 2.1875,
976
+ "learning_rate": 1.4471842157320443e-05,
977
+ "loss": 0.7898551940917968,
978
+ "mean_token_accuracy": 0.7971277523040772,
979
+ "num_tokens": 13315266.0,
980
+ "step": 2150
981
+ },
982
+ {
983
+ "entropy": 0.8508910453319549,
984
+ "epoch": 0.7529859788817725,
985
+ "grad_norm": 2.140625,
986
+ "learning_rate": 1.4346051450156689e-05,
987
+ "loss": 0.8204324340820313,
988
+ "mean_token_accuracy": 0.7856499540805817,
989
+ "num_tokens": 13473686.0,
990
+ "step": 2175
991
+ },
992
+ {
993
+ "entropy": 0.8430261933803558,
994
+ "epoch": 0.7616409901332871,
995
+ "grad_norm": 2.125,
996
+ "learning_rate": 1.4219407109858532e-05,
997
+ "loss": 0.8170342254638672,
998
+ "mean_token_accuracy": 0.7883759355545044,
999
+ "num_tokens": 13627766.0,
1000
+ "step": 2200
1001
+ },
1002
+ {
1003
+ "epoch": 0.7616409901332871,
1004
+ "eval_entropy": 0.8279429495334625,
1005
+ "eval_loss": 0.8048078417778015,
1006
+ "eval_mean_token_accuracy": 0.7913855454960808,
1007
+ "eval_num_tokens": 13627766.0,
1008
+ "eval_runtime": 16.2774,
1009
+ "eval_samples_per_second": 298.88,
1010
+ "eval_steps_per_second": 18.738,
1011
+ "step": 2200
1012
+ },
1013
+ {
1014
+ "entropy": 0.8365470349788666,
1015
+ "epoch": 0.7702960013848018,
1016
+ "grad_norm": 2.3125,
1017
+ "learning_rate": 1.4091934011374281e-05,
1018
+ "loss": 0.8083656311035157,
1019
+ "mean_token_accuracy": 0.7891746735572815,
1020
+ "num_tokens": 13781809.0,
1021
+ "step": 2225
1022
+ },
1023
+ {
1024
+ "entropy": 0.8262781822681426,
1025
+ "epoch": 0.7789510126363164,
1026
+ "grad_norm": 2.265625,
1027
+ "learning_rate": 1.396365719243344e-05,
1028
+ "loss": 0.8065071868896484,
1029
+ "mean_token_accuracy": 0.7918438529968261,
1030
+ "num_tokens": 13937124.0,
1031
+ "step": 2250
1032
+ },
1033
+ {
1034
+ "entropy": 0.8354354929924012,
1035
+ "epoch": 0.7876060238878311,
1036
+ "grad_norm": 2.03125,
1037
+ "learning_rate": 1.3834601848628897e-05,
1038
+ "loss": 0.8083207702636719,
1039
+ "mean_token_accuracy": 0.7916932463645935,
1040
+ "num_tokens": 14088345.0,
1041
+ "step": 2275
1042
+ },
1043
+ {
1044
+ "entropy": 0.8217170929908753,
1045
+ "epoch": 0.7962610351393457,
1046
+ "grad_norm": 2.203125,
1047
+ "learning_rate": 1.3704793328468122e-05,
1048
+ "loss": 0.7991712951660156,
1049
+ "mean_token_accuracy": 0.7924394917488098,
1050
+ "num_tokens": 14241089.0,
1051
+ "step": 2300
1052
+ },
1053
+ {
1054
+ "entropy": 0.8267798566818237,
1055
+ "epoch": 0.8049160463908603,
1056
+ "grad_norm": 2.09375,
1057
+ "learning_rate": 1.3574257128394315e-05,
1058
+ "loss": 0.8018778228759765,
1059
+ "mean_token_accuracy": 0.7922985780239106,
1060
+ "num_tokens": 14392337.0,
1061
+ "step": 2325
1062
+ },
1063
+ {
1064
+ "entropy": 0.8261617541313171,
1065
+ "epoch": 0.8135710576423749,
1066
+ "grad_norm": 2.171875,
1067
+ "learning_rate": 1.3443018887778494e-05,
1068
+ "loss": 0.8028769683837891,
1069
+ "mean_token_accuracy": 0.7926994669437408,
1070
+ "num_tokens": 14544382.0,
1071
+ "step": 2350
1072
+ },
1073
+ {
1074
+ "entropy": 0.8380094921588898,
1075
+ "epoch": 0.8222260688938896,
1076
+ "grad_norm": 2.3125,
1077
+ "learning_rate": 1.3311104383883537e-05,
1078
+ "loss": 0.8226547241210938,
1079
+ "mean_token_accuracy": 0.7886342179775238,
1080
+ "num_tokens": 14703460.0,
1081
+ "step": 2375
1082
+ },
1083
+ {
1084
+ "entropy": 0.8281042277812958,
1085
+ "epoch": 0.8308810801454042,
1086
+ "grad_norm": 2.296875,
1087
+ "learning_rate": 1.3178539526801106e-05,
1088
+ "loss": 0.8025864410400391,
1089
+ "mean_token_accuracy": 0.7910430765151978,
1090
+ "num_tokens": 14860294.0,
1091
+ "step": 2400
1092
+ },
1093
+ {
1094
+ "epoch": 0.8308810801454042,
1095
+ "eval_entropy": 0.8327872855741469,
1096
+ "eval_loss": 0.8026647567749023,
1097
+ "eval_mean_token_accuracy": 0.7919463300314106,
1098
+ "eval_num_tokens": 14860294.0,
1099
+ "eval_runtime": 16.2486,
1100
+ "eval_samples_per_second": 299.41,
1101
+ "eval_steps_per_second": 18.771,
1102
+ "step": 2400
1103
+ },
1104
+ {
1105
+ "entropy": 0.811096111536026,
1106
+ "epoch": 0.8395360913969189,
1107
+ "grad_norm": 2.4375,
1108
+ "learning_rate": 1.3045350354362495e-05,
1109
+ "loss": 0.7822922515869141,
1110
+ "mean_token_accuracy": 0.7949872577190399,
1111
+ "num_tokens": 15015328.0,
1112
+ "step": 2425
1113
+ },
1114
+ {
1115
+ "entropy": 0.8189793074131012,
1116
+ "epoch": 0.8481911026484334,
1117
+ "grad_norm": 2.3125,
1118
+ "learning_rate": 1.2911563027024392e-05,
1119
+ "loss": 0.8003229522705078,
1120
+ "mean_token_accuracy": 0.7943521749973297,
1121
+ "num_tokens": 15170350.0,
1122
+ "step": 2450
1123
+ },
1124
+ {
1125
+ "entropy": 0.8213041162490845,
1126
+ "epoch": 0.856846113899948,
1127
+ "grad_norm": 2.21875,
1128
+ "learning_rate": 1.2777203822730557e-05,
1129
+ "loss": 0.7970082092285157,
1130
+ "mean_token_accuracy": 0.7927951776981353,
1131
+ "num_tokens": 15324153.0,
1132
+ "step": 2475
1133
+ },
1134
+ {
1135
+ "entropy": 0.824720960855484,
1136
+ "epoch": 0.8655011251514627,
1137
+ "grad_norm": 2.125,
1138
+ "learning_rate": 1.2642299131750417e-05,
1139
+ "loss": 0.7908332824707032,
1140
+ "mean_token_accuracy": 0.7957222282886505,
1141
+ "num_tokens": 15473303.0,
1142
+ "step": 2500
1143
+ },
1144
+ {
1145
+ "entropy": 0.838714919090271,
1146
+ "epoch": 0.8741561364029773,
1147
+ "grad_norm": 2.09375,
1148
+ "learning_rate": 1.250687545149559e-05,
1149
+ "loss": 0.8137520599365234,
1150
+ "mean_token_accuracy": 0.7890112113952636,
1151
+ "num_tokens": 15629017.0,
1152
+ "step": 2525
1153
+ },
1154
+ {
1155
+ "entropy": 0.8311066961288452,
1156
+ "epoch": 0.882811147654492,
1157
+ "grad_norm": 2.1875,
1158
+ "learning_rate": 1.2370959381315393e-05,
1159
+ "loss": 0.8034115600585937,
1160
+ "mean_token_accuracy": 0.7926496362686157,
1161
+ "num_tokens": 15777876.0,
1162
+ "step": 2550
1163
+ },
1164
+ {
1165
+ "entropy": 0.8097344720363617,
1166
+ "epoch": 0.8914661589060066,
1167
+ "grad_norm": 2.203125,
1168
+ "learning_rate": 1.2234577617272282e-05,
1169
+ "loss": 0.786445083618164,
1170
+ "mean_token_accuracy": 0.7966820681095124,
1171
+ "num_tokens": 15929184.0,
1172
+ "step": 2575
1173
+ },
1174
+ {
1175
+ "entropy": 0.8254410099983215,
1176
+ "epoch": 0.9001211701575212,
1177
+ "grad_norm": 2.296875,
1178
+ "learning_rate": 1.209775694689837e-05,
1179
+ "loss": 0.8023792266845703,
1180
+ "mean_token_accuracy": 0.7923547208309174,
1181
+ "num_tokens": 16087411.0,
1182
+ "step": 2600
1183
+ },
1184
+ {
1185
+ "epoch": 0.9001211701575212,
1186
+ "eval_entropy": 0.8213896471945966,
1187
+ "eval_loss": 0.8008216619491577,
1188
+ "eval_mean_token_accuracy": 0.7924871415388389,
1189
+ "eval_num_tokens": 16087411.0,
1190
+ "eval_runtime": 16.2722,
1191
+ "eval_samples_per_second": 298.976,
1192
+ "eval_steps_per_second": 18.744,
1193
+ "step": 2600
1194
+ },
1195
+ {
1196
+ "entropy": 0.8255444514751434,
1197
+ "epoch": 0.9087761814090358,
1198
+ "grad_norm": 1.9140625,
1199
+ "learning_rate": 1.1960524243933875e-05,
1200
+ "loss": 0.8017765808105469,
1201
+ "mean_token_accuracy": 0.7921687912940979,
1202
+ "num_tokens": 16244042.0,
1203
+ "step": 2625
1204
+ },
1205
+ {
1206
+ "entropy": 0.8252915942668915,
1207
+ "epoch": 0.9174311926605505,
1208
+ "grad_norm": 2.21875,
1209
+ "learning_rate": 1.182290646304874e-05,
1210
+ "loss": 0.8029595947265625,
1211
+ "mean_token_accuracy": 0.7924495422840119,
1212
+ "num_tokens": 16399768.0,
1213
+ "step": 2650
1214
+ },
1215
+ {
1216
+ "entropy": 0.8625216126441956,
1217
+ "epoch": 0.9260862039120651,
1218
+ "grad_norm": 2.171875,
1219
+ "learning_rate": 1.1684930634548287e-05,
1220
+ "loss": 0.838686752319336,
1221
+ "mean_token_accuracy": 0.7851159811019898,
1222
+ "num_tokens": 16557371.0,
1223
+ "step": 2675
1224
+ },
1225
+ {
1226
+ "entropy": 0.8215836095809936,
1227
+ "epoch": 0.9347412151635797,
1228
+ "grad_norm": 2.265625,
1229
+ "learning_rate": 1.1546623859064053e-05,
1230
+ "loss": 0.7977464294433594,
1231
+ "mean_token_accuracy": 0.7929444241523743,
1232
+ "num_tokens": 16713045.0,
1233
+ "step": 2700
1234
+ },
1235
+ {
1236
+ "entropy": 0.8286124646663666,
1237
+ "epoch": 0.9433962264150944,
1238
+ "grad_norm": 2.078125,
1239
+ "learning_rate": 1.1408013302230765e-05,
1240
+ "loss": 0.802153091430664,
1241
+ "mean_token_accuracy": 0.7924544358253479,
1242
+ "num_tokens": 16867145.0,
1243
+ "step": 2725
1244
+ },
1245
+ {
1246
+ "entropy": 0.8237406086921691,
1247
+ "epoch": 0.9520512376666089,
1248
+ "grad_norm": 2.15625,
1249
+ "learning_rate": 1.126912618935063e-05,
1250
+ "loss": 0.8021310424804687,
1251
+ "mean_token_accuracy": 0.7944400310516357,
1252
+ "num_tokens": 17014455.0,
1253
+ "step": 2750
1254
+ },
1255
+ {
1256
+ "entropy": 0.826482676267624,
1257
+ "epoch": 0.9607062489181236,
1258
+ "grad_norm": 2.140625,
1259
+ "learning_rate": 1.1129989800045817e-05,
1260
+ "loss": 0.8052169036865234,
1261
+ "mean_token_accuracy": 0.7911915636062622,
1262
+ "num_tokens": 17175484.0,
1263
+ "step": 2775
1264
+ },
1265
+ {
1266
+ "entropy": 0.8201354122161866,
1267
+ "epoch": 0.9693612601696382,
1268
+ "grad_norm": 2.421875,
1269
+ "learning_rate": 1.099063146290032e-05,
1270
+ "loss": 0.7985061645507813,
1271
+ "mean_token_accuracy": 0.7938509380817413,
1272
+ "num_tokens": 17334068.0,
1273
+ "step": 2800
1274
+ },
1275
+ {
1276
+ "epoch": 0.9693612601696382,
1277
+ "eval_entropy": 0.8281321289109402,
1278
+ "eval_loss": 0.7994962334632874,
1279
+ "eval_mean_token_accuracy": 0.792740264681519,
1280
+ "eval_num_tokens": 17334068.0,
1281
+ "eval_runtime": 16.2806,
1282
+ "eval_samples_per_second": 298.822,
1283
+ "eval_steps_per_second": 18.734,
1284
+ "step": 2800
1285
+ },
1286
+ {
1287
+ "entropy": 0.8046394038200378,
1288
+ "epoch": 0.9780162714211529,
1289
+ "grad_norm": 2.46875,
1290
+ "learning_rate": 1.08510785500922e-05,
1291
+ "loss": 0.7775811004638672,
1292
+ "mean_token_accuracy": 0.7966744971275329,
1293
+ "num_tokens": 17483883.0,
1294
+ "step": 2825
1295
+ },
1296
+ {
1297
+ "entropy": 0.8306651365756988,
1298
+ "epoch": 0.9866712826726675,
1299
+ "grad_norm": 2.359375,
1300
+ "learning_rate": 1.0711358472017244e-05,
1301
+ "loss": 0.7954608154296875,
1302
+ "mean_token_accuracy": 0.7919278919696808,
1303
+ "num_tokens": 17639972.0,
1304
+ "step": 2850
1305
+ },
1306
+ {
1307
+ "entropy": 0.8295222818851471,
1308
+ "epoch": 0.9953262939241821,
1309
+ "grad_norm": 2.125,
1310
+ "learning_rate": 1.0571498671905146e-05,
1311
+ "loss": 0.8072916412353516,
1312
+ "mean_token_accuracy": 0.7907999694347382,
1313
+ "num_tokens": 17794430.0,
1314
+ "step": 2875
1315
+ },
1316
+ {
1317
+ "entropy": 0.8070371369926297,
1318
+ "epoch": 1.0038082049506665,
1319
+ "grad_norm": 2.09375,
1320
+ "learning_rate": 1.0431526620429207e-05,
1321
+ "loss": 0.760282211303711,
1322
+ "mean_token_accuracy": 0.7995899867038337,
1323
+ "num_tokens": 17946140.0,
1324
+ "step": 2900
1325
+ },
1326
+ {
1327
+ "entropy": 0.7756562912464142,
1328
+ "epoch": 1.0124632162021812,
1329
+ "grad_norm": 2.140625,
1330
+ "learning_rate": 1.0291469810310669e-05,
1331
+ "loss": 0.741202621459961,
1332
+ "mean_token_accuracy": 0.803013744354248,
1333
+ "num_tokens": 18100839.0,
1334
+ "step": 2925
1335
+ },
1336
+ {
1337
+ "entropy": 0.7452013885974884,
1338
+ "epoch": 1.0211182274536956,
1339
+ "grad_norm": 2.15625,
1340
+ "learning_rate": 1.0151355750918712e-05,
1341
+ "loss": 0.7064004516601563,
1342
+ "mean_token_accuracy": 0.8126378917694091,
1343
+ "num_tokens": 18256691.0,
1344
+ "step": 2950
1345
+ },
1346
+ {
1347
+ "entropy": 0.7718643927574158,
1348
+ "epoch": 1.0297732387052103,
1349
+ "grad_norm": 1.8671875,
1350
+ "learning_rate": 1.001121196286718e-05,
1351
+ "loss": 0.7311404418945312,
1352
+ "mean_token_accuracy": 0.8063716220855713,
1353
+ "num_tokens": 18418483.0,
1354
+ "step": 2975
1355
+ },
1356
+ {
1357
+ "entropy": 0.765803085565567,
1358
+ "epoch": 1.038428249956725,
1359
+ "grad_norm": 2.0,
1360
+ "learning_rate": 9.871065972609089e-06,
1361
+ "loss": 0.7332243347167968,
1362
+ "mean_token_accuracy": 0.8049838674068451,
1363
+ "num_tokens": 18576253.0,
1364
+ "step": 3000
1365
+ },
1366
+ {
1367
+ "epoch": 1.038428249956725,
1368
+ "eval_entropy": 0.7786700289757525,
1369
+ "eval_loss": 0.8012996912002563,
1370
+ "eval_mean_token_accuracy": 0.792937566022404,
1371
+ "eval_num_tokens": 18576253.0,
1372
+ "eval_runtime": 16.277,
1373
+ "eval_samples_per_second": 298.888,
1374
+ "eval_steps_per_second": 18.738,
1375
+ "step": 3000
1376
+ },
1377
+ {
1378
+ "entropy": 0.7836507785320282,
1379
+ "epoch": 1.0470832612082395,
1380
+ "grad_norm": 2.0625,
1381
+ "learning_rate": 9.730945307030009e-06,
1382
+ "loss": 0.7418560028076172,
1383
+ "mean_token_accuracy": 0.8028053915500641,
1384
+ "num_tokens": 18729564.0,
1385
+ "step": 3025
1386
+ },
1387
+ {
1388
+ "entropy": 0.7653721117973328,
1389
+ "epoch": 1.0557382724597542,
1390
+ "grad_norm": 2.03125,
1391
+ "learning_rate": 9.590877488041336e-06,
1392
+ "loss": 0.7263675689697265,
1393
+ "mean_token_accuracy": 0.807190809249878,
1394
+ "num_tokens": 18891538.0,
1395
+ "step": 3050
1396
+ },
1397
+ {
1398
+ "entropy": 0.7604654181003571,
1399
+ "epoch": 1.0643932837112688,
1400
+ "grad_norm": 2.296875,
1401
+ "learning_rate": 9.45089002717457e-06,
1402
+ "loss": 0.72638916015625,
1403
+ "mean_token_accuracy": 0.8084564554691315,
1404
+ "num_tokens": 19050606.0,
1405
+ "step": 3075
1406
+ },
1407
+ {
1408
+ "entropy": 0.7386153995990753,
1409
+ "epoch": 1.0730482949627835,
1410
+ "grad_norm": 1.9765625,
1411
+ "learning_rate": 9.311010420177597e-06,
1412
+ "loss": 0.706187973022461,
1413
+ "mean_token_accuracy": 0.8120154666900635,
1414
+ "num_tokens": 19208707.0,
1415
+ "step": 3100
1416
+ },
1417
+ {
1418
+ "entropy": 0.7525534296035766,
1419
+ "epoch": 1.081703306214298,
1420
+ "grad_norm": 2.171875,
1421
+ "learning_rate": 9.171266141614112e-06,
1422
+ "loss": 0.7135391235351562,
1423
+ "mean_token_accuracy": 0.8123032712936401,
1424
+ "num_tokens": 19362737.0,
1425
+ "step": 3125
1426
+ },
1427
+ {
1428
+ "entropy": 0.7687537455558777,
1429
+ "epoch": 1.0903583174658127,
1430
+ "grad_norm": 2.109375,
1431
+ "learning_rate": 9.031684639467159e-06,
1432
+ "loss": 0.734495849609375,
1433
+ "mean_token_accuracy": 0.8068291318416595,
1434
+ "num_tokens": 19521058.0,
1435
+ "step": 3150
1436
+ },
1437
+ {
1438
+ "entropy": 0.7806685280799865,
1439
+ "epoch": 1.0990133287173274,
1440
+ "grad_norm": 2.046875,
1441
+ "learning_rate": 8.892293329747923e-06,
1442
+ "loss": 0.7469673156738281,
1443
+ "mean_token_accuracy": 0.8041970551013946,
1444
+ "num_tokens": 19675603.0,
1445
+ "step": 3175
1446
+ },
1447
+ {
1448
+ "entropy": 0.7709089267253876,
1449
+ "epoch": 1.1076683399688418,
1450
+ "grad_norm": 2.15625,
1451
+ "learning_rate": 8.753119591110802e-06,
1452
+ "loss": 0.7363986206054688,
1453
+ "mean_token_accuracy": 0.8061197555065155,
1454
+ "num_tokens": 19828898.0,
1455
+ "step": 3200
1456
+ },
1457
+ {
1458
+ "epoch": 1.1076683399688418,
1459
+ "eval_entropy": 0.7721671974072691,
1460
+ "eval_loss": 0.8010231852531433,
1461
+ "eval_mean_token_accuracy": 0.7929807965872717,
1462
+ "eval_num_tokens": 19828898.0,
1463
+ "eval_runtime": 16.2529,
1464
+ "eval_samples_per_second": 299.331,
1465
+ "eval_steps_per_second": 18.766,
1466
+ "step": 3200
1467
+ },
1468
+ {
1469
+ "entropy": 0.7211595928668976,
1470
+ "epoch": 1.1163233512203565,
1471
+ "grad_norm": 2.046875,
1472
+ "learning_rate": 8.614190759475789e-06,
1473
+ "loss": 0.6826175689697266,
1474
+ "mean_token_accuracy": 0.8186909162998199,
1475
+ "num_tokens": 19982072.0,
1476
+ "step": 3225
1477
+ },
1478
+ {
1479
+ "entropy": 0.7708668720722198,
1480
+ "epoch": 1.1249783624718712,
1481
+ "grad_norm": 2.15625,
1482
+ "learning_rate": 8.475534122659275e-06,
1483
+ "loss": 0.7388634490966797,
1484
+ "mean_token_accuracy": 0.8042692136764527,
1485
+ "num_tokens": 20137744.0,
1486
+ "step": 3250
1487
+ },
1488
+ {
1489
+ "entropy": 0.7538685095310211,
1490
+ "epoch": 1.133633373723386,
1491
+ "grad_norm": 2.171875,
1492
+ "learning_rate": 8.337176915014293e-06,
1493
+ "loss": 0.7132722473144532,
1494
+ "mean_token_accuracy": 0.8105318355560303,
1495
+ "num_tokens": 20290524.0,
1496
+ "step": 3275
1497
+ },
1498
+ {
1499
+ "entropy": 0.7487189078330994,
1500
+ "epoch": 1.1422883849749004,
1501
+ "grad_norm": 2.046875,
1502
+ "learning_rate": 8.199146312081265e-06,
1503
+ "loss": 0.7089384460449218,
1504
+ "mean_token_accuracy": 0.8108018398284912,
1505
+ "num_tokens": 20443485.0,
1506
+ "step": 3300
1507
+ },
1508
+ {
1509
+ "entropy": 0.7543841946125031,
1510
+ "epoch": 1.150943396226415,
1511
+ "grad_norm": 2.0625,
1512
+ "learning_rate": 8.061469425250273e-06,
1513
+ "loss": 0.7227223205566407,
1514
+ "mean_token_accuracy": 0.8090525925159454,
1515
+ "num_tokens": 20595797.0,
1516
+ "step": 3325
1517
+ },
1518
+ {
1519
+ "entropy": 0.7593919610977173,
1520
+ "epoch": 1.1595984074779297,
1521
+ "grad_norm": 2.015625,
1522
+ "learning_rate": 7.924173296435987e-06,
1523
+ "loss": 0.7215187072753906,
1524
+ "mean_token_accuracy": 0.8093890881538391,
1525
+ "num_tokens": 20749554.0,
1526
+ "step": 3350
1527
+ },
1528
+ {
1529
+ "entropy": 0.7829481947422028,
1530
+ "epoch": 1.1682534187294444,
1531
+ "grad_norm": 2.171875,
1532
+ "learning_rate": 7.78728489276618e-06,
1533
+ "loss": 0.7491289520263672,
1534
+ "mean_token_accuracy": 0.8015105819702149,
1535
+ "num_tokens": 20909534.0,
1536
+ "step": 3375
1537
+ },
1538
+ {
1539
+ "entropy": 0.7392001152038574,
1540
+ "epoch": 1.176908429980959,
1541
+ "grad_norm": 2.1875,
1542
+ "learning_rate": 7.650831101284977e-06,
1543
+ "loss": 0.7020195007324219,
1544
+ "mean_token_accuracy": 0.8150424540042878,
1545
+ "num_tokens": 21059158.0,
1546
+ "step": 3400
1547
+ },
1548
+ {
1549
+ "epoch": 1.176908429980959,
1550
+ "eval_entropy": 0.7698653085309951,
1551
+ "eval_loss": 0.8008679747581482,
1552
+ "eval_mean_token_accuracy": 0.793300522155449,
1553
+ "eval_num_tokens": 21059158.0,
1554
+ "eval_runtime": 16.2799,
1555
+ "eval_samples_per_second": 298.835,
1556
+ "eval_steps_per_second": 18.735,
1557
+ "step": 3400
1558
+ },
1559
+ {
1560
+ "entropy": 0.768788868188858,
1561
+ "epoch": 1.1855634412324736,
1562
+ "grad_norm": 2.046875,
1563
+ "learning_rate": 7.514838723671799e-06,
1564
+ "loss": 0.7372692108154297,
1565
+ "mean_token_accuracy": 0.8063059449195862,
1566
+ "num_tokens": 21208688.0,
1567
+ "step": 3425
1568
+ },
1569
+ {
1570
+ "entropy": 0.7443365502357483,
1571
+ "epoch": 1.1942184524839883,
1572
+ "grad_norm": 1.875,
1573
+ "learning_rate": 7.379334470977093e-06,
1574
+ "loss": 0.710152587890625,
1575
+ "mean_token_accuracy": 0.811426283121109,
1576
+ "num_tokens": 21365731.0,
1577
+ "step": 3450
1578
+ },
1579
+ {
1580
+ "entropy": 0.7438737845420837,
1581
+ "epoch": 1.2028734637355027,
1582
+ "grad_norm": 1.96875,
1583
+ "learning_rate": 7.244344958375881e-06,
1584
+ "loss": 0.710167465209961,
1585
+ "mean_token_accuracy": 0.8120112872123718,
1586
+ "num_tokens": 21519688.0,
1587
+ "step": 3475
1588
+ },
1589
+ {
1590
+ "entropy": 0.7728407967090607,
1591
+ "epoch": 1.2115284749870174,
1592
+ "grad_norm": 2.046875,
1593
+ "learning_rate": 7.109896699940084e-06,
1594
+ "loss": 0.7461888885498047,
1595
+ "mean_token_accuracy": 0.8037091171741486,
1596
+ "num_tokens": 21676493.0,
1597
+ "step": 3500
1598
+ },
1599
+ {
1600
+ "entropy": 0.7640272963047028,
1601
+ "epoch": 1.2201834862385321,
1602
+ "grad_norm": 2.203125,
1603
+ "learning_rate": 6.976016103430765e-06,
1604
+ "loss": 0.7235647583007813,
1605
+ "mean_token_accuracy": 0.8083086097240448,
1606
+ "num_tokens": 21825664.0,
1607
+ "step": 3525
1608
+ },
1609
+ {
1610
+ "entropy": 0.7460694527626037,
1611
+ "epoch": 1.2288384974900468,
1612
+ "grad_norm": 2.03125,
1613
+ "learning_rate": 6.842729465111224e-06,
1614
+ "loss": 0.7100708770751953,
1615
+ "mean_token_accuracy": 0.8116568970680237,
1616
+ "num_tokens": 21976504.0,
1617
+ "step": 3550
1618
+ },
1619
+ {
1620
+ "entropy": 0.759437519311905,
1621
+ "epoch": 1.2374935087415613,
1622
+ "grad_norm": 2.609375,
1623
+ "learning_rate": 6.710062964581988e-06,
1624
+ "loss": 0.722847900390625,
1625
+ "mean_token_accuracy": 0.807286411523819,
1626
+ "num_tokens": 22133165.0,
1627
+ "step": 3575
1628
+ },
1629
+ {
1630
+ "entropy": 0.7708945345878601,
1631
+ "epoch": 1.246148519993076,
1632
+ "grad_norm": 2.015625,
1633
+ "learning_rate": 6.578042659638726e-06,
1634
+ "loss": 0.7426469421386719,
1635
+ "mean_token_accuracy": 0.8035933816432953,
1636
+ "num_tokens": 22290456.0,
1637
+ "step": 3600
1638
+ },
1639
+ {
1640
+ "epoch": 1.246148519993076,
1641
+ "eval_entropy": 0.7726592179204597,
1642
+ "eval_loss": 0.8001708984375,
1643
+ "eval_mean_token_accuracy": 0.7932597899046101,
1644
+ "eval_num_tokens": 22290456.0,
1645
+ "eval_runtime": 16.2656,
1646
+ "eval_samples_per_second": 299.097,
1647
+ "eval_steps_per_second": 18.751,
1648
+ "step": 3600
1649
+ },
1650
+ {
1651
+ "entropy": 0.7578576147556305,
1652
+ "epoch": 1.2548035312445907,
1653
+ "grad_norm": 2.078125,
1654
+ "learning_rate": 6.446694481154104e-06,
1655
+ "loss": 0.7225240325927734,
1656
+ "mean_token_accuracy": 0.8097064507007599,
1657
+ "num_tokens": 22446824.0,
1658
+ "step": 3625
1659
+ },
1660
+ {
1661
+ "entropy": 0.7762425947189331,
1662
+ "epoch": 1.2634585424961053,
1663
+ "grad_norm": 2.265625,
1664
+ "learning_rate": 6.3160442279845206e-06,
1665
+ "loss": 0.7412718200683593,
1666
+ "mean_token_accuracy": 0.805392199754715,
1667
+ "num_tokens": 22602588.0,
1668
+ "step": 3650
1669
+ },
1670
+ {
1671
+ "entropy": 0.7637201595306397,
1672
+ "epoch": 1.2721135537476198,
1673
+ "grad_norm": 2.25,
1674
+ "learning_rate": 6.186117561902836e-06,
1675
+ "loss": 0.7297930908203125,
1676
+ "mean_token_accuracy": 0.8078333747386932,
1677
+ "num_tokens": 22752707.0,
1678
+ "step": 3675
1679
+ },
1680
+ {
1681
+ "entropy": 0.7704725599288941,
1682
+ "epoch": 1.2807685649991345,
1683
+ "grad_norm": 2.125,
1684
+ "learning_rate": 6.056940002557969e-06,
1685
+ "loss": 0.7372798919677734,
1686
+ "mean_token_accuracy": 0.8056825184822083,
1687
+ "num_tokens": 22907803.0,
1688
+ "step": 3700
1689
+ },
1690
+ {
1691
+ "entropy": 0.7552802407741547,
1692
+ "epoch": 1.2894235762506492,
1693
+ "grad_norm": 2.15625,
1694
+ "learning_rate": 5.928536922462449e-06,
1695
+ "loss": 0.7166344451904297,
1696
+ "mean_token_accuracy": 0.8109099209308624,
1697
+ "num_tokens": 23059819.0,
1698
+ "step": 3725
1699
+ },
1700
+ {
1701
+ "entropy": 0.7541340529918671,
1702
+ "epoch": 1.2980785875021636,
1703
+ "grad_norm": 2.3125,
1704
+ "learning_rate": 5.80093354200886e-06,
1705
+ "loss": 0.7238423919677734,
1706
+ "mean_token_accuracy": 0.809191288948059,
1707
+ "num_tokens": 23217392.0,
1708
+ "step": 3750
1709
+ },
1710
+ {
1711
+ "entropy": 0.7676697385311126,
1712
+ "epoch": 1.3067335987536783,
1713
+ "grad_norm": 2.21875,
1714
+ "learning_rate": 5.674154924516129e-06,
1715
+ "loss": 0.7366065979003906,
1716
+ "mean_token_accuracy": 0.8054041469097137,
1717
+ "num_tokens": 23371073.0,
1718
+ "step": 3775
1719
+ },
1720
+ {
1721
+ "entropy": 0.7785192513465882,
1722
+ "epoch": 1.315388610005193,
1723
+ "grad_norm": 1.9140625,
1724
+ "learning_rate": 5.548225971306731e-06,
1725
+ "loss": 0.7439189147949219,
1726
+ "mean_token_accuracy": 0.8034746491909027,
1727
+ "num_tokens": 23524592.0,
1728
+ "step": 3800
1729
+ },
1730
+ {
1731
+ "epoch": 1.315388610005193,
1732
+ "eval_entropy": 0.7713183484116538,
1733
+ "eval_loss": 0.7998276352882385,
1734
+ "eval_mean_token_accuracy": 0.7933589818047695,
1735
+ "eval_num_tokens": 23524592.0,
1736
+ "eval_runtime": 16.2844,
1737
+ "eval_samples_per_second": 298.753,
1738
+ "eval_steps_per_second": 18.73,
1739
+ "step": 3800
1740
+ },
1741
+ {
1742
+ "entropy": 0.7570435631275177,
1743
+ "epoch": 1.3240436212567077,
1744
+ "grad_norm": 2.078125,
1745
+ "learning_rate": 5.423171416815651e-06,
1746
+ "loss": 0.7266172790527343,
1747
+ "mean_token_accuracy": 0.8094574224948883,
1748
+ "num_tokens": 23682141.0,
1749
+ "step": 3825
1750
+ },
1751
+ {
1752
+ "entropy": 0.76507066488266,
1753
+ "epoch": 1.3326986325082222,
1754
+ "grad_norm": 2.15625,
1755
+ "learning_rate": 5.299015823732163e-06,
1756
+ "loss": 0.7294864654541016,
1757
+ "mean_token_accuracy": 0.8064786922931672,
1758
+ "num_tokens": 23839050.0,
1759
+ "step": 3850
1760
+ },
1761
+ {
1762
+ "entropy": 0.7562107217311859,
1763
+ "epoch": 1.3413536437597369,
1764
+ "grad_norm": 2.0625,
1765
+ "learning_rate": 5.175783578175309e-06,
1766
+ "loss": 0.719031982421875,
1767
+ "mean_token_accuracy": 0.8091187167167664,
1768
+ "num_tokens": 23992205.0,
1769
+ "step": 3875
1770
+ },
1771
+ {
1772
+ "entropy": 0.7576862835884094,
1773
+ "epoch": 1.3500086550112516,
1774
+ "grad_norm": 2.109375,
1775
+ "learning_rate": 5.053498884904122e-06,
1776
+ "loss": 0.7270169830322266,
1777
+ "mean_token_accuracy": 0.8094841420650483,
1778
+ "num_tokens": 24139859.0,
1779
+ "step": 3900
1780
+ },
1781
+ {
1782
+ "entropy": 0.7541490840911865,
1783
+ "epoch": 1.3586636662627662,
1784
+ "grad_norm": 2.109375,
1785
+ "learning_rate": 4.932185762563381e-06,
1786
+ "loss": 0.7113311004638672,
1787
+ "mean_token_accuracy": 0.8112999868392944,
1788
+ "num_tokens": 24289196.0,
1789
+ "step": 3925
1790
+ },
1791
+ {
1792
+ "entropy": 0.7688007843494415,
1793
+ "epoch": 1.3673186775142807,
1794
+ "grad_norm": 2.1875,
1795
+ "learning_rate": 4.811868038965999e-06,
1796
+ "loss": 0.745551986694336,
1797
+ "mean_token_accuracy": 0.8050249302387238,
1798
+ "num_tokens": 24447938.0,
1799
+ "step": 3950
1800
+ },
1801
+ {
1802
+ "entropy": 0.7551292729377747,
1803
+ "epoch": 1.3759736887657954,
1804
+ "grad_norm": 2.1875,
1805
+ "learning_rate": 4.692569346412864e-06,
1806
+ "loss": 0.7206728363037109,
1807
+ "mean_token_accuracy": 0.8098892831802368,
1808
+ "num_tokens": 24598742.0,
1809
+ "step": 3975
1810
+ },
1811
+ {
1812
+ "entropy": 0.7587357807159424,
1813
+ "epoch": 1.38462870001731,
1814
+ "grad_norm": 2.125,
1815
+ "learning_rate": 4.574313117051059e-06,
1816
+ "loss": 0.7276847076416015,
1817
+ "mean_token_accuracy": 0.8077455127239227,
1818
+ "num_tokens": 24750406.0,
1819
+ "step": 4000
1820
+ },
1821
+ {
1822
+ "epoch": 1.38462870001731,
1823
+ "eval_entropy": 0.7717335020909544,
1824
+ "eval_loss": 0.7996781468391418,
1825
+ "eval_mean_token_accuracy": 0.7934077425081222,
1826
+ "eval_num_tokens": 24750406.0,
1827
+ "eval_runtime": 16.2676,
1828
+ "eval_samples_per_second": 299.061,
1829
+ "eval_steps_per_second": 18.749,
1830
+ "step": 4000
1831
+ },
1832
+ {
1833
+ "entropy": 0.7805033266544342,
1834
+ "epoch": 1.3932837112688246,
1835
+ "grad_norm": 2.078125,
1836
+ "learning_rate": 4.457122578271457e-06,
1837
+ "loss": 0.7509593200683594,
1838
+ "mean_token_accuracy": 0.8028152561187745,
1839
+ "num_tokens": 24906659.0,
1840
+ "step": 4025
1841
+ },
1842
+ {
1843
+ "entropy": 0.7705829620361329,
1844
+ "epoch": 1.4019387225203392,
1845
+ "grad_norm": 2.125,
1846
+ "learning_rate": 4.341020748146449e-06,
1847
+ "loss": 0.7392497253417969,
1848
+ "mean_token_accuracy": 0.8049503290653228,
1849
+ "num_tokens": 25066654.0,
1850
+ "step": 4050
1851
+ },
1852
+ {
1853
+ "entropy": 0.7581047600507737,
1854
+ "epoch": 1.410593733771854,
1855
+ "grad_norm": 2.0,
1856
+ "learning_rate": 4.226030430908869e-06,
1857
+ "loss": 0.7285484313964844,
1858
+ "mean_token_accuracy": 0.8085175144672394,
1859
+ "num_tokens": 25220415.0,
1860
+ "step": 4075
1861
+ },
1862
+ {
1863
+ "entropy": 0.7656786620616913,
1864
+ "epoch": 1.4192487450233684,
1865
+ "grad_norm": 2.21875,
1866
+ "learning_rate": 4.112174212472864e-06,
1867
+ "loss": 0.7323892211914063,
1868
+ "mean_token_accuracy": 0.8051603281497955,
1869
+ "num_tokens": 25374458.0,
1870
+ "step": 4100
1871
+ },
1872
+ {
1873
+ "entropy": 0.7522096228599549,
1874
+ "epoch": 1.427903756274883,
1875
+ "grad_norm": 2.171875,
1876
+ "learning_rate": 3.99947445599767e-06,
1877
+ "loss": 0.7212752532958985,
1878
+ "mean_token_accuracy": 0.8096666383743286,
1879
+ "num_tokens": 25525969.0,
1880
+ "step": 4125
1881
+ },
1882
+ {
1883
+ "entropy": 0.7608710610866547,
1884
+ "epoch": 1.4365587675263978,
1885
+ "grad_norm": 2.40625,
1886
+ "learning_rate": 3.887953297495121e-06,
1887
+ "loss": 0.7316089630126953,
1888
+ "mean_token_accuracy": 0.8074719154834747,
1889
+ "num_tokens": 25681452.0,
1890
+ "step": 4150
1891
+ },
1892
+ {
1893
+ "entropy": 0.7639850771427155,
1894
+ "epoch": 1.4452137787779125,
1895
+ "grad_norm": 2.140625,
1896
+ "learning_rate": 3.77763264148181e-06,
1897
+ "loss": 0.7320518493652344,
1898
+ "mean_token_accuracy": 0.809201009273529,
1899
+ "num_tokens": 25838398.0,
1900
+ "step": 4175
1901
+ },
1902
+ {
1903
+ "entropy": 0.7513358175754548,
1904
+ "epoch": 1.4538687900294271,
1905
+ "grad_norm": 2.21875,
1906
+ "learning_rate": 3.6685341566766785e-06,
1907
+ "loss": 0.7206896209716797,
1908
+ "mean_token_accuracy": 0.8095373523235321,
1909
+ "num_tokens": 25992640.0,
1910
+ "step": 4200
1911
+ },
1912
+ {
1913
+ "epoch": 1.4538687900294271,
1914
+ "eval_entropy": 0.7707103953986871,
1915
+ "eval_loss": 0.7995045185089111,
1916
+ "eval_mean_token_accuracy": 0.7933821330305005,
1917
+ "eval_num_tokens": 25992640.0,
1918
+ "eval_runtime": 16.2677,
1919
+ "eval_samples_per_second": 299.058,
1920
+ "eval_steps_per_second": 18.749,
1921
+ "step": 4200
1922
+ },
1923
+ {
1924
+ "entropy": 0.7595930647850037,
1925
+ "epoch": 1.4625238012809416,
1926
+ "grad_norm": 2.171875,
1927
+ "learning_rate": 3.5606792717449455e-06,
1928
+ "loss": 0.7264796447753906,
1929
+ "mean_token_accuracy": 0.8094517874717713,
1930
+ "num_tokens": 26141304.0,
1931
+ "step": 4225
1932
+ },
1933
+ {
1934
+ "entropy": 0.7630014491081237,
1935
+ "epoch": 1.4711788125324563,
1936
+ "grad_norm": 2.15625,
1937
+ "learning_rate": 3.4540891710891768e-06,
1938
+ "loss": 0.7287496185302734,
1939
+ "mean_token_accuracy": 0.808629115819931,
1940
+ "num_tokens": 26295315.0,
1941
+ "step": 4250
1942
+ },
1943
+ {
1944
+ "entropy": 0.7566706609725952,
1945
+ "epoch": 1.479833823783971,
1946
+ "grad_norm": 2.34375,
1947
+ "learning_rate": 3.3487847906883232e-06,
1948
+ "loss": 0.7267755126953125,
1949
+ "mean_token_accuracy": 0.8091002118587494,
1950
+ "num_tokens": 26450821.0,
1951
+ "step": 4275
1952
+ },
1953
+ {
1954
+ "entropy": 0.7702415359020233,
1955
+ "epoch": 1.4884888350354855,
1956
+ "grad_norm": 2.09375,
1957
+ "learning_rate": 3.2447868139855954e-06,
1958
+ "loss": 0.74352294921875,
1959
+ "mean_token_accuracy": 0.8064157390594482,
1960
+ "num_tokens": 26610364.0,
1961
+ "step": 4300
1962
+ },
1963
+ {
1964
+ "entropy": 0.78652991771698,
1965
+ "epoch": 1.4971438462870001,
1966
+ "grad_norm": 2.359375,
1967
+ "learning_rate": 3.142115667825868e-06,
1968
+ "loss": 0.7553633117675781,
1969
+ "mean_token_accuracy": 0.8014207696914672,
1970
+ "num_tokens": 26766393.0,
1971
+ "step": 4325
1972
+ },
1973
+ {
1974
+ "entropy": 0.7758943355083465,
1975
+ "epoch": 1.5057988575385148,
1976
+ "grad_norm": 2.0,
1977
+ "learning_rate": 3.0407915184435544e-06,
1978
+ "loss": 0.7414107513427735,
1979
+ "mean_token_accuracy": 0.8034427273273468,
1980
+ "num_tokens": 26928821.0,
1981
+ "step": 4350
1982
+ },
1983
+ {
1984
+ "entropy": 0.7746455824375152,
1985
+ "epoch": 1.5144538687900293,
1986
+ "grad_norm": 2.15625,
1987
+ "learning_rate": 2.9408342675016288e-06,
1988
+ "loss": 0.7338740539550781,
1989
+ "mean_token_accuracy": 0.8053066456317901,
1990
+ "num_tokens": 27081955.0,
1991
+ "step": 4375
1992
+ },
1993
+ {
1994
+ "entropy": 0.7544065153598786,
1995
+ "epoch": 1.5231088800415442,
1996
+ "grad_norm": 2.125,
1997
+ "learning_rate": 2.84226354818263e-06,
1998
+ "loss": 0.7168594360351562,
1999
+ "mean_token_accuracy": 0.8104126238822937,
2000
+ "num_tokens": 27233223.0,
2001
+ "step": 4400
2002
+ },
2003
+ {
2004
+ "epoch": 1.5231088800415442,
2005
+ "eval_entropy": 0.770115079156688,
2006
+ "eval_loss": 0.7993313074111938,
2007
+ "eval_mean_token_accuracy": 0.7934993093131019,
2008
+ "eval_num_tokens": 27233223.0,
2009
+ "eval_runtime": 16.2903,
2010
+ "eval_samples_per_second": 298.644,
2011
+ "eval_steps_per_second": 18.723,
2012
+ "step": 4400
2013
+ },
2014
+ {
2015
+ "entropy": 0.7756598269939423,
2016
+ "epoch": 1.5317638912930587,
2017
+ "grad_norm": 2.15625,
2018
+ "learning_rate": 2.74509872133238e-06,
2019
+ "loss": 0.750801010131836,
2020
+ "mean_token_accuracy": 0.8043915629386902,
2021
+ "num_tokens": 27385671.0,
2022
+ "step": 4425
2023
+ },
2024
+ {
2025
+ "entropy": 0.7870103931427002,
2026
+ "epoch": 1.5404189025445731,
2027
+ "grad_norm": 2.46875,
2028
+ "learning_rate": 2.6493588716572306e-06,
2029
+ "loss": 0.7561086273193359,
2030
+ "mean_token_accuracy": 0.8031144332885742,
2031
+ "num_tokens": 27542323.0,
2032
+ "step": 4450
2033
+ },
2034
+ {
2035
+ "entropy": 0.7758941638469696,
2036
+ "epoch": 1.549073913796088,
2037
+ "grad_norm": 2.359375,
2038
+ "learning_rate": 2.555062803975512e-06,
2039
+ "loss": 0.7479061889648437,
2040
+ "mean_token_accuracy": 0.8041390895843505,
2041
+ "num_tokens": 27695755.0,
2042
+ "step": 4475
2043
+ },
2044
+ {
2045
+ "entropy": 0.754768637418747,
2046
+ "epoch": 1.5577289250476025,
2047
+ "grad_norm": 2.4375,
2048
+ "learning_rate": 2.4622290395239613e-06,
2049
+ "loss": 0.7247459411621093,
2050
+ "mean_token_accuracy": 0.8090137577056885,
2051
+ "num_tokens": 27847694.0,
2052
+ "step": 4500
2053
+ },
2054
+ {
2055
+ "entropy": 0.7636888045072555,
2056
+ "epoch": 1.5663839362991172,
2057
+ "grad_norm": 2.140625,
2058
+ "learning_rate": 2.3708758123198904e-06,
2059
+ "loss": 0.7294938659667969,
2060
+ "mean_token_accuracy": 0.8078456258773804,
2061
+ "num_tokens": 28001389.0,
2062
+ "step": 4525
2063
+ },
2064
+ {
2065
+ "entropy": 0.7723946702480317,
2066
+ "epoch": 1.575038947550632,
2067
+ "grad_norm": 1.9453125,
2068
+ "learning_rate": 2.2810210655796915e-06,
2069
+ "loss": 0.7421239471435547,
2070
+ "mean_token_accuracy": 0.8051141202449799,
2071
+ "num_tokens": 28155633.0,
2072
+ "step": 4550
2073
+ },
2074
+ {
2075
+ "entropy": 0.7729098165035247,
2076
+ "epoch": 1.5836939588021464,
2077
+ "grad_norm": 1.9375,
2078
+ "learning_rate": 2.192682448194555e-06,
2079
+ "loss": 0.7385528564453125,
2080
+ "mean_token_accuracy": 0.8061604249477387,
2081
+ "num_tokens": 28316815.0,
2082
+ "step": 4575
2083
+ },
2084
+ {
2085
+ "entropy": 0.7528582894802094,
2086
+ "epoch": 1.592348970053661,
2087
+ "grad_norm": 2.234375,
2088
+ "learning_rate": 2.1058773112639054e-06,
2089
+ "loss": 0.719280014038086,
2090
+ "mean_token_accuracy": 0.8096978998184204,
2091
+ "num_tokens": 28473438.0,
2092
+ "step": 4600
2093
+ },
2094
+ {
2095
+ "epoch": 1.592348970053661,
2096
+ "eval_entropy": 0.7718443285246365,
2097
+ "eval_loss": 0.7991446852684021,
2098
+ "eval_mean_token_accuracy": 0.7934910416603088,
2099
+ "eval_num_tokens": 28473438.0,
2100
+ "eval_runtime": 16.2845,
2101
+ "eval_samples_per_second": 298.751,
2102
+ "eval_steps_per_second": 18.729,
2103
+ "step": 4600
2104
+ }
2105
+ ],
2106
+ "logging_steps": 25,
2107
+ "max_steps": 5778,
2108
+ "num_input_tokens_seen": 0,
2109
+ "num_train_epochs": 2,
2110
+ "save_steps": 200,
2111
+ "stateful_callbacks": {
2112
+ "TrainerControl": {
2113
+ "args": {
2114
+ "should_epoch_stop": false,
2115
+ "should_evaluate": false,
2116
+ "should_log": false,
2117
+ "should_save": true,
2118
+ "should_training_stop": false
2119
+ },
2120
+ "attributes": {}
2121
+ }
2122
+ },
2123
+ "total_flos": 1.1640330094942925e+17,
2124
+ "train_batch_size": 16,
2125
+ "trial_name": null,
2126
+ "trial_params": null
2127
+ }
checkpoint-4600/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fe1d1ac135572b0e564d6340370a449cb9281b7839efca5df924e60c2ca5dbe
3
+ size 5585
checkpoint-5600/chat_template.jinja ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set keep_past_thinking = keep_past_thinking | default(false) -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set sys_content = messages[0]["content"] -%}
6
+ {%- if sys_content is not string -%}
7
+ {%- for item in sys_content -%}
8
+ {%- if item["type"] == "text" -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + item["text"] -%}
10
+ {%- endif -%}
11
+ {%- endfor -%}
12
+ {%- else -%}
13
+ {%- set ns.system_prompt = sys_content -%}
14
+ {%- endif -%}
15
+ {%- set messages = messages[1:] -%}
16
+ {%- endif -%}
17
+ {%- if tools -%}
18
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
19
+ {%- for tool in tools -%}
20
+ {%- if tool is not string -%}
21
+ {%- set tool = tool | tojson -%}
22
+ {%- endif -%}
23
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
24
+ {%- if not loop.last -%}
25
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- set ns.system_prompt = ns.system_prompt + "]" -%}
29
+ {%- endif -%}
30
+ {%- if ns.system_prompt -%}
31
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
32
+ {%- endif -%}
33
+ {%- set ns.last_assistant_index = -1 -%}
34
+ {%- for message in messages -%}
35
+ {%- if message["role"] == "assistant" -%}
36
+ {%- set ns.last_assistant_index = loop.index0 -%}
37
+ {%- endif -%}
38
+ {%- endfor -%}
39
+ {%- for message in messages -%}
40
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
41
+ {%- set content = message["content"] -%}
42
+ {%- if content is not string -%}
43
+ {%- set ns.content = "" -%}
44
+ {%- for item in content -%}
45
+ {%- if item["type"] == "image" -%}
46
+ {%- set ns.content = ns.content + "<image>" -%}
47
+ {%- elif item["type"] == "text" -%}
48
+ {%- set ns.content = ns.content + item["text"] -%}
49
+ {%- else -%}
50
+ {%- set ns.content = ns.content + item | tojson -%}
51
+ {%- endif -%}
52
+ {%- endfor -%}
53
+ {%- set content = ns.content -%}
54
+ {%- endif -%}
55
+ {%- if message["role"] == "assistant" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}
56
+ {%- if "</think>" in content -%}
57
+ {%- set content = content.split("</think>")[-1] | trim -%}
58
+ {%- endif -%}
59
+ {%- endif -%}
60
+ {{- content + "<|im_end|>\n" -}}
61
+ {%- endfor -%}
62
+ {%- if add_generation_prompt -%}
63
+ {{- "<|im_start|>assistant\n" -}}
64
+ {%- endif -%}
checkpoint-5600/config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2ForCausalLM"
4
+ ],
5
+ "block_auto_adjust_ff_dim": true,
6
+ "block_dim": 1024,
7
+ "block_ffn_dim_multiplier": 1.0,
8
+ "block_mlp_init_scale": 1.0,
9
+ "block_multiple_of": 256,
10
+ "block_norm_eps": 1e-05,
11
+ "block_out_init_scale": 1.0,
12
+ "block_use_swiglu": true,
13
+ "block_use_xavier_init": true,
14
+ "bos_token_id": 1,
15
+ "conv_L_cache": 3,
16
+ "conv_bias": false,
17
+ "conv_dim": 1024,
18
+ "conv_use_xavier_init": true,
19
+ "dtype": "bfloat16",
20
+ "eos_token_id": 7,
21
+ "full_attn_idxs": null,
22
+ "hidden_size": 1024,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 6656,
25
+ "layer_types": [
26
+ "conv",
27
+ "conv",
28
+ "full_attention",
29
+ "conv",
30
+ "conv",
31
+ "full_attention",
32
+ "conv",
33
+ "conv",
34
+ "full_attention",
35
+ "conv",
36
+ "full_attention",
37
+ "conv",
38
+ "full_attention",
39
+ "conv",
40
+ "full_attention",
41
+ "conv"
42
+ ],
43
+ "max_position_embeddings": 128000,
44
+ "model_type": "lfm2",
45
+ "norm_eps": 1e-05,
46
+ "num_attention_heads": 16,
47
+ "num_heads": 16,
48
+ "num_hidden_layers": 16,
49
+ "num_key_value_heads": 8,
50
+ "pad_token_id": 0,
51
+ "rope_parameters": {
52
+ "rope_theta": 1000000.0,
53
+ "rope_type": "default"
54
+ },
55
+ "tie_word_embeddings": true,
56
+ "transformers_version": "5.5.0",
57
+ "use_cache": false,
58
+ "use_pos_enc": true,
59
+ "vocab_size": 65536
60
+ }
checkpoint-5600/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 7
6
+ ],
7
+ "pad_token_id": 0,
8
+ "transformers_version": "5.5.0"
9
+ }
checkpoint-5600/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33be506f3c59b5a4be6805f0dc6e79086f97f039d33112e1442a8c30628b9aee
3
+ size 708984464
checkpoint-5600/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7e6b7b93ad8219a24d21e48ff24769e07fc7ba806aeb0e1e9c91998ec8c4cae
3
+ size 1418061003
checkpoint-5600/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf39545e0855266b895e9e8aa78628a050a7b25e5fca8eb923e549a23a89c550
3
+ size 14645
checkpoint-5600/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:728d2612f0f802b8d590b20c31e35481c0dde7e6d6db6b68c23ae074019a92a7
3
+ size 1465
checkpoint-5600/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-5600/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|im_end|>",
6
+ "extra_special_tokens": [],
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "model_input_names": [
10
+ "input_ids",
11
+ "attention_mask"
12
+ ],
13
+ "model_max_length": 1000000000000000019884624838656,
14
+ "model_specific_special_tokens": {},
15
+ "pad_token": "<|pad|>",
16
+ "sp_model_kwargs": {},
17
+ "spaces_between_special_tokens": false,
18
+ "tokenizer_class": "TokenizersBackend",
19
+ "use_default_system_prompt": false,
20
+ "use_fast": true
21
+ }
checkpoint-5600/trainer_state.json ADDED
@@ -0,0 +1,2582 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 4600,
3
+ "best_metric": 0.7991446852684021,
4
+ "best_model_checkpoint": "lfm2.5-350m-multilang-code-math/checkpoint-4600",
5
+ "epoch": 1.9385494201142461,
6
+ "eval_steps": 200,
7
+ "global_step": 5600,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2043220043182372,
14
+ "epoch": 0.008655011251514627,
15
+ "grad_norm": 23.5,
16
+ "learning_rate": 2.7586206896551725e-06,
17
+ "loss": 3.0389138793945314,
18
+ "mean_token_accuracy": 0.5589780753850937,
19
+ "num_tokens": 151155.0,
20
+ "step": 25
21
+ },
22
+ {
23
+ "entropy": 1.11516796708107,
24
+ "epoch": 0.017310022503029255,
25
+ "grad_norm": 7.34375,
26
+ "learning_rate": 5.6321839080459775e-06,
27
+ "loss": 1.8928765869140625,
28
+ "mean_token_accuracy": 0.6762128543853759,
29
+ "num_tokens": 310957.0,
30
+ "step": 50
31
+ },
32
+ {
33
+ "entropy": 1.2031074380874633,
34
+ "epoch": 0.02596503375454388,
35
+ "grad_norm": 3.78125,
36
+ "learning_rate": 8.505747126436782e-06,
37
+ "loss": 1.2897039794921874,
38
+ "mean_token_accuracy": 0.7282541489601135,
39
+ "num_tokens": 461068.0,
40
+ "step": 75
41
+ },
42
+ {
43
+ "entropy": 1.0323714792728425,
44
+ "epoch": 0.03462004500605851,
45
+ "grad_norm": 3.71875,
46
+ "learning_rate": 1.1379310344827587e-05,
47
+ "loss": 1.0183975982666016,
48
+ "mean_token_accuracy": 0.757768098115921,
49
+ "num_tokens": 615118.0,
50
+ "step": 100
51
+ },
52
+ {
53
+ "entropy": 0.9247142732143402,
54
+ "epoch": 0.043275056257573136,
55
+ "grad_norm": 2.578125,
56
+ "learning_rate": 1.4252873563218392e-05,
57
+ "loss": 0.8757442474365235,
58
+ "mean_token_accuracy": 0.7792941844463348,
59
+ "num_tokens": 762906.0,
60
+ "step": 125
61
+ },
62
+ {
63
+ "entropy": 0.9190187740325928,
64
+ "epoch": 0.05193006750908776,
65
+ "grad_norm": 2.296875,
66
+ "learning_rate": 1.7126436781609197e-05,
67
+ "loss": 0.891803970336914,
68
+ "mean_token_accuracy": 0.7753661525249481,
69
+ "num_tokens": 921356.0,
70
+ "step": 150
71
+ },
72
+ {
73
+ "entropy": 0.886636551618576,
74
+ "epoch": 0.06058507876060239,
75
+ "grad_norm": 2.296875,
76
+ "learning_rate": 2e-05,
77
+ "loss": 0.861456298828125,
78
+ "mean_token_accuracy": 0.7826532816886902,
79
+ "num_tokens": 1076183.0,
80
+ "step": 175
81
+ },
82
+ {
83
+ "entropy": 0.9015009069442749,
84
+ "epoch": 0.06924009001211702,
85
+ "grad_norm": 2.5625,
86
+ "learning_rate": 1.9999017921043748e-05,
87
+ "loss": 0.8748048400878906,
88
+ "mean_token_accuracy": 0.7792416322231293,
89
+ "num_tokens": 1229170.0,
90
+ "step": 200
91
+ },
92
+ {
93
+ "epoch": 0.06924009001211702,
94
+ "eval_entropy": 0.8920288547140653,
95
+ "eval_loss": 0.8666635155677795,
96
+ "eval_mean_token_accuracy": 0.7797926357535065,
97
+ "eval_num_tokens": 1229170.0,
98
+ "eval_runtime": 16.3134,
99
+ "eval_samples_per_second": 298.222,
100
+ "eval_steps_per_second": 18.696,
101
+ "step": 200
102
+ },
103
+ {
104
+ "entropy": 0.9062747704982758,
105
+ "epoch": 0.07789510126363164,
106
+ "grad_norm": 2.40625,
107
+ "learning_rate": 1.999607187707079e-05,
108
+ "loss": 0.8789938354492187,
109
+ "mean_token_accuracy": 0.7778702569007874,
110
+ "num_tokens": 1381380.0,
111
+ "step": 225
112
+ },
113
+ {
114
+ "entropy": 0.8776320171356201,
115
+ "epoch": 0.08655011251514627,
116
+ "grad_norm": 2.46875,
117
+ "learning_rate": 1.9991162446730694e-05,
118
+ "loss": 0.8517158508300782,
119
+ "mean_token_accuracy": 0.7819743084907532,
120
+ "num_tokens": 1538186.0,
121
+ "step": 250
122
+ },
123
+ {
124
+ "entropy": 0.8657179689407348,
125
+ "epoch": 0.09520512376666089,
126
+ "grad_norm": 2.25,
127
+ "learning_rate": 1.9984290594313102e-05,
128
+ "loss": 0.8408647155761719,
129
+ "mean_token_accuracy": 0.7840179252624512,
130
+ "num_tokens": 1690067.0,
131
+ "step": 275
132
+ },
133
+ {
134
+ "entropy": 0.8955290937423706,
135
+ "epoch": 0.10386013501817552,
136
+ "grad_norm": 2.328125,
137
+ "learning_rate": 1.9975457669558346e-05,
138
+ "loss": 0.8683735656738282,
139
+ "mean_token_accuracy": 0.7763688516616821,
140
+ "num_tokens": 1845254.0,
141
+ "step": 300
142
+ },
143
+ {
144
+ "entropy": 0.8744900381565094,
145
+ "epoch": 0.11251514626969016,
146
+ "grad_norm": 2.34375,
147
+ "learning_rate": 1.996466540739233e-05,
148
+ "loss": 0.8470248413085938,
149
+ "mean_token_accuracy": 0.7828968441486359,
150
+ "num_tokens": 2000855.0,
151
+ "step": 325
152
+ },
153
+ {
154
+ "entropy": 0.8875207018852234,
155
+ "epoch": 0.12117015752120477,
156
+ "grad_norm": 2.34375,
157
+ "learning_rate": 1.9951915927585762e-05,
158
+ "loss": 0.8633122253417969,
159
+ "mean_token_accuracy": 0.7807920384407043,
160
+ "num_tokens": 2159255.0,
161
+ "step": 350
162
+ },
163
+ {
164
+ "entropy": 0.8727393555641174,
165
+ "epoch": 0.1298251687727194,
166
+ "grad_norm": 2.40625,
167
+ "learning_rate": 1.9937211734337812e-05,
168
+ "loss": 0.8484575653076172,
169
+ "mean_token_accuracy": 0.7856597316265106,
170
+ "num_tokens": 2313980.0,
171
+ "step": 375
172
+ },
173
+ {
174
+ "entropy": 0.8450514304637909,
175
+ "epoch": 0.13848018002423404,
176
+ "grad_norm": 2.390625,
177
+ "learning_rate": 1.992055571578423e-05,
178
+ "loss": 0.8231178283691406,
179
+ "mean_token_accuracy": 0.7884540009498596,
180
+ "num_tokens": 2468392.0,
181
+ "step": 400
182
+ },
183
+ {
184
+ "epoch": 0.13848018002423404,
185
+ "eval_entropy": 0.8820633643963298,
186
+ "eval_loss": 0.8462674021720886,
187
+ "eval_mean_token_accuracy": 0.7830698802822926,
188
+ "eval_num_tokens": 2468392.0,
189
+ "eval_runtime": 16.2902,
190
+ "eval_samples_per_second": 298.645,
191
+ "eval_steps_per_second": 18.723,
192
+ "step": 400
193
+ },
194
+ {
195
+ "entropy": 0.8852161896228791,
196
+ "epoch": 0.14713519127574864,
197
+ "grad_norm": 2.25,
198
+ "learning_rate": 1.990195114343008e-05,
199
+ "loss": 0.8604666137695313,
200
+ "mean_token_accuracy": 0.7796429479122162,
201
+ "num_tokens": 2626504.0,
202
+ "step": 425
203
+ },
204
+ {
205
+ "entropy": 0.8595891773700715,
206
+ "epoch": 0.15579020252726328,
207
+ "grad_norm": 2.109375,
208
+ "learning_rate": 1.988140167150716e-05,
209
+ "loss": 0.8375725555419922,
210
+ "mean_token_accuracy": 0.7837563192844391,
211
+ "num_tokens": 2785992.0,
212
+ "step": 450
213
+ },
214
+ {
215
+ "entropy": 0.8787437427043915,
216
+ "epoch": 0.1644452137787779,
217
+ "grad_norm": 2.21875,
218
+ "learning_rate": 1.9858911336256258e-05,
219
+ "loss": 0.8602984619140625,
220
+ "mean_token_accuracy": 0.7801311421394348,
221
+ "num_tokens": 2943389.0,
222
+ "step": 475
223
+ },
224
+ {
225
+ "entropy": 0.8887673962116241,
226
+ "epoch": 0.17310022503029254,
227
+ "grad_norm": 2.34375,
228
+ "learning_rate": 1.983448455513437e-05,
229
+ "loss": 0.8559275054931641,
230
+ "mean_token_accuracy": 0.7799410057067871,
231
+ "num_tokens": 3098910.0,
232
+ "step": 500
233
+ },
234
+ {
235
+ "entropy": 0.8632562351226807,
236
+ "epoch": 0.18175523628180718,
237
+ "grad_norm": 2.359375,
238
+ "learning_rate": 1.9808126125947035e-05,
239
+ "loss": 0.8414566040039062,
240
+ "mean_token_accuracy": 0.7838216722011566,
241
+ "num_tokens": 3259790.0,
242
+ "step": 525
243
+ },
244
+ {
245
+ "entropy": 0.8468562078475952,
246
+ "epoch": 0.19041024753332178,
247
+ "grad_norm": 2.921875,
248
+ "learning_rate": 1.977984122590598e-05,
249
+ "loss": 0.8214312744140625,
250
+ "mean_token_accuracy": 0.7904518485069275,
251
+ "num_tokens": 3408921.0,
252
+ "step": 550
253
+ },
254
+ {
255
+ "entropy": 0.862267234325409,
256
+ "epoch": 0.1990652587848364,
257
+ "grad_norm": 2.21875,
258
+ "learning_rate": 1.9749635410612222e-05,
259
+ "loss": 0.8381607818603516,
260
+ "mean_token_accuracy": 0.7850737714767456,
261
+ "num_tokens": 3564021.0,
262
+ "step": 575
263
+ },
264
+ {
265
+ "entropy": 0.8673117184638977,
266
+ "epoch": 0.20772027003635105,
267
+ "grad_norm": 2.28125,
268
+ "learning_rate": 1.971751461296488e-05,
269
+ "loss": 0.8422865295410156,
270
+ "mean_token_accuracy": 0.7839160299301148,
271
+ "num_tokens": 3720222.0,
272
+ "step": 600
273
+ },
274
+ {
275
+ "epoch": 0.20772027003635105,
276
+ "eval_entropy": 0.8581681211463741,
277
+ "eval_loss": 0.8380036950111389,
278
+ "eval_mean_token_accuracy": 0.7849426593936858,
279
+ "eval_num_tokens": 3720222.0,
280
+ "eval_runtime": 16.2659,
281
+ "eval_samples_per_second": 299.091,
282
+ "eval_steps_per_second": 18.751,
283
+ "step": 600
284
+ },
285
+ {
286
+ "entropy": 0.896509393453598,
287
+ "epoch": 0.21637528128786568,
288
+ "grad_norm": 2.25,
289
+ "learning_rate": 1.9683485141995832e-05,
290
+ "loss": 0.8751831817626953,
291
+ "mean_token_accuracy": 0.778223477602005,
292
+ "num_tokens": 3868288.0,
293
+ "step": 625
294
+ },
295
+ {
296
+ "entropy": 0.8858666527271271,
297
+ "epoch": 0.2250302925393803,
298
+ "grad_norm": 2.4375,
299
+ "learning_rate": 1.964755368163055e-05,
300
+ "loss": 0.8597821807861328,
301
+ "mean_token_accuracy": 0.7796040976047516,
302
+ "num_tokens": 4020387.0,
303
+ "step": 650
304
+ },
305
+ {
306
+ "entropy": 0.8457015335559845,
307
+ "epoch": 0.23368530379089492,
308
+ "grad_norm": 2.0625,
309
+ "learning_rate": 1.9609727289375254e-05,
310
+ "loss": 0.8197212982177734,
311
+ "mean_token_accuracy": 0.7878750622272491,
312
+ "num_tokens": 4175769.0,
313
+ "step": 675
314
+ },
315
+ {
316
+ "entropy": 0.8535351765155792,
317
+ "epoch": 0.24234031504240955,
318
+ "grad_norm": 2.328125,
319
+ "learning_rate": 1.9570013394930704e-05,
320
+ "loss": 0.8300620269775391,
321
+ "mean_token_accuracy": 0.786250706911087,
322
+ "num_tokens": 4331114.0,
323
+ "step": 700
324
+ },
325
+ {
326
+ "entropy": 0.8614680600166321,
327
+ "epoch": 0.2509953262939242,
328
+ "grad_norm": 2.0625,
329
+ "learning_rate": 1.9528419798732902e-05,
330
+ "loss": 0.8407713317871094,
331
+ "mean_token_accuracy": 0.7833027255535125,
332
+ "num_tokens": 4488237.0,
333
+ "step": 725
334
+ },
335
+ {
336
+ "entropy": 0.8501963889598847,
337
+ "epoch": 0.2596503375454388,
338
+ "grad_norm": 2.328125,
339
+ "learning_rate": 1.948495467042095e-05,
340
+ "loss": 0.8218775177001953,
341
+ "mean_token_accuracy": 0.788744968175888,
342
+ "num_tokens": 4642068.0,
343
+ "step": 750
344
+ },
345
+ {
346
+ "entropy": 0.8383999621868133,
347
+ "epoch": 0.26830534879695345,
348
+ "grad_norm": 2.296875,
349
+ "learning_rate": 1.9439626547232433e-05,
350
+ "loss": 0.80978515625,
351
+ "mean_token_accuracy": 0.7903480684757233,
352
+ "num_tokens": 4788314.0,
353
+ "step": 775
354
+ },
355
+ {
356
+ "entropy": 0.8568655729293824,
357
+ "epoch": 0.2769603600484681,
358
+ "grad_norm": 2.53125,
359
+ "learning_rate": 1.939244433232652e-05,
360
+ "loss": 0.8343601226806641,
361
+ "mean_token_accuracy": 0.7855056190490722,
362
+ "num_tokens": 4941131.0,
363
+ "step": 800
364
+ },
365
+ {
366
+ "epoch": 0.2769603600484681,
367
+ "eval_entropy": 0.859573404906226,
368
+ "eval_loss": 0.8307726383209229,
369
+ "eval_mean_token_accuracy": 0.7864908138259512,
370
+ "eval_num_tokens": 4941131.0,
371
+ "eval_runtime": 16.2516,
372
+ "eval_samples_per_second": 299.355,
373
+ "eval_steps_per_second": 18.767,
374
+ "step": 800
375
+ },
376
+ {
377
+ "entropy": 0.8670335674285888,
378
+ "epoch": 0.2856153712999827,
379
+ "grad_norm": 1.9765625,
380
+ "learning_rate": 1.934341729303529e-05,
381
+ "loss": 0.8394245910644531,
382
+ "mean_token_accuracy": 0.7841019070148468,
383
+ "num_tokens": 5096940.0,
384
+ "step": 825
385
+ },
386
+ {
387
+ "entropy": 0.867160736322403,
388
+ "epoch": 0.2942703825514973,
389
+ "grad_norm": 2.21875,
390
+ "learning_rate": 1.9292555059043457e-05,
391
+ "loss": 0.8409418487548828,
392
+ "mean_token_accuracy": 0.7830280113220215,
393
+ "num_tokens": 5250616.0,
394
+ "step": 850
395
+ },
396
+ {
397
+ "entropy": 0.8582142984867096,
398
+ "epoch": 0.3029253938030119,
399
+ "grad_norm": 2.15625,
400
+ "learning_rate": 1.9239867620496952e-05,
401
+ "loss": 0.8346880340576172,
402
+ "mean_token_accuracy": 0.783586630821228,
403
+ "num_tokens": 5408253.0,
404
+ "step": 875
405
+ },
406
+ {
407
+ "entropy": 0.84959406375885,
408
+ "epoch": 0.31158040505452655,
409
+ "grad_norm": 2.1875,
410
+ "learning_rate": 1.9185365326040713e-05,
411
+ "loss": 0.826163330078125,
412
+ "mean_token_accuracy": 0.7881129395961761,
413
+ "num_tokens": 5564921.0,
414
+ "step": 900
415
+ },
416
+ {
417
+ "entropy": 0.8895779585838318,
418
+ "epoch": 0.3202354163060412,
419
+ "grad_norm": 2.1875,
420
+ "learning_rate": 1.9129058880786026e-05,
421
+ "loss": 0.866309585571289,
422
+ "mean_token_accuracy": 0.7782393181324005,
423
+ "num_tokens": 5722704.0,
424
+ "step": 925
425
+ },
426
+ {
427
+ "entropy": 0.8372803354263305,
428
+ "epoch": 0.3288904275575558,
429
+ "grad_norm": 1.890625,
430
+ "learning_rate": 1.907095934420789e-05,
431
+ "loss": 0.8083054351806641,
432
+ "mean_token_accuracy": 0.7903852713108063,
433
+ "num_tokens": 5881153.0,
434
+ "step": 950
435
+ },
436
+ {
437
+ "entropy": 0.8762626206874847,
438
+ "epoch": 0.33754543880907045,
439
+ "grad_norm": 1.96875,
440
+ "learning_rate": 1.9011078127972748e-05,
441
+ "loss": 0.8495908355712891,
442
+ "mean_token_accuracy": 0.7841223049163818,
443
+ "num_tokens": 6039323.0,
444
+ "step": 975
445
+ },
446
+ {
447
+ "entropy": 0.8670505321025849,
448
+ "epoch": 0.3462004500605851,
449
+ "grad_norm": 2.25,
450
+ "learning_rate": 1.8949426993697074e-05,
451
+ "loss": 0.8448488616943359,
452
+ "mean_token_accuracy": 0.785959769487381,
453
+ "num_tokens": 6197683.0,
454
+ "step": 1000
455
+ },
456
+ {
457
+ "epoch": 0.3462004500605851,
458
+ "eval_entropy": 0.8536441159052927,
459
+ "eval_loss": 0.8251546025276184,
460
+ "eval_mean_token_accuracy": 0.7873384729760592,
461
+ "eval_num_tokens": 6197683.0,
462
+ "eval_runtime": 16.2591,
463
+ "eval_samples_per_second": 299.217,
464
+ "eval_steps_per_second": 18.759,
465
+ "step": 1000
466
+ },
467
+ {
468
+ "entropy": 0.8527129173278809,
469
+ "epoch": 0.3548554613120997,
470
+ "grad_norm": 2.171875,
471
+ "learning_rate": 1.888601805063719e-05,
472
+ "loss": 0.8221394348144532,
473
+ "mean_token_accuracy": 0.7878592002391815,
474
+ "num_tokens": 6353591.0,
475
+ "step": 1025
476
+ },
477
+ {
478
+ "entropy": 0.8309027600288391,
479
+ "epoch": 0.36351047256361435,
480
+ "grad_norm": 2.1875,
481
+ "learning_rate": 1.8820863753310812e-05,
482
+ "loss": 0.804949951171875,
483
+ "mean_token_accuracy": 0.7907659840583802,
484
+ "num_tokens": 6503522.0,
485
+ "step": 1050
486
+ },
487
+ {
488
+ "entropy": 0.8664905667304993,
489
+ "epoch": 0.372165483815129,
490
+ "grad_norm": 2.234375,
491
+ "learning_rate": 1.875397689905081e-05,
492
+ "loss": 0.8386592102050782,
493
+ "mean_token_accuracy": 0.783461629152298,
494
+ "num_tokens": 6657038.0,
495
+ "step": 1075
496
+ },
497
+ {
498
+ "entropy": 0.8521995890140533,
499
+ "epoch": 0.38082049506664356,
500
+ "grad_norm": 2.140625,
501
+ "learning_rate": 1.8685370625491582e-05,
502
+ "loss": 0.8274839782714843,
503
+ "mean_token_accuracy": 0.7857646262645721,
504
+ "num_tokens": 6813403.0,
505
+ "step": 1100
506
+ },
507
+ {
508
+ "entropy": 0.85324840426445,
509
+ "epoch": 0.3894755063181582,
510
+ "grad_norm": 2.03125,
511
+ "learning_rate": 1.861505840798864e-05,
512
+ "loss": 0.8251450347900391,
513
+ "mean_token_accuracy": 0.7872388279438018,
514
+ "num_tokens": 6969337.0,
515
+ "step": 1125
516
+ },
517
+ {
518
+ "entropy": 0.8578063642978668,
519
+ "epoch": 0.3981305175696728,
520
+ "grad_norm": 2.09375,
521
+ "learning_rate": 1.8543054056971815e-05,
522
+ "loss": 0.834039077758789,
523
+ "mean_token_accuracy": 0.7848770570755005,
524
+ "num_tokens": 7124334.0,
525
+ "step": 1150
526
+ },
527
+ {
528
+ "entropy": 0.8491312241554261,
529
+ "epoch": 0.40678552882118746,
530
+ "grad_norm": 2.0625,
531
+ "learning_rate": 1.8469371715232686e-05,
532
+ "loss": 0.8220222473144532,
533
+ "mean_token_accuracy": 0.7871887254714965,
534
+ "num_tokens": 7277475.0,
535
+ "step": 1175
536
+ },
537
+ {
538
+ "entropy": 0.8443890881538391,
539
+ "epoch": 0.4154405400727021,
540
+ "grad_norm": 2.203125,
541
+ "learning_rate": 1.839402585514671e-05,
542
+ "loss": 0.8166568756103516,
543
+ "mean_token_accuracy": 0.7897883796691895,
544
+ "num_tokens": 7428616.0,
545
+ "step": 1200
546
+ },
547
+ {
548
+ "epoch": 0.4154405400727021,
549
+ "eval_entropy": 0.8290919256014903,
550
+ "eval_loss": 0.8211944103240967,
551
+ "eval_mean_token_accuracy": 0.788363546621604,
552
+ "eval_num_tokens": 7428616.0,
553
+ "eval_runtime": 16.2451,
554
+ "eval_samples_per_second": 299.474,
555
+ "eval_steps_per_second": 18.775,
556
+ "step": 1200
557
+ },
558
+ {
559
+ "entropy": 0.8553554141521453,
560
+ "epoch": 0.4240955513242167,
561
+ "grad_norm": 2.140625,
562
+ "learning_rate": 1.8317031275830607e-05,
563
+ "loss": 0.8335649108886719,
564
+ "mean_token_accuracy": 0.7848791778087616,
565
+ "num_tokens": 7586653.0,
566
+ "step": 1225
567
+ },
568
+ {
569
+ "entropy": 0.8513217878341675,
570
+ "epoch": 0.43275056257573136,
571
+ "grad_norm": 2.078125,
572
+ "learning_rate": 1.8238403100235602e-05,
573
+ "loss": 0.8226930999755859,
574
+ "mean_token_accuracy": 0.7892848730087281,
575
+ "num_tokens": 7736619.0,
576
+ "step": 1250
577
+ },
578
+ {
579
+ "entropy": 0.8567298305034637,
580
+ "epoch": 0.441405573827246,
581
+ "grad_norm": 2.25,
582
+ "learning_rate": 1.8158156772177015e-05,
583
+ "loss": 0.8301739501953125,
584
+ "mean_token_accuracy": 0.7857687485218048,
585
+ "num_tokens": 7895674.0,
586
+ "step": 1275
587
+ },
588
+ {
589
+ "entropy": 0.8383160126209259,
590
+ "epoch": 0.4500605850787606,
591
+ "grad_norm": 2.28125,
592
+ "learning_rate": 1.807630805330087e-05,
593
+ "loss": 0.8143462371826172,
594
+ "mean_token_accuracy": 0.7904347920417786,
595
+ "num_tokens": 8052368.0,
596
+ "step": 1300
597
+ },
598
+ {
599
+ "entropy": 0.858016027212143,
600
+ "epoch": 0.45871559633027525,
601
+ "grad_norm": 2.328125,
602
+ "learning_rate": 1.7992873019988044e-05,
603
+ "loss": 0.8361105346679687,
604
+ "mean_token_accuracy": 0.7837494373321533,
605
+ "num_tokens": 8209037.0,
606
+ "step": 1325
607
+ },
608
+ {
609
+ "entropy": 0.8485116994380951,
610
+ "epoch": 0.46737060758178983,
611
+ "grad_norm": 2.078125,
612
+ "learning_rate": 1.7907868060196627e-05,
613
+ "loss": 0.8189002990722656,
614
+ "mean_token_accuracy": 0.7860724258422852,
615
+ "num_tokens": 8366536.0,
616
+ "step": 1350
617
+ },
618
+ {
619
+ "entropy": 0.8689926111698151,
620
+ "epoch": 0.47602561883330446,
621
+ "grad_norm": 2.28125,
622
+ "learning_rate": 1.7821309870243056e-05,
623
+ "loss": 0.8450764465332031,
624
+ "mean_token_accuracy": 0.7817527639865876,
625
+ "num_tokens": 8524698.0,
626
+ "step": 1375
627
+ },
628
+ {
629
+ "entropy": 0.8510251712799072,
630
+ "epoch": 0.4846806300848191,
631
+ "grad_norm": 2.03125,
632
+ "learning_rate": 1.77332154515227e-05,
633
+ "loss": 0.8352996063232422,
634
+ "mean_token_accuracy": 0.786122510433197,
635
+ "num_tokens": 8680719.0,
636
+ "step": 1400
637
+ },
638
+ {
639
+ "epoch": 0.4846806300848191,
640
+ "eval_entropy": 0.8555903234442727,
641
+ "eval_loss": 0.8170819878578186,
642
+ "eval_mean_token_accuracy": 0.7890570513537672,
643
+ "eval_num_tokens": 8680719.0,
644
+ "eval_runtime": 16.2534,
645
+ "eval_samples_per_second": 299.322,
646
+ "eval_steps_per_second": 18.765,
647
+ "step": 1400
648
+ },
649
+ {
650
+ "entropy": 0.8443090248107911,
651
+ "epoch": 0.49333564133633373,
652
+ "grad_norm": 2.34375,
653
+ "learning_rate": 1.764360210717051e-05,
654
+ "loss": 0.8160585784912109,
655
+ "mean_token_accuracy": 0.7888865935802459,
656
+ "num_tokens": 8839183.0,
657
+ "step": 1425
658
+ },
659
+ {
660
+ "entropy": 0.8397502684593201,
661
+ "epoch": 0.5019906525878484,
662
+ "grad_norm": 2.015625,
663
+ "learning_rate": 1.7552487438662432e-05,
664
+ "loss": 0.8148053741455078,
665
+ "mean_token_accuracy": 0.7885977244377136,
666
+ "num_tokens": 8995586.0,
667
+ "step": 1450
668
+ },
669
+ {
670
+ "entropy": 0.8441075325012207,
671
+ "epoch": 0.5106456638393629,
672
+ "grad_norm": 2.34375,
673
+ "learning_rate": 1.7459889342358176e-05,
674
+ "loss": 0.8217481994628906,
675
+ "mean_token_accuracy": 0.7888561117649079,
676
+ "num_tokens": 9146595.0,
677
+ "step": 1475
678
+ },
679
+ {
680
+ "entropy": 0.8394048309326172,
681
+ "epoch": 0.5193006750908776,
682
+ "grad_norm": 2.296875,
683
+ "learning_rate": 1.736582600598609e-05,
684
+ "loss": 0.8105857849121094,
685
+ "mean_token_accuracy": 0.7910154783725738,
686
+ "num_tokens": 9297789.0,
687
+ "step": 1500
688
+ },
689
+ {
690
+ "entropy": 0.8256420779228211,
691
+ "epoch": 0.5279556863423922,
692
+ "grad_norm": 2.140625,
693
+ "learning_rate": 1.7270315905070822e-05,
694
+ "loss": 0.7967173004150391,
695
+ "mean_token_accuracy": 0.7917508590221405,
696
+ "num_tokens": 9450532.0,
697
+ "step": 1525
698
+ },
699
+ {
700
+ "entropy": 0.817123122215271,
701
+ "epoch": 0.5366106975939069,
702
+ "grad_norm": 1.8828125,
703
+ "learning_rate": 1.717337779930441e-05,
704
+ "loss": 0.7964045715332031,
705
+ "mean_token_accuracy": 0.793145546913147,
706
+ "num_tokens": 9605286.0,
707
+ "step": 1550
708
+ },
709
+ {
710
+ "entropy": 0.8218163585662842,
711
+ "epoch": 0.5452657088454215,
712
+ "grad_norm": 2.25,
713
+ "learning_rate": 1.7075030728861602e-05,
714
+ "loss": 0.7938800048828125,
715
+ "mean_token_accuracy": 0.7939867222309113,
716
+ "num_tokens": 9755402.0,
717
+ "step": 1575
718
+ },
719
+ {
720
+ "entropy": 0.8473324692249298,
721
+ "epoch": 0.5539207200969362,
722
+ "grad_norm": 2.171875,
723
+ "learning_rate": 1.697529401066006e-05,
724
+ "loss": 0.8210481262207031,
725
+ "mean_token_accuracy": 0.7879379904270172,
726
+ "num_tokens": 9915397.0,
727
+ "step": 1600
728
+ },
729
+ {
730
+ "epoch": 0.5539207200969362,
731
+ "eval_entropy": 0.8279588664164309,
732
+ "eval_loss": 0.8136925101280212,
733
+ "eval_mean_token_accuracy": 0.7897968792524495,
734
+ "eval_num_tokens": 9915397.0,
735
+ "eval_runtime": 16.2401,
736
+ "eval_samples_per_second": 299.567,
737
+ "eval_steps_per_second": 18.781,
738
+ "step": 1600
739
+ },
740
+ {
741
+ "entropy": 0.83288947224617,
742
+ "epoch": 0.5625757313484507,
743
+ "grad_norm": 2.125,
744
+ "learning_rate": 1.6874187234566198e-05,
745
+ "loss": 0.8104652404785156,
746
+ "mean_token_accuracy": 0.7881759464740753,
747
+ "num_tokens": 10066176.0,
748
+ "step": 1625
749
+ },
750
+ {
751
+ "entropy": 0.8406240797042847,
752
+ "epoch": 0.5712307425999654,
753
+ "grad_norm": 2.21875,
754
+ "learning_rate": 1.677173025954745e-05,
755
+ "loss": 0.8155978393554687,
756
+ "mean_token_accuracy": 0.7885338914394379,
757
+ "num_tokens": 10216630.0,
758
+ "step": 1650
759
+ },
760
+ {
761
+ "entropy": 0.8361802196502686,
762
+ "epoch": 0.57988575385148,
763
+ "grad_norm": 2.078125,
764
+ "learning_rate": 1.6667943209771634e-05,
765
+ "loss": 0.8124593353271484,
766
+ "mean_token_accuracy": 0.7909375095367431,
767
+ "num_tokens": 10368775.0,
768
+ "step": 1675
769
+ },
770
+ {
771
+ "entropy": 0.8546354687213897,
772
+ "epoch": 0.5885407651029946,
773
+ "grad_norm": 2.390625,
774
+ "learning_rate": 1.6562846470654253e-05,
775
+ "loss": 0.8320878601074219,
776
+ "mean_token_accuracy": 0.7862278306484223,
777
+ "num_tokens": 10522306.0,
778
+ "step": 1700
779
+ },
780
+ {
781
+ "entropy": 0.8271597623825073,
782
+ "epoch": 0.5971957763545093,
783
+ "grad_norm": 2.359375,
784
+ "learning_rate": 1.6456460684854474e-05,
785
+ "loss": 0.802950210571289,
786
+ "mean_token_accuracy": 0.7914960932731628,
787
+ "num_tokens": 10677278.0,
788
+ "step": 1725
789
+ },
790
+ {
791
+ "entropy": 0.8680684280395508,
792
+ "epoch": 0.6058507876060238,
793
+ "grad_norm": 2.21875,
794
+ "learning_rate": 1.6348806748220596e-05,
795
+ "loss": 0.8468040466308594,
796
+ "mean_token_accuracy": 0.7825022864341736,
797
+ "num_tokens": 10832759.0,
798
+ "step": 1750
799
+ },
800
+ {
801
+ "entropy": 0.8460379076004029,
802
+ "epoch": 0.6145057988575385,
803
+ "grad_norm": 1.90625,
804
+ "learning_rate": 1.6239905805685766e-05,
805
+ "loss": 0.81728515625,
806
+ "mean_token_accuracy": 0.7885628819465638,
807
+ "num_tokens": 10987982.0,
808
+ "step": 1775
809
+ },
810
+ {
811
+ "entropy": 0.8258041095733643,
812
+ "epoch": 0.6231608101090531,
813
+ "grad_norm": 1.9609375,
814
+ "learning_rate": 1.6129779247114776e-05,
815
+ "loss": 0.8052913665771484,
816
+ "mean_token_accuracy": 0.7923011314868927,
817
+ "num_tokens": 11142409.0,
818
+ "step": 1800
819
+ },
820
+ {
821
+ "epoch": 0.6231608101090531,
822
+ "eval_entropy": 0.8428774887421092,
823
+ "eval_loss": 0.8103926181793213,
824
+ "eval_mean_token_accuracy": 0.790385250967057,
825
+ "eval_num_tokens": 11142409.0,
826
+ "eval_runtime": 16.2616,
827
+ "eval_samples_per_second": 299.17,
828
+ "eval_steps_per_second": 18.756,
829
+ "step": 1800
830
+ },
831
+ {
832
+ "entropy": 0.8314215159416198,
833
+ "epoch": 0.6318158213605678,
834
+ "grad_norm": 2.09375,
835
+ "learning_rate": 1.601844870310277e-05,
836
+ "loss": 0.8011739349365234,
837
+ "mean_token_accuracy": 0.7905190980434418,
838
+ "num_tokens": 11295629.0,
839
+ "step": 1825
840
+ },
841
+ {
842
+ "entropy": 0.8174971842765808,
843
+ "epoch": 0.6404708326120824,
844
+ "grad_norm": 2.03125,
845
+ "learning_rate": 1.5905936040726636e-05,
846
+ "loss": 0.7869554138183594,
847
+ "mean_token_accuracy": 0.7940086150169372,
848
+ "num_tokens": 11449328.0,
849
+ "step": 1850
850
+ },
851
+ {
852
+ "entropy": 0.8386989748477935,
853
+ "epoch": 0.6491258438635971,
854
+ "grad_norm": 2.03125,
855
+ "learning_rate": 1.579226335924999e-05,
856
+ "loss": 0.8210865783691407,
857
+ "mean_token_accuracy": 0.7893707096576691,
858
+ "num_tokens": 11605530.0,
859
+ "step": 1875
860
+ },
861
+ {
862
+ "entropy": 0.8110369992256165,
863
+ "epoch": 0.6577808551151116,
864
+ "grad_norm": 2.3125,
865
+ "learning_rate": 1.5677452985782493e-05,
866
+ "loss": 0.7894587707519531,
867
+ "mean_token_accuracy": 0.7973484849929809,
868
+ "num_tokens": 11760213.0,
869
+ "step": 1900
870
+ },
871
+ {
872
+ "entropy": 0.8280953919887543,
873
+ "epoch": 0.6664358663666263,
874
+ "grad_norm": 2.203125,
875
+ "learning_rate": 1.5561527470894503e-05,
876
+ "loss": 0.7997794342041016,
877
+ "mean_token_accuracy": 0.7926359629631042,
878
+ "num_tokens": 11913380.0,
879
+ "step": 1925
880
+ },
881
+ {
882
+ "entropy": 0.8196328341960907,
883
+ "epoch": 0.6750908776181409,
884
+ "grad_norm": 2.171875,
885
+ "learning_rate": 1.544450958418775e-05,
886
+ "loss": 0.7956825256347656,
887
+ "mean_token_accuracy": 0.7946672177314759,
888
+ "num_tokens": 12065793.0,
889
+ "step": 1950
890
+ },
891
+ {
892
+ "entropy": 0.8513234782218934,
893
+ "epoch": 0.6837458888696555,
894
+ "grad_norm": 1.8671875,
895
+ "learning_rate": 1.5326422309823046e-05,
896
+ "loss": 0.8301435089111329,
897
+ "mean_token_accuracy": 0.7851076364517212,
898
+ "num_tokens": 12229815.0,
899
+ "step": 1975
900
+ },
901
+ {
902
+ "entropy": 0.8259239780902863,
903
+ "epoch": 0.6924009001211702,
904
+ "grad_norm": 2.078125,
905
+ "learning_rate": 1.5207288842005813e-05,
906
+ "loss": 0.8005783081054687,
907
+ "mean_token_accuracy": 0.79151895403862,
908
+ "num_tokens": 12391391.0,
909
+ "step": 2000
910
+ },
911
+ {
912
+ "epoch": 0.6924009001211702,
913
+ "eval_entropy": 0.828711828149733,
914
+ "eval_loss": 0.8071080446243286,
915
+ "eval_mean_token_accuracy": 0.7911858433582744,
916
+ "eval_num_tokens": 12391391.0,
917
+ "eval_runtime": 16.2568,
918
+ "eval_samples_per_second": 299.259,
919
+ "eval_steps_per_second": 18.761,
920
+ "step": 2000
921
+ },
922
+ {
923
+ "entropy": 0.8361215782165528,
924
+ "epoch": 0.7010559113726847,
925
+ "grad_norm": 2.109375,
926
+ "learning_rate": 1.508713258043041e-05,
927
+ "loss": 0.8149121856689453,
928
+ "mean_token_accuracy": 0.7890836715698242,
929
+ "num_tokens": 12548530.0,
930
+ "step": 2025
931
+ },
932
+ {
933
+ "entropy": 0.8404046738147736,
934
+ "epoch": 0.7097109226241994,
935
+ "grad_norm": 2.0,
936
+ "learning_rate": 1.4965977125684018e-05,
937
+ "loss": 0.814455337524414,
938
+ "mean_token_accuracy": 0.7903144693374634,
939
+ "num_tokens": 12703495.0,
940
+ "step": 2050
941
+ },
942
+ {
943
+ "entropy": 0.8286141097545624,
944
+ "epoch": 0.718365933875714,
945
+ "grad_norm": 2.296875,
946
+ "learning_rate": 1.484384627461115e-05,
947
+ "loss": 0.8082135009765625,
948
+ "mean_token_accuracy": 0.7908019495010375,
949
+ "num_tokens": 12856585.0,
950
+ "step": 2075
951
+ },
952
+ {
953
+ "entropy": 0.8540136122703552,
954
+ "epoch": 0.7270209451272287,
955
+ "grad_norm": 2.09375,
956
+ "learning_rate": 1.4720764015639555e-05,
957
+ "loss": 0.8297102355957031,
958
+ "mean_token_accuracy": 0.7875827658176422,
959
+ "num_tokens": 13011641.0,
960
+ "step": 2100
961
+ },
962
+ {
963
+ "entropy": 0.8180252170562744,
964
+ "epoch": 0.7356759563787433,
965
+ "grad_norm": 2.171875,
966
+ "learning_rate": 1.4596754524068515e-05,
967
+ "loss": 0.791978988647461,
968
+ "mean_token_accuracy": 0.7945426023006439,
969
+ "num_tokens": 13167258.0,
970
+ "step": 2125
971
+ },
972
+ {
973
+ "entropy": 0.8130170261859894,
974
+ "epoch": 0.744330967630258,
975
+ "grad_norm": 2.1875,
976
+ "learning_rate": 1.4471842157320443e-05,
977
+ "loss": 0.7898551940917968,
978
+ "mean_token_accuracy": 0.7971277523040772,
979
+ "num_tokens": 13315266.0,
980
+ "step": 2150
981
+ },
982
+ {
983
+ "entropy": 0.8508910453319549,
984
+ "epoch": 0.7529859788817725,
985
+ "grad_norm": 2.140625,
986
+ "learning_rate": 1.4346051450156689e-05,
987
+ "loss": 0.8204324340820313,
988
+ "mean_token_accuracy": 0.7856499540805817,
989
+ "num_tokens": 13473686.0,
990
+ "step": 2175
991
+ },
992
+ {
993
+ "entropy": 0.8430261933803558,
994
+ "epoch": 0.7616409901332871,
995
+ "grad_norm": 2.125,
996
+ "learning_rate": 1.4219407109858532e-05,
997
+ "loss": 0.8170342254638672,
998
+ "mean_token_accuracy": 0.7883759355545044,
999
+ "num_tokens": 13627766.0,
1000
+ "step": 2200
1001
+ },
1002
+ {
1003
+ "epoch": 0.7616409901332871,
1004
+ "eval_entropy": 0.8279429495334625,
1005
+ "eval_loss": 0.8048078417778015,
1006
+ "eval_mean_token_accuracy": 0.7913855454960808,
1007
+ "eval_num_tokens": 13627766.0,
1008
+ "eval_runtime": 16.2774,
1009
+ "eval_samples_per_second": 298.88,
1010
+ "eval_steps_per_second": 18.738,
1011
+ "step": 2200
1012
+ },
1013
+ {
1014
+ "entropy": 0.8365470349788666,
1015
+ "epoch": 0.7702960013848018,
1016
+ "grad_norm": 2.3125,
1017
+ "learning_rate": 1.4091934011374281e-05,
1018
+ "loss": 0.8083656311035157,
1019
+ "mean_token_accuracy": 0.7891746735572815,
1020
+ "num_tokens": 13781809.0,
1021
+ "step": 2225
1022
+ },
1023
+ {
1024
+ "entropy": 0.8262781822681426,
1025
+ "epoch": 0.7789510126363164,
1026
+ "grad_norm": 2.265625,
1027
+ "learning_rate": 1.396365719243344e-05,
1028
+ "loss": 0.8065071868896484,
1029
+ "mean_token_accuracy": 0.7918438529968261,
1030
+ "num_tokens": 13937124.0,
1031
+ "step": 2250
1032
+ },
1033
+ {
1034
+ "entropy": 0.8354354929924012,
1035
+ "epoch": 0.7876060238878311,
1036
+ "grad_norm": 2.03125,
1037
+ "learning_rate": 1.3834601848628897e-05,
1038
+ "loss": 0.8083207702636719,
1039
+ "mean_token_accuracy": 0.7916932463645935,
1040
+ "num_tokens": 14088345.0,
1041
+ "step": 2275
1042
+ },
1043
+ {
1044
+ "entropy": 0.8217170929908753,
1045
+ "epoch": 0.7962610351393457,
1046
+ "grad_norm": 2.203125,
1047
+ "learning_rate": 1.3704793328468122e-05,
1048
+ "loss": 0.7991712951660156,
1049
+ "mean_token_accuracy": 0.7924394917488098,
1050
+ "num_tokens": 14241089.0,
1051
+ "step": 2300
1052
+ },
1053
+ {
1054
+ "entropy": 0.8267798566818237,
1055
+ "epoch": 0.8049160463908603,
1056
+ "grad_norm": 2.09375,
1057
+ "learning_rate": 1.3574257128394315e-05,
1058
+ "loss": 0.8018778228759765,
1059
+ "mean_token_accuracy": 0.7922985780239106,
1060
+ "num_tokens": 14392337.0,
1061
+ "step": 2325
1062
+ },
1063
+ {
1064
+ "entropy": 0.8261617541313171,
1065
+ "epoch": 0.8135710576423749,
1066
+ "grad_norm": 2.171875,
1067
+ "learning_rate": 1.3443018887778494e-05,
1068
+ "loss": 0.8028769683837891,
1069
+ "mean_token_accuracy": 0.7926994669437408,
1070
+ "num_tokens": 14544382.0,
1071
+ "step": 2350
1072
+ },
1073
+ {
1074
+ "entropy": 0.8380094921588898,
1075
+ "epoch": 0.8222260688938896,
1076
+ "grad_norm": 2.3125,
1077
+ "learning_rate": 1.3311104383883537e-05,
1078
+ "loss": 0.8226547241210938,
1079
+ "mean_token_accuracy": 0.7886342179775238,
1080
+ "num_tokens": 14703460.0,
1081
+ "step": 2375
1082
+ },
1083
+ {
1084
+ "entropy": 0.8281042277812958,
1085
+ "epoch": 0.8308810801454042,
1086
+ "grad_norm": 2.296875,
1087
+ "learning_rate": 1.3178539526801106e-05,
1088
+ "loss": 0.8025864410400391,
1089
+ "mean_token_accuracy": 0.7910430765151978,
1090
+ "num_tokens": 14860294.0,
1091
+ "step": 2400
1092
+ },
1093
+ {
1094
+ "epoch": 0.8308810801454042,
1095
+ "eval_entropy": 0.8327872855741469,
1096
+ "eval_loss": 0.8026647567749023,
1097
+ "eval_mean_token_accuracy": 0.7919463300314106,
1098
+ "eval_num_tokens": 14860294.0,
1099
+ "eval_runtime": 16.2486,
1100
+ "eval_samples_per_second": 299.41,
1101
+ "eval_steps_per_second": 18.771,
1102
+ "step": 2400
1103
+ },
1104
+ {
1105
+ "entropy": 0.811096111536026,
1106
+ "epoch": 0.8395360913969189,
1107
+ "grad_norm": 2.4375,
1108
+ "learning_rate": 1.3045350354362495e-05,
1109
+ "loss": 0.7822922515869141,
1110
+ "mean_token_accuracy": 0.7949872577190399,
1111
+ "num_tokens": 15015328.0,
1112
+ "step": 2425
1113
+ },
1114
+ {
1115
+ "entropy": 0.8189793074131012,
1116
+ "epoch": 0.8481911026484334,
1117
+ "grad_norm": 2.3125,
1118
+ "learning_rate": 1.2911563027024392e-05,
1119
+ "loss": 0.8003229522705078,
1120
+ "mean_token_accuracy": 0.7943521749973297,
1121
+ "num_tokens": 15170350.0,
1122
+ "step": 2450
1123
+ },
1124
+ {
1125
+ "entropy": 0.8213041162490845,
1126
+ "epoch": 0.856846113899948,
1127
+ "grad_norm": 2.21875,
1128
+ "learning_rate": 1.2777203822730557e-05,
1129
+ "loss": 0.7970082092285157,
1130
+ "mean_token_accuracy": 0.7927951776981353,
1131
+ "num_tokens": 15324153.0,
1132
+ "step": 2475
1133
+ },
1134
+ {
1135
+ "entropy": 0.824720960855484,
1136
+ "epoch": 0.8655011251514627,
1137
+ "grad_norm": 2.125,
1138
+ "learning_rate": 1.2642299131750417e-05,
1139
+ "loss": 0.7908332824707032,
1140
+ "mean_token_accuracy": 0.7957222282886505,
1141
+ "num_tokens": 15473303.0,
1142
+ "step": 2500
1143
+ },
1144
+ {
1145
+ "entropy": 0.838714919090271,
1146
+ "epoch": 0.8741561364029773,
1147
+ "grad_norm": 2.09375,
1148
+ "learning_rate": 1.250687545149559e-05,
1149
+ "loss": 0.8137520599365234,
1150
+ "mean_token_accuracy": 0.7890112113952636,
1151
+ "num_tokens": 15629017.0,
1152
+ "step": 2525
1153
+ },
1154
+ {
1155
+ "entropy": 0.8311066961288452,
1156
+ "epoch": 0.882811147654492,
1157
+ "grad_norm": 2.1875,
1158
+ "learning_rate": 1.2370959381315393e-05,
1159
+ "loss": 0.8034115600585937,
1160
+ "mean_token_accuracy": 0.7926496362686157,
1161
+ "num_tokens": 15777876.0,
1162
+ "step": 2550
1163
+ },
1164
+ {
1165
+ "entropy": 0.8097344720363617,
1166
+ "epoch": 0.8914661589060066,
1167
+ "grad_norm": 2.203125,
1168
+ "learning_rate": 1.2234577617272282e-05,
1169
+ "loss": 0.786445083618164,
1170
+ "mean_token_accuracy": 0.7966820681095124,
1171
+ "num_tokens": 15929184.0,
1172
+ "step": 2575
1173
+ },
1174
+ {
1175
+ "entropy": 0.8254410099983215,
1176
+ "epoch": 0.9001211701575212,
1177
+ "grad_norm": 2.296875,
1178
+ "learning_rate": 1.209775694689837e-05,
1179
+ "loss": 0.8023792266845703,
1180
+ "mean_token_accuracy": 0.7923547208309174,
1181
+ "num_tokens": 16087411.0,
1182
+ "step": 2600
1183
+ },
1184
+ {
1185
+ "epoch": 0.9001211701575212,
1186
+ "eval_entropy": 0.8213896471945966,
1187
+ "eval_loss": 0.8008216619491577,
1188
+ "eval_mean_token_accuracy": 0.7924871415388389,
1189
+ "eval_num_tokens": 16087411.0,
1190
+ "eval_runtime": 16.2722,
1191
+ "eval_samples_per_second": 298.976,
1192
+ "eval_steps_per_second": 18.744,
1193
+ "step": 2600
1194
+ },
1195
+ {
1196
+ "entropy": 0.8255444514751434,
1197
+ "epoch": 0.9087761814090358,
1198
+ "grad_norm": 1.9140625,
1199
+ "learning_rate": 1.1960524243933875e-05,
1200
+ "loss": 0.8017765808105469,
1201
+ "mean_token_accuracy": 0.7921687912940979,
1202
+ "num_tokens": 16244042.0,
1203
+ "step": 2625
1204
+ },
1205
+ {
1206
+ "entropy": 0.8252915942668915,
1207
+ "epoch": 0.9174311926605505,
1208
+ "grad_norm": 2.21875,
1209
+ "learning_rate": 1.182290646304874e-05,
1210
+ "loss": 0.8029595947265625,
1211
+ "mean_token_accuracy": 0.7924495422840119,
1212
+ "num_tokens": 16399768.0,
1213
+ "step": 2650
1214
+ },
1215
+ {
1216
+ "entropy": 0.8625216126441956,
1217
+ "epoch": 0.9260862039120651,
1218
+ "grad_norm": 2.171875,
1219
+ "learning_rate": 1.1684930634548287e-05,
1220
+ "loss": 0.838686752319336,
1221
+ "mean_token_accuracy": 0.7851159811019898,
1222
+ "num_tokens": 16557371.0,
1223
+ "step": 2675
1224
+ },
1225
+ {
1226
+ "entropy": 0.8215836095809936,
1227
+ "epoch": 0.9347412151635797,
1228
+ "grad_norm": 2.265625,
1229
+ "learning_rate": 1.1546623859064053e-05,
1230
+ "loss": 0.7977464294433594,
1231
+ "mean_token_accuracy": 0.7929444241523743,
1232
+ "num_tokens": 16713045.0,
1233
+ "step": 2700
1234
+ },
1235
+ {
1236
+ "entropy": 0.8286124646663666,
1237
+ "epoch": 0.9433962264150944,
1238
+ "grad_norm": 2.078125,
1239
+ "learning_rate": 1.1408013302230765e-05,
1240
+ "loss": 0.802153091430664,
1241
+ "mean_token_accuracy": 0.7924544358253479,
1242
+ "num_tokens": 16867145.0,
1243
+ "step": 2725
1244
+ },
1245
+ {
1246
+ "entropy": 0.8237406086921691,
1247
+ "epoch": 0.9520512376666089,
1248
+ "grad_norm": 2.15625,
1249
+ "learning_rate": 1.126912618935063e-05,
1250
+ "loss": 0.8021310424804687,
1251
+ "mean_token_accuracy": 0.7944400310516357,
1252
+ "num_tokens": 17014455.0,
1253
+ "step": 2750
1254
+ },
1255
+ {
1256
+ "entropy": 0.826482676267624,
1257
+ "epoch": 0.9607062489181236,
1258
+ "grad_norm": 2.140625,
1259
+ "learning_rate": 1.1129989800045817e-05,
1260
+ "loss": 0.8052169036865234,
1261
+ "mean_token_accuracy": 0.7911915636062622,
1262
+ "num_tokens": 17175484.0,
1263
+ "step": 2775
1264
+ },
1265
+ {
1266
+ "entropy": 0.8201354122161866,
1267
+ "epoch": 0.9693612601696382,
1268
+ "grad_norm": 2.421875,
1269
+ "learning_rate": 1.099063146290032e-05,
1270
+ "loss": 0.7985061645507813,
1271
+ "mean_token_accuracy": 0.7938509380817413,
1272
+ "num_tokens": 17334068.0,
1273
+ "step": 2800
1274
+ },
1275
+ {
1276
+ "epoch": 0.9693612601696382,
1277
+ "eval_entropy": 0.8281321289109402,
1278
+ "eval_loss": 0.7994962334632874,
1279
+ "eval_mean_token_accuracy": 0.792740264681519,
1280
+ "eval_num_tokens": 17334068.0,
1281
+ "eval_runtime": 16.2806,
1282
+ "eval_samples_per_second": 298.822,
1283
+ "eval_steps_per_second": 18.734,
1284
+ "step": 2800
1285
+ },
1286
+ {
1287
+ "entropy": 0.8046394038200378,
1288
+ "epoch": 0.9780162714211529,
1289
+ "grad_norm": 2.46875,
1290
+ "learning_rate": 1.08510785500922e-05,
1291
+ "loss": 0.7775811004638672,
1292
+ "mean_token_accuracy": 0.7966744971275329,
1293
+ "num_tokens": 17483883.0,
1294
+ "step": 2825
1295
+ },
1296
+ {
1297
+ "entropy": 0.8306651365756988,
1298
+ "epoch": 0.9866712826726675,
1299
+ "grad_norm": 2.359375,
1300
+ "learning_rate": 1.0711358472017244e-05,
1301
+ "loss": 0.7954608154296875,
1302
+ "mean_token_accuracy": 0.7919278919696808,
1303
+ "num_tokens": 17639972.0,
1304
+ "step": 2850
1305
+ },
1306
+ {
1307
+ "entropy": 0.8295222818851471,
1308
+ "epoch": 0.9953262939241821,
1309
+ "grad_norm": 2.125,
1310
+ "learning_rate": 1.0571498671905146e-05,
1311
+ "loss": 0.8072916412353516,
1312
+ "mean_token_accuracy": 0.7907999694347382,
1313
+ "num_tokens": 17794430.0,
1314
+ "step": 2875
1315
+ },
1316
+ {
1317
+ "entropy": 0.8070371369926297,
1318
+ "epoch": 1.0038082049506665,
1319
+ "grad_norm": 2.09375,
1320
+ "learning_rate": 1.0431526620429207e-05,
1321
+ "loss": 0.760282211303711,
1322
+ "mean_token_accuracy": 0.7995899867038337,
1323
+ "num_tokens": 17946140.0,
1324
+ "step": 2900
1325
+ },
1326
+ {
1327
+ "entropy": 0.7756562912464142,
1328
+ "epoch": 1.0124632162021812,
1329
+ "grad_norm": 2.140625,
1330
+ "learning_rate": 1.0291469810310669e-05,
1331
+ "loss": 0.741202621459961,
1332
+ "mean_token_accuracy": 0.803013744354248,
1333
+ "num_tokens": 18100839.0,
1334
+ "step": 2925
1335
+ },
1336
+ {
1337
+ "entropy": 0.7452013885974884,
1338
+ "epoch": 1.0211182274536956,
1339
+ "grad_norm": 2.15625,
1340
+ "learning_rate": 1.0151355750918712e-05,
1341
+ "loss": 0.7064004516601563,
1342
+ "mean_token_accuracy": 0.8126378917694091,
1343
+ "num_tokens": 18256691.0,
1344
+ "step": 2950
1345
+ },
1346
+ {
1347
+ "entropy": 0.7718643927574158,
1348
+ "epoch": 1.0297732387052103,
1349
+ "grad_norm": 1.8671875,
1350
+ "learning_rate": 1.001121196286718e-05,
1351
+ "loss": 0.7311404418945312,
1352
+ "mean_token_accuracy": 0.8063716220855713,
1353
+ "num_tokens": 18418483.0,
1354
+ "step": 2975
1355
+ },
1356
+ {
1357
+ "entropy": 0.765803085565567,
1358
+ "epoch": 1.038428249956725,
1359
+ "grad_norm": 2.0,
1360
+ "learning_rate": 9.871065972609089e-06,
1361
+ "loss": 0.7332243347167968,
1362
+ "mean_token_accuracy": 0.8049838674068451,
1363
+ "num_tokens": 18576253.0,
1364
+ "step": 3000
1365
+ },
1366
+ {
1367
+ "epoch": 1.038428249956725,
1368
+ "eval_entropy": 0.7786700289757525,
1369
+ "eval_loss": 0.8012996912002563,
1370
+ "eval_mean_token_accuracy": 0.792937566022404,
1371
+ "eval_num_tokens": 18576253.0,
1372
+ "eval_runtime": 16.277,
1373
+ "eval_samples_per_second": 298.888,
1374
+ "eval_steps_per_second": 18.738,
1375
+ "step": 3000
1376
+ },
1377
+ {
1378
+ "entropy": 0.7836507785320282,
1379
+ "epoch": 1.0470832612082395,
1380
+ "grad_norm": 2.0625,
1381
+ "learning_rate": 9.730945307030009e-06,
1382
+ "loss": 0.7418560028076172,
1383
+ "mean_token_accuracy": 0.8028053915500641,
1384
+ "num_tokens": 18729564.0,
1385
+ "step": 3025
1386
+ },
1387
+ {
1388
+ "entropy": 0.7653721117973328,
1389
+ "epoch": 1.0557382724597542,
1390
+ "grad_norm": 2.03125,
1391
+ "learning_rate": 9.590877488041336e-06,
1392
+ "loss": 0.7263675689697265,
1393
+ "mean_token_accuracy": 0.807190809249878,
1394
+ "num_tokens": 18891538.0,
1395
+ "step": 3050
1396
+ },
1397
+ {
1398
+ "entropy": 0.7604654181003571,
1399
+ "epoch": 1.0643932837112688,
1400
+ "grad_norm": 2.296875,
1401
+ "learning_rate": 9.45089002717457e-06,
1402
+ "loss": 0.72638916015625,
1403
+ "mean_token_accuracy": 0.8084564554691315,
1404
+ "num_tokens": 19050606.0,
1405
+ "step": 3075
1406
+ },
1407
+ {
1408
+ "entropy": 0.7386153995990753,
1409
+ "epoch": 1.0730482949627835,
1410
+ "grad_norm": 1.9765625,
1411
+ "learning_rate": 9.311010420177597e-06,
1412
+ "loss": 0.706187973022461,
1413
+ "mean_token_accuracy": 0.8120154666900635,
1414
+ "num_tokens": 19208707.0,
1415
+ "step": 3100
1416
+ },
1417
+ {
1418
+ "entropy": 0.7525534296035766,
1419
+ "epoch": 1.081703306214298,
1420
+ "grad_norm": 2.171875,
1421
+ "learning_rate": 9.171266141614112e-06,
1422
+ "loss": 0.7135391235351562,
1423
+ "mean_token_accuracy": 0.8123032712936401,
1424
+ "num_tokens": 19362737.0,
1425
+ "step": 3125
1426
+ },
1427
+ {
1428
+ "entropy": 0.7687537455558777,
1429
+ "epoch": 1.0903583174658127,
1430
+ "grad_norm": 2.109375,
1431
+ "learning_rate": 9.031684639467159e-06,
1432
+ "loss": 0.734495849609375,
1433
+ "mean_token_accuracy": 0.8068291318416595,
1434
+ "num_tokens": 19521058.0,
1435
+ "step": 3150
1436
+ },
1437
+ {
1438
+ "entropy": 0.7806685280799865,
1439
+ "epoch": 1.0990133287173274,
1440
+ "grad_norm": 2.046875,
1441
+ "learning_rate": 8.892293329747923e-06,
1442
+ "loss": 0.7469673156738281,
1443
+ "mean_token_accuracy": 0.8041970551013946,
1444
+ "num_tokens": 19675603.0,
1445
+ "step": 3175
1446
+ },
1447
+ {
1448
+ "entropy": 0.7709089267253876,
1449
+ "epoch": 1.1076683399688418,
1450
+ "grad_norm": 2.15625,
1451
+ "learning_rate": 8.753119591110802e-06,
1452
+ "loss": 0.7363986206054688,
1453
+ "mean_token_accuracy": 0.8061197555065155,
1454
+ "num_tokens": 19828898.0,
1455
+ "step": 3200
1456
+ },
1457
+ {
1458
+ "epoch": 1.1076683399688418,
1459
+ "eval_entropy": 0.7721671974072691,
1460
+ "eval_loss": 0.8010231852531433,
1461
+ "eval_mean_token_accuracy": 0.7929807965872717,
1462
+ "eval_num_tokens": 19828898.0,
1463
+ "eval_runtime": 16.2529,
1464
+ "eval_samples_per_second": 299.331,
1465
+ "eval_steps_per_second": 18.766,
1466
+ "step": 3200
1467
+ },
1468
+ {
1469
+ "entropy": 0.7211595928668976,
1470
+ "epoch": 1.1163233512203565,
1471
+ "grad_norm": 2.046875,
1472
+ "learning_rate": 8.614190759475789e-06,
1473
+ "loss": 0.6826175689697266,
1474
+ "mean_token_accuracy": 0.8186909162998199,
1475
+ "num_tokens": 19982072.0,
1476
+ "step": 3225
1477
+ },
1478
+ {
1479
+ "entropy": 0.7708668720722198,
1480
+ "epoch": 1.1249783624718712,
1481
+ "grad_norm": 2.15625,
1482
+ "learning_rate": 8.475534122659275e-06,
1483
+ "loss": 0.7388634490966797,
1484
+ "mean_token_accuracy": 0.8042692136764527,
1485
+ "num_tokens": 20137744.0,
1486
+ "step": 3250
1487
+ },
1488
+ {
1489
+ "entropy": 0.7538685095310211,
1490
+ "epoch": 1.133633373723386,
1491
+ "grad_norm": 2.171875,
1492
+ "learning_rate": 8.337176915014293e-06,
1493
+ "loss": 0.7132722473144532,
1494
+ "mean_token_accuracy": 0.8105318355560303,
1495
+ "num_tokens": 20290524.0,
1496
+ "step": 3275
1497
+ },
1498
+ {
1499
+ "entropy": 0.7487189078330994,
1500
+ "epoch": 1.1422883849749004,
1501
+ "grad_norm": 2.046875,
1502
+ "learning_rate": 8.199146312081265e-06,
1503
+ "loss": 0.7089384460449218,
1504
+ "mean_token_accuracy": 0.8108018398284912,
1505
+ "num_tokens": 20443485.0,
1506
+ "step": 3300
1507
+ },
1508
+ {
1509
+ "entropy": 0.7543841946125031,
1510
+ "epoch": 1.150943396226415,
1511
+ "grad_norm": 2.0625,
1512
+ "learning_rate": 8.061469425250273e-06,
1513
+ "loss": 0.7227223205566407,
1514
+ "mean_token_accuracy": 0.8090525925159454,
1515
+ "num_tokens": 20595797.0,
1516
+ "step": 3325
1517
+ },
1518
+ {
1519
+ "entropy": 0.7593919610977173,
1520
+ "epoch": 1.1595984074779297,
1521
+ "grad_norm": 2.015625,
1522
+ "learning_rate": 7.924173296435987e-06,
1523
+ "loss": 0.7215187072753906,
1524
+ "mean_token_accuracy": 0.8093890881538391,
1525
+ "num_tokens": 20749554.0,
1526
+ "step": 3350
1527
+ },
1528
+ {
1529
+ "entropy": 0.7829481947422028,
1530
+ "epoch": 1.1682534187294444,
1531
+ "grad_norm": 2.171875,
1532
+ "learning_rate": 7.78728489276618e-06,
1533
+ "loss": 0.7491289520263672,
1534
+ "mean_token_accuracy": 0.8015105819702149,
1535
+ "num_tokens": 20909534.0,
1536
+ "step": 3375
1537
+ },
1538
+ {
1539
+ "entropy": 0.7392001152038574,
1540
+ "epoch": 1.176908429980959,
1541
+ "grad_norm": 2.1875,
1542
+ "learning_rate": 7.650831101284977e-06,
1543
+ "loss": 0.7020195007324219,
1544
+ "mean_token_accuracy": 0.8150424540042878,
1545
+ "num_tokens": 21059158.0,
1546
+ "step": 3400
1547
+ },
1548
+ {
1549
+ "epoch": 1.176908429980959,
1550
+ "eval_entropy": 0.7698653085309951,
1551
+ "eval_loss": 0.8008679747581482,
1552
+ "eval_mean_token_accuracy": 0.793300522155449,
1553
+ "eval_num_tokens": 21059158.0,
1554
+ "eval_runtime": 16.2799,
1555
+ "eval_samples_per_second": 298.835,
1556
+ "eval_steps_per_second": 18.735,
1557
+ "step": 3400
1558
+ },
1559
+ {
1560
+ "entropy": 0.768788868188858,
1561
+ "epoch": 1.1855634412324736,
1562
+ "grad_norm": 2.046875,
1563
+ "learning_rate": 7.514838723671799e-06,
1564
+ "loss": 0.7372692108154297,
1565
+ "mean_token_accuracy": 0.8063059449195862,
1566
+ "num_tokens": 21208688.0,
1567
+ "step": 3425
1568
+ },
1569
+ {
1570
+ "entropy": 0.7443365502357483,
1571
+ "epoch": 1.1942184524839883,
1572
+ "grad_norm": 1.875,
1573
+ "learning_rate": 7.379334470977093e-06,
1574
+ "loss": 0.710152587890625,
1575
+ "mean_token_accuracy": 0.811426283121109,
1576
+ "num_tokens": 21365731.0,
1577
+ "step": 3450
1578
+ },
1579
+ {
1580
+ "entropy": 0.7438737845420837,
1581
+ "epoch": 1.2028734637355027,
1582
+ "grad_norm": 1.96875,
1583
+ "learning_rate": 7.244344958375881e-06,
1584
+ "loss": 0.710167465209961,
1585
+ "mean_token_accuracy": 0.8120112872123718,
1586
+ "num_tokens": 21519688.0,
1587
+ "step": 3475
1588
+ },
1589
+ {
1590
+ "entropy": 0.7728407967090607,
1591
+ "epoch": 1.2115284749870174,
1592
+ "grad_norm": 2.046875,
1593
+ "learning_rate": 7.109896699940084e-06,
1594
+ "loss": 0.7461888885498047,
1595
+ "mean_token_accuracy": 0.8037091171741486,
1596
+ "num_tokens": 21676493.0,
1597
+ "step": 3500
1598
+ },
1599
+ {
1600
+ "entropy": 0.7640272963047028,
1601
+ "epoch": 1.2201834862385321,
1602
+ "grad_norm": 2.203125,
1603
+ "learning_rate": 6.976016103430765e-06,
1604
+ "loss": 0.7235647583007813,
1605
+ "mean_token_accuracy": 0.8083086097240448,
1606
+ "num_tokens": 21825664.0,
1607
+ "step": 3525
1608
+ },
1609
+ {
1610
+ "entropy": 0.7460694527626037,
1611
+ "epoch": 1.2288384974900468,
1612
+ "grad_norm": 2.03125,
1613
+ "learning_rate": 6.842729465111224e-06,
1614
+ "loss": 0.7100708770751953,
1615
+ "mean_token_accuracy": 0.8116568970680237,
1616
+ "num_tokens": 21976504.0,
1617
+ "step": 3550
1618
+ },
1619
+ {
1620
+ "entropy": 0.759437519311905,
1621
+ "epoch": 1.2374935087415613,
1622
+ "grad_norm": 2.609375,
1623
+ "learning_rate": 6.710062964581988e-06,
1624
+ "loss": 0.722847900390625,
1625
+ "mean_token_accuracy": 0.807286411523819,
1626
+ "num_tokens": 22133165.0,
1627
+ "step": 3575
1628
+ },
1629
+ {
1630
+ "entropy": 0.7708945345878601,
1631
+ "epoch": 1.246148519993076,
1632
+ "grad_norm": 2.015625,
1633
+ "learning_rate": 6.578042659638726e-06,
1634
+ "loss": 0.7426469421386719,
1635
+ "mean_token_accuracy": 0.8035933816432953,
1636
+ "num_tokens": 22290456.0,
1637
+ "step": 3600
1638
+ },
1639
+ {
1640
+ "epoch": 1.246148519993076,
1641
+ "eval_entropy": 0.7726592179204597,
1642
+ "eval_loss": 0.8001708984375,
1643
+ "eval_mean_token_accuracy": 0.7932597899046101,
1644
+ "eval_num_tokens": 22290456.0,
1645
+ "eval_runtime": 16.2656,
1646
+ "eval_samples_per_second": 299.097,
1647
+ "eval_steps_per_second": 18.751,
1648
+ "step": 3600
1649
+ },
1650
+ {
1651
+ "entropy": 0.7578576147556305,
1652
+ "epoch": 1.2548035312445907,
1653
+ "grad_norm": 2.078125,
1654
+ "learning_rate": 6.446694481154104e-06,
1655
+ "loss": 0.7225240325927734,
1656
+ "mean_token_accuracy": 0.8097064507007599,
1657
+ "num_tokens": 22446824.0,
1658
+ "step": 3625
1659
+ },
1660
+ {
1661
+ "entropy": 0.7762425947189331,
1662
+ "epoch": 1.2634585424961053,
1663
+ "grad_norm": 2.265625,
1664
+ "learning_rate": 6.3160442279845206e-06,
1665
+ "loss": 0.7412718200683593,
1666
+ "mean_token_accuracy": 0.805392199754715,
1667
+ "num_tokens": 22602588.0,
1668
+ "step": 3650
1669
+ },
1670
+ {
1671
+ "entropy": 0.7637201595306397,
1672
+ "epoch": 1.2721135537476198,
1673
+ "grad_norm": 2.25,
1674
+ "learning_rate": 6.186117561902836e-06,
1675
+ "loss": 0.7297930908203125,
1676
+ "mean_token_accuracy": 0.8078333747386932,
1677
+ "num_tokens": 22752707.0,
1678
+ "step": 3675
1679
+ },
1680
+ {
1681
+ "entropy": 0.7704725599288941,
1682
+ "epoch": 1.2807685649991345,
1683
+ "grad_norm": 2.125,
1684
+ "learning_rate": 6.056940002557969e-06,
1685
+ "loss": 0.7372798919677734,
1686
+ "mean_token_accuracy": 0.8056825184822083,
1687
+ "num_tokens": 22907803.0,
1688
+ "step": 3700
1689
+ },
1690
+ {
1691
+ "entropy": 0.7552802407741547,
1692
+ "epoch": 1.2894235762506492,
1693
+ "grad_norm": 2.15625,
1694
+ "learning_rate": 5.928536922462449e-06,
1695
+ "loss": 0.7166344451904297,
1696
+ "mean_token_accuracy": 0.8109099209308624,
1697
+ "num_tokens": 23059819.0,
1698
+ "step": 3725
1699
+ },
1700
+ {
1701
+ "entropy": 0.7541340529918671,
1702
+ "epoch": 1.2980785875021636,
1703
+ "grad_norm": 2.3125,
1704
+ "learning_rate": 5.80093354200886e-06,
1705
+ "loss": 0.7238423919677734,
1706
+ "mean_token_accuracy": 0.809191288948059,
1707
+ "num_tokens": 23217392.0,
1708
+ "step": 3750
1709
+ },
1710
+ {
1711
+ "entropy": 0.7676697385311126,
1712
+ "epoch": 1.3067335987536783,
1713
+ "grad_norm": 2.21875,
1714
+ "learning_rate": 5.674154924516129e-06,
1715
+ "loss": 0.7366065979003906,
1716
+ "mean_token_accuracy": 0.8054041469097137,
1717
+ "num_tokens": 23371073.0,
1718
+ "step": 3775
1719
+ },
1720
+ {
1721
+ "entropy": 0.7785192513465882,
1722
+ "epoch": 1.315388610005193,
1723
+ "grad_norm": 1.9140625,
1724
+ "learning_rate": 5.548225971306731e-06,
1725
+ "loss": 0.7439189147949219,
1726
+ "mean_token_accuracy": 0.8034746491909027,
1727
+ "num_tokens": 23524592.0,
1728
+ "step": 3800
1729
+ },
1730
+ {
1731
+ "epoch": 1.315388610005193,
1732
+ "eval_entropy": 0.7713183484116538,
1733
+ "eval_loss": 0.7998276352882385,
1734
+ "eval_mean_token_accuracy": 0.7933589818047695,
1735
+ "eval_num_tokens": 23524592.0,
1736
+ "eval_runtime": 16.2844,
1737
+ "eval_samples_per_second": 298.753,
1738
+ "eval_steps_per_second": 18.73,
1739
+ "step": 3800
1740
+ },
1741
+ {
1742
+ "entropy": 0.7570435631275177,
1743
+ "epoch": 1.3240436212567077,
1744
+ "grad_norm": 2.078125,
1745
+ "learning_rate": 5.423171416815651e-06,
1746
+ "loss": 0.7266172790527343,
1747
+ "mean_token_accuracy": 0.8094574224948883,
1748
+ "num_tokens": 23682141.0,
1749
+ "step": 3825
1750
+ },
1751
+ {
1752
+ "entropy": 0.76507066488266,
1753
+ "epoch": 1.3326986325082222,
1754
+ "grad_norm": 2.15625,
1755
+ "learning_rate": 5.299015823732163e-06,
1756
+ "loss": 0.7294864654541016,
1757
+ "mean_token_accuracy": 0.8064786922931672,
1758
+ "num_tokens": 23839050.0,
1759
+ "step": 3850
1760
+ },
1761
+ {
1762
+ "entropy": 0.7562107217311859,
1763
+ "epoch": 1.3413536437597369,
1764
+ "grad_norm": 2.0625,
1765
+ "learning_rate": 5.175783578175309e-06,
1766
+ "loss": 0.719031982421875,
1767
+ "mean_token_accuracy": 0.8091187167167664,
1768
+ "num_tokens": 23992205.0,
1769
+ "step": 3875
1770
+ },
1771
+ {
1772
+ "entropy": 0.7576862835884094,
1773
+ "epoch": 1.3500086550112516,
1774
+ "grad_norm": 2.109375,
1775
+ "learning_rate": 5.053498884904122e-06,
1776
+ "loss": 0.7270169830322266,
1777
+ "mean_token_accuracy": 0.8094841420650483,
1778
+ "num_tokens": 24139859.0,
1779
+ "step": 3900
1780
+ },
1781
+ {
1782
+ "entropy": 0.7541490840911865,
1783
+ "epoch": 1.3586636662627662,
1784
+ "grad_norm": 2.109375,
1785
+ "learning_rate": 4.932185762563381e-06,
1786
+ "loss": 0.7113311004638672,
1787
+ "mean_token_accuracy": 0.8112999868392944,
1788
+ "num_tokens": 24289196.0,
1789
+ "step": 3925
1790
+ },
1791
+ {
1792
+ "entropy": 0.7688007843494415,
1793
+ "epoch": 1.3673186775142807,
1794
+ "grad_norm": 2.1875,
1795
+ "learning_rate": 4.811868038965999e-06,
1796
+ "loss": 0.745551986694336,
1797
+ "mean_token_accuracy": 0.8050249302387238,
1798
+ "num_tokens": 24447938.0,
1799
+ "step": 3950
1800
+ },
1801
+ {
1802
+ "entropy": 0.7551292729377747,
1803
+ "epoch": 1.3759736887657954,
1804
+ "grad_norm": 2.1875,
1805
+ "learning_rate": 4.692569346412864e-06,
1806
+ "loss": 0.7206728363037109,
1807
+ "mean_token_accuracy": 0.8098892831802368,
1808
+ "num_tokens": 24598742.0,
1809
+ "step": 3975
1810
+ },
1811
+ {
1812
+ "entropy": 0.7587357807159424,
1813
+ "epoch": 1.38462870001731,
1814
+ "grad_norm": 2.125,
1815
+ "learning_rate": 4.574313117051059e-06,
1816
+ "loss": 0.7276847076416015,
1817
+ "mean_token_accuracy": 0.8077455127239227,
1818
+ "num_tokens": 24750406.0,
1819
+ "step": 4000
1820
+ },
1821
+ {
1822
+ "epoch": 1.38462870001731,
1823
+ "eval_entropy": 0.7717335020909544,
1824
+ "eval_loss": 0.7996781468391418,
1825
+ "eval_mean_token_accuracy": 0.7934077425081222,
1826
+ "eval_num_tokens": 24750406.0,
1827
+ "eval_runtime": 16.2676,
1828
+ "eval_samples_per_second": 299.061,
1829
+ "eval_steps_per_second": 18.749,
1830
+ "step": 4000
1831
+ },
1832
+ {
1833
+ "entropy": 0.7805033266544342,
1834
+ "epoch": 1.3932837112688246,
1835
+ "grad_norm": 2.078125,
1836
+ "learning_rate": 4.457122578271457e-06,
1837
+ "loss": 0.7509593200683594,
1838
+ "mean_token_accuracy": 0.8028152561187745,
1839
+ "num_tokens": 24906659.0,
1840
+ "step": 4025
1841
+ },
1842
+ {
1843
+ "entropy": 0.7705829620361329,
1844
+ "epoch": 1.4019387225203392,
1845
+ "grad_norm": 2.125,
1846
+ "learning_rate": 4.341020748146449e-06,
1847
+ "loss": 0.7392497253417969,
1848
+ "mean_token_accuracy": 0.8049503290653228,
1849
+ "num_tokens": 25066654.0,
1850
+ "step": 4050
1851
+ },
1852
+ {
1853
+ "entropy": 0.7581047600507737,
1854
+ "epoch": 1.410593733771854,
1855
+ "grad_norm": 2.0,
1856
+ "learning_rate": 4.226030430908869e-06,
1857
+ "loss": 0.7285484313964844,
1858
+ "mean_token_accuracy": 0.8085175144672394,
1859
+ "num_tokens": 25220415.0,
1860
+ "step": 4075
1861
+ },
1862
+ {
1863
+ "entropy": 0.7656786620616913,
1864
+ "epoch": 1.4192487450233684,
1865
+ "grad_norm": 2.21875,
1866
+ "learning_rate": 4.112174212472864e-06,
1867
+ "loss": 0.7323892211914063,
1868
+ "mean_token_accuracy": 0.8051603281497955,
1869
+ "num_tokens": 25374458.0,
1870
+ "step": 4100
1871
+ },
1872
+ {
1873
+ "entropy": 0.7522096228599549,
1874
+ "epoch": 1.427903756274883,
1875
+ "grad_norm": 2.171875,
1876
+ "learning_rate": 3.99947445599767e-06,
1877
+ "loss": 0.7212752532958985,
1878
+ "mean_token_accuracy": 0.8096666383743286,
1879
+ "num_tokens": 25525969.0,
1880
+ "step": 4125
1881
+ },
1882
+ {
1883
+ "entropy": 0.7608710610866547,
1884
+ "epoch": 1.4365587675263978,
1885
+ "grad_norm": 2.40625,
1886
+ "learning_rate": 3.887953297495121e-06,
1887
+ "loss": 0.7316089630126953,
1888
+ "mean_token_accuracy": 0.8074719154834747,
1889
+ "num_tokens": 25681452.0,
1890
+ "step": 4150
1891
+ },
1892
+ {
1893
+ "entropy": 0.7639850771427155,
1894
+ "epoch": 1.4452137787779125,
1895
+ "grad_norm": 2.140625,
1896
+ "learning_rate": 3.77763264148181e-06,
1897
+ "loss": 0.7320518493652344,
1898
+ "mean_token_accuracy": 0.809201009273529,
1899
+ "num_tokens": 25838398.0,
1900
+ "step": 4175
1901
+ },
1902
+ {
1903
+ "entropy": 0.7513358175754548,
1904
+ "epoch": 1.4538687900294271,
1905
+ "grad_norm": 2.21875,
1906
+ "learning_rate": 3.6685341566766785e-06,
1907
+ "loss": 0.7206896209716797,
1908
+ "mean_token_accuracy": 0.8095373523235321,
1909
+ "num_tokens": 25992640.0,
1910
+ "step": 4200
1911
+ },
1912
+ {
1913
+ "epoch": 1.4538687900294271,
1914
+ "eval_entropy": 0.7707103953986871,
1915
+ "eval_loss": 0.7995045185089111,
1916
+ "eval_mean_token_accuracy": 0.7933821330305005,
1917
+ "eval_num_tokens": 25992640.0,
1918
+ "eval_runtime": 16.2677,
1919
+ "eval_samples_per_second": 299.058,
1920
+ "eval_steps_per_second": 18.749,
1921
+ "step": 4200
1922
+ },
1923
+ {
1924
+ "entropy": 0.7595930647850037,
1925
+ "epoch": 1.4625238012809416,
1926
+ "grad_norm": 2.171875,
1927
+ "learning_rate": 3.5606792717449455e-06,
1928
+ "loss": 0.7264796447753906,
1929
+ "mean_token_accuracy": 0.8094517874717713,
1930
+ "num_tokens": 26141304.0,
1931
+ "step": 4225
1932
+ },
1933
+ {
1934
+ "entropy": 0.7630014491081237,
1935
+ "epoch": 1.4711788125324563,
1936
+ "grad_norm": 2.15625,
1937
+ "learning_rate": 3.4540891710891768e-06,
1938
+ "loss": 0.7287496185302734,
1939
+ "mean_token_accuracy": 0.808629115819931,
1940
+ "num_tokens": 26295315.0,
1941
+ "step": 4250
1942
+ },
1943
+ {
1944
+ "entropy": 0.7566706609725952,
1945
+ "epoch": 1.479833823783971,
1946
+ "grad_norm": 2.34375,
1947
+ "learning_rate": 3.3487847906883232e-06,
1948
+ "loss": 0.7267755126953125,
1949
+ "mean_token_accuracy": 0.8091002118587494,
1950
+ "num_tokens": 26450821.0,
1951
+ "step": 4275
1952
+ },
1953
+ {
1954
+ "entropy": 0.7702415359020233,
1955
+ "epoch": 1.4884888350354855,
1956
+ "grad_norm": 2.09375,
1957
+ "learning_rate": 3.2447868139855954e-06,
1958
+ "loss": 0.74352294921875,
1959
+ "mean_token_accuracy": 0.8064157390594482,
1960
+ "num_tokens": 26610364.0,
1961
+ "step": 4300
1962
+ },
1963
+ {
1964
+ "entropy": 0.78652991771698,
1965
+ "epoch": 1.4971438462870001,
1966
+ "grad_norm": 2.359375,
1967
+ "learning_rate": 3.142115667825868e-06,
1968
+ "loss": 0.7553633117675781,
1969
+ "mean_token_accuracy": 0.8014207696914672,
1970
+ "num_tokens": 26766393.0,
1971
+ "step": 4325
1972
+ },
1973
+ {
1974
+ "entropy": 0.7758943355083465,
1975
+ "epoch": 1.5057988575385148,
1976
+ "grad_norm": 2.0,
1977
+ "learning_rate": 3.0407915184435544e-06,
1978
+ "loss": 0.7414107513427735,
1979
+ "mean_token_accuracy": 0.8034427273273468,
1980
+ "num_tokens": 26928821.0,
1981
+ "step": 4350
1982
+ },
1983
+ {
1984
+ "entropy": 0.7746455824375152,
1985
+ "epoch": 1.5144538687900293,
1986
+ "grad_norm": 2.15625,
1987
+ "learning_rate": 2.9408342675016288e-06,
1988
+ "loss": 0.7338740539550781,
1989
+ "mean_token_accuracy": 0.8053066456317901,
1990
+ "num_tokens": 27081955.0,
1991
+ "step": 4375
1992
+ },
1993
+ {
1994
+ "entropy": 0.7544065153598786,
1995
+ "epoch": 1.5231088800415442,
1996
+ "grad_norm": 2.125,
1997
+ "learning_rate": 2.84226354818263e-06,
1998
+ "loss": 0.7168594360351562,
1999
+ "mean_token_accuracy": 0.8104126238822937,
2000
+ "num_tokens": 27233223.0,
2001
+ "step": 4400
2002
+ },
2003
+ {
2004
+ "epoch": 1.5231088800415442,
2005
+ "eval_entropy": 0.770115079156688,
2006
+ "eval_loss": 0.7993313074111938,
2007
+ "eval_mean_token_accuracy": 0.7934993093131019,
2008
+ "eval_num_tokens": 27233223.0,
2009
+ "eval_runtime": 16.2903,
2010
+ "eval_samples_per_second": 298.644,
2011
+ "eval_steps_per_second": 18.723,
2012
+ "step": 4400
2013
+ },
2014
+ {
2015
+ "entropy": 0.7756598269939423,
2016
+ "epoch": 1.5317638912930587,
2017
+ "grad_norm": 2.15625,
2018
+ "learning_rate": 2.74509872133238e-06,
2019
+ "loss": 0.750801010131836,
2020
+ "mean_token_accuracy": 0.8043915629386902,
2021
+ "num_tokens": 27385671.0,
2022
+ "step": 4425
2023
+ },
2024
+ {
2025
+ "entropy": 0.7870103931427002,
2026
+ "epoch": 1.5404189025445731,
2027
+ "grad_norm": 2.46875,
2028
+ "learning_rate": 2.6493588716572306e-06,
2029
+ "loss": 0.7561086273193359,
2030
+ "mean_token_accuracy": 0.8031144332885742,
2031
+ "num_tokens": 27542323.0,
2032
+ "step": 4450
2033
+ },
2034
+ {
2035
+ "entropy": 0.7758941638469696,
2036
+ "epoch": 1.549073913796088,
2037
+ "grad_norm": 2.359375,
2038
+ "learning_rate": 2.555062803975512e-06,
2039
+ "loss": 0.7479061889648437,
2040
+ "mean_token_accuracy": 0.8041390895843505,
2041
+ "num_tokens": 27695755.0,
2042
+ "step": 4475
2043
+ },
2044
+ {
2045
+ "entropy": 0.754768637418747,
2046
+ "epoch": 1.5577289250476025,
2047
+ "grad_norm": 2.4375,
2048
+ "learning_rate": 2.4622290395239613e-06,
2049
+ "loss": 0.7247459411621093,
2050
+ "mean_token_accuracy": 0.8090137577056885,
2051
+ "num_tokens": 27847694.0,
2052
+ "step": 4500
2053
+ },
2054
+ {
2055
+ "entropy": 0.7636888045072555,
2056
+ "epoch": 1.5663839362991172,
2057
+ "grad_norm": 2.140625,
2058
+ "learning_rate": 2.3708758123198904e-06,
2059
+ "loss": 0.7294938659667969,
2060
+ "mean_token_accuracy": 0.8078456258773804,
2061
+ "num_tokens": 28001389.0,
2062
+ "step": 4525
2063
+ },
2064
+ {
2065
+ "entropy": 0.7723946702480317,
2066
+ "epoch": 1.575038947550632,
2067
+ "grad_norm": 1.9453125,
2068
+ "learning_rate": 2.2810210655796915e-06,
2069
+ "loss": 0.7421239471435547,
2070
+ "mean_token_accuracy": 0.8051141202449799,
2071
+ "num_tokens": 28155633.0,
2072
+ "step": 4550
2073
+ },
2074
+ {
2075
+ "entropy": 0.7729098165035247,
2076
+ "epoch": 1.5836939588021464,
2077
+ "grad_norm": 1.9375,
2078
+ "learning_rate": 2.192682448194555e-06,
2079
+ "loss": 0.7385528564453125,
2080
+ "mean_token_accuracy": 0.8061604249477387,
2081
+ "num_tokens": 28316815.0,
2082
+ "step": 4575
2083
+ },
2084
+ {
2085
+ "entropy": 0.7528582894802094,
2086
+ "epoch": 1.592348970053661,
2087
+ "grad_norm": 2.234375,
2088
+ "learning_rate": 2.1058773112639054e-06,
2089
+ "loss": 0.719280014038086,
2090
+ "mean_token_accuracy": 0.8096978998184204,
2091
+ "num_tokens": 28473438.0,
2092
+ "step": 4600
2093
+ },
2094
+ {
2095
+ "epoch": 1.592348970053661,
2096
+ "eval_entropy": 0.7718443285246365,
2097
+ "eval_loss": 0.7991446852684021,
2098
+ "eval_mean_token_accuracy": 0.7934910416603088,
2099
+ "eval_num_tokens": 28473438.0,
2100
+ "eval_runtime": 16.2845,
2101
+ "eval_samples_per_second": 298.751,
2102
+ "eval_steps_per_second": 18.729,
2103
+ "step": 4600
2104
+ },
2105
+ {
2106
+ "entropy": 0.7752782785892487,
2107
+ "epoch": 1.6010039813051757,
2108
+ "grad_norm": 2.203125,
2109
+ "learning_rate": 2.020622704687398e-06,
2110
+ "loss": 0.7455803680419922,
2111
+ "mean_token_accuracy": 0.8029440557956695,
2112
+ "num_tokens": 28629933.0,
2113
+ "step": 4625
2114
+ },
2115
+ {
2116
+ "entropy": 0.7682564723491668,
2117
+ "epoch": 1.6096589925566902,
2118
+ "grad_norm": 2.28125,
2119
+ "learning_rate": 1.936935373816046e-06,
2120
+ "loss": 0.7349263000488281,
2121
+ "mean_token_accuracy": 0.8075251269340515,
2122
+ "num_tokens": 28779447.0,
2123
+ "step": 4650
2124
+ },
2125
+ {
2126
+ "entropy": 0.7602540409564972,
2127
+ "epoch": 1.6183140038082051,
2128
+ "grad_norm": 2.15625,
2129
+ "learning_rate": 1.8548317561631534e-06,
2130
+ "loss": 0.722344741821289,
2131
+ "mean_token_accuracy": 0.809203851222992,
2132
+ "num_tokens": 28935999.0,
2133
+ "step": 4675
2134
+ },
2135
+ {
2136
+ "entropy": 0.7502141880989075,
2137
+ "epoch": 1.6269690150597196,
2138
+ "grad_norm": 1.984375,
2139
+ "learning_rate": 1.7743279781757493e-06,
2140
+ "loss": 0.718406982421875,
2141
+ "mean_token_accuracy": 0.8091218245029449,
2142
+ "num_tokens": 29092219.0,
2143
+ "step": 4700
2144
+ },
2145
+ {
2146
+ "entropy": 0.7485624659061432,
2147
+ "epoch": 1.635624026311234,
2148
+ "grad_norm": 1.9375,
2149
+ "learning_rate": 1.6954398520670867e-06,
2150
+ "loss": 0.7138198852539063,
2151
+ "mean_token_accuracy": 0.8104638087749482,
2152
+ "num_tokens": 29249386.0,
2153
+ "step": 4725
2154
+ },
2155
+ {
2156
+ "entropy": 0.7689664399623871,
2157
+ "epoch": 1.644279037562749,
2158
+ "grad_norm": 2.0625,
2159
+ "learning_rate": 1.6181828727108772e-06,
2160
+ "loss": 0.7358683776855469,
2161
+ "mean_token_accuracy": 0.8066857266426086,
2162
+ "num_tokens": 29406880.0,
2163
+ "step": 4750
2164
+ },
2165
+ {
2166
+ "entropy": 0.7672972738742828,
2167
+ "epoch": 1.6529340488142634,
2168
+ "grad_norm": 2.40625,
2169
+ "learning_rate": 1.542572214597844e-06,
2170
+ "loss": 0.7319490051269532,
2171
+ "mean_token_accuracy": 0.8055110704898835,
2172
+ "num_tokens": 29558251.0,
2173
+ "step": 4775
2174
+ },
2175
+ {
2176
+ "entropy": 0.7510581874847412,
2177
+ "epoch": 1.6615890600657781,
2178
+ "grad_norm": 2.078125,
2179
+ "learning_rate": 1.4686227288552357e-06,
2180
+ "loss": 0.7089730072021484,
2181
+ "mean_token_accuracy": 0.811128660440445,
2182
+ "num_tokens": 29713889.0,
2183
+ "step": 4800
2184
+ },
2185
+ {
2186
+ "epoch": 1.6615890600657781,
2187
+ "eval_entropy": 0.7707234706057877,
2188
+ "eval_loss": 0.7992180585861206,
2189
+ "eval_mean_token_accuracy": 0.7935016180648178,
2190
+ "eval_num_tokens": 29713889.0,
2191
+ "eval_runtime": 16.3186,
2192
+ "eval_samples_per_second": 298.126,
2193
+ "eval_steps_per_second": 18.69,
2194
+ "step": 4800
2195
+ },
2196
+ {
2197
+ "entropy": 0.7817390990257264,
2198
+ "epoch": 1.6702440713172928,
2199
+ "grad_norm": 2.25,
2200
+ "learning_rate": 1.3963489403297992e-06,
2201
+ "loss": 0.7543077087402343,
2202
+ "mean_token_accuracy": 0.8011208462715149,
2203
+ "num_tokens": 29870315.0,
2204
+ "step": 4825
2205
+ },
2206
+ {
2207
+ "entropy": 0.7862332105636597,
2208
+ "epoch": 1.6788990825688073,
2209
+ "grad_norm": 2.125,
2210
+ "learning_rate": 1.3257650447349035e-06,
2211
+ "loss": 0.7527214050292969,
2212
+ "mean_token_accuracy": 0.8031284868717193,
2213
+ "num_tokens": 30023428.0,
2214
+ "step": 4850
2215
+ },
2216
+ {
2217
+ "entropy": 0.7391238558292389,
2218
+ "epoch": 1.687554093820322,
2219
+ "grad_norm": 2.046875,
2220
+ "learning_rate": 1.2568849058622435e-06,
2221
+ "loss": 0.7039041137695312,
2222
+ "mean_token_accuracy": 0.811707170009613,
2223
+ "num_tokens": 30181568.0,
2224
+ "step": 4875
2225
+ },
2226
+ {
2227
+ "entropy": 0.7737973690032959,
2228
+ "epoch": 1.6962091050718366,
2229
+ "grad_norm": 2.125,
2230
+ "learning_rate": 1.1897220528588005e-06,
2231
+ "loss": 0.7400728607177735,
2232
+ "mean_token_accuracy": 0.8040676033496856,
2233
+ "num_tokens": 30334041.0,
2234
+ "step": 4900
2235
+ },
2236
+ {
2237
+ "entropy": 0.7637208473682403,
2238
+ "epoch": 1.704864116323351,
2239
+ "grad_norm": 2.265625,
2240
+ "learning_rate": 1.1242896775694923e-06,
2241
+ "loss": 0.7268269348144532,
2242
+ "mean_token_accuracy": 0.8077472651004791,
2243
+ "num_tokens": 30488073.0,
2244
+ "step": 4925
2245
+ },
2246
+ {
2247
+ "entropy": 0.7567976391315461,
2248
+ "epoch": 1.7135191275748658,
2249
+ "grad_norm": 1.984375,
2250
+ "learning_rate": 1.0606006319460794e-06,
2251
+ "loss": 0.7226065826416016,
2252
+ "mean_token_accuracy": 0.8086667931079865,
2253
+ "num_tokens": 30643659.0,
2254
+ "step": 4950
2255
+ },
2256
+ {
2257
+ "entropy": 0.7528193962574005,
2258
+ "epoch": 1.7221741388263805,
2259
+ "grad_norm": 2.09375,
2260
+ "learning_rate": 9.986674255228546e-07,
2261
+ "loss": 0.7181925964355469,
2262
+ "mean_token_accuracy": 0.8110329484939576,
2263
+ "num_tokens": 30794462.0,
2264
+ "step": 4975
2265
+ },
2266
+ {
2267
+ "entropy": 0.7771501529216767,
2268
+ "epoch": 1.730829150077895,
2269
+ "grad_norm": 2.203125,
2270
+ "learning_rate": 9.385022229595631e-07,
2271
+ "loss": 0.7418533325195312,
2272
+ "mean_token_accuracy": 0.8045794427394867,
2273
+ "num_tokens": 30945283.0,
2274
+ "step": 5000
2275
+ },
2276
+ {
2277
+ "epoch": 1.730829150077895,
2278
+ "eval_entropy": 0.7709188702653667,
2279
+ "eval_loss": 0.7991884350776672,
2280
+ "eval_mean_token_accuracy": 0.7934240538565839,
2281
+ "eval_num_tokens": 30945283.0,
2282
+ "eval_runtime": 16.2662,
2283
+ "eval_samples_per_second": 299.086,
2284
+ "eval_steps_per_second": 18.751,
2285
+ "step": 5000
2286
+ },
2287
+ {
2288
+ "entropy": 0.7722408485412597,
2289
+ "epoch": 1.7394841613294099,
2290
+ "grad_norm": 2.0625,
2291
+ "learning_rate": 8.801168416520733e-07,
2292
+ "loss": 0.7394268035888671,
2293
+ "mean_token_accuracy": 0.806780993938446,
2294
+ "num_tokens": 31105352.0,
2295
+ "step": 5025
2296
+ },
2297
+ {
2298
+ "entropy": 0.7758919060230255,
2299
+ "epoch": 1.7481391725809243,
2300
+ "grad_norm": 2.0625,
2301
+ "learning_rate": 8.235227494112463e-07,
2302
+ "loss": 0.7348842620849609,
2303
+ "mean_token_accuracy": 0.8069607448577881,
2304
+ "num_tokens": 31253216.0,
2305
+ "step": 5050
2306
+ },
2307
+ {
2308
+ "entropy": 0.7573688685894012,
2309
+ "epoch": 1.756794183832439,
2310
+ "grad_norm": 2.0,
2311
+ "learning_rate": 7.687310622104982e-07,
2312
+ "loss": 0.7183684539794922,
2313
+ "mean_token_accuracy": 0.8097042238712311,
2314
+ "num_tokens": 31404175.0,
2315
+ "step": 5075
2316
+ },
2317
+ {
2318
+ "entropy": 0.7699829995632171,
2319
+ "epoch": 1.7654491950839537,
2320
+ "grad_norm": 2.21875,
2321
+ "learning_rate": 7.157525420024181e-07,
2322
+ "loss": 0.7363372802734375,
2323
+ "mean_token_accuracy": 0.8065412914752961,
2324
+ "num_tokens": 31553043.0,
2325
+ "step": 5100
2326
+ },
2327
+ {
2328
+ "entropy": 0.7569133055210113,
2329
+ "epoch": 1.7741042063354682,
2330
+ "grad_norm": 1.9609375,
2331
+ "learning_rate": 6.645975946049732e-07,
2332
+ "loss": 0.7242645263671875,
2333
+ "mean_token_accuracy": 0.8096802568435669,
2334
+ "num_tokens": 31708719.0,
2335
+ "step": 5125
2336
+ },
2337
+ {
2338
+ "entropy": 0.763712112903595,
2339
+ "epoch": 1.7827592175869829,
2340
+ "grad_norm": 2.125,
2341
+ "learning_rate": 6.152762676576352e-07,
2342
+ "loss": 0.7330033111572266,
2343
+ "mean_token_accuracy": 0.8064149105548859,
2344
+ "num_tokens": 31869942.0,
2345
+ "step": 5150
2346
+ },
2347
+ {
2348
+ "entropy": 0.7759097361564636,
2349
+ "epoch": 1.7914142288384975,
2350
+ "grad_norm": 2.203125,
2351
+ "learning_rate": 5.677982486478595e-07,
2352
+ "loss": 0.7459344482421875,
2353
+ "mean_token_accuracy": 0.8050018990039826,
2354
+ "num_tokens": 32024066.0,
2355
+ "step": 5175
2356
+ },
2357
+ {
2358
+ "entropy": 0.7608749055862427,
2359
+ "epoch": 1.800069240090012,
2360
+ "grad_norm": 2.09375,
2361
+ "learning_rate": 5.221728630083201e-07,
2362
+ "loss": 0.7280139923095703,
2363
+ "mean_token_accuracy": 0.8076766312122345,
2364
+ "num_tokens": 32180495.0,
2365
+ "step": 5200
2366
+ },
2367
+ {
2368
+ "epoch": 1.800069240090012,
2369
+ "eval_entropy": 0.7705254261610938,
2370
+ "eval_loss": 0.7991676330566406,
2371
+ "eval_mean_token_accuracy": 0.7934689746528375,
2372
+ "eval_num_tokens": 32180495.0,
2373
+ "eval_runtime": 16.2695,
2374
+ "eval_samples_per_second": 299.026,
2375
+ "eval_steps_per_second": 18.747,
2376
+ "step": 5200
2377
+ },
2378
+ {
2379
+ "entropy": 0.7587528014183045,
2380
+ "epoch": 1.8087242513415267,
2381
+ "grad_norm": 2.3125,
2382
+ "learning_rate": 4.784090722852353e-07,
2383
+ "loss": 0.7158480834960937,
2384
+ "mean_token_accuracy": 0.809622370004654,
2385
+ "num_tokens": 32338906.0,
2386
+ "step": 5225
2387
+ },
2388
+ {
2389
+ "entropy": 0.7616716659069062,
2390
+ "epoch": 1.8173792625930414,
2391
+ "grad_norm": 2.109375,
2392
+ "learning_rate": 4.365154723781895e-07,
2393
+ "loss": 0.7293673706054687,
2394
+ "mean_token_accuracy": 0.8077958106994629,
2395
+ "num_tokens": 32495631.0,
2396
+ "step": 5250
2397
+ },
2398
+ {
2399
+ "entropy": 0.737936383485794,
2400
+ "epoch": 1.8260342738445559,
2401
+ "grad_norm": 2.046875,
2402
+ "learning_rate": 3.965002918517569e-07,
2403
+ "loss": 0.7038777160644532,
2404
+ "mean_token_accuracy": 0.8131058704853058,
2405
+ "num_tokens": 32651647.0,
2406
+ "step": 5275
2407
+ },
2408
+ {
2409
+ "entropy": 0.7654093158245087,
2410
+ "epoch": 1.8346892850960708,
2411
+ "grad_norm": 2.21875,
2412
+ "learning_rate": 3.5837139031928315e-07,
2413
+ "loss": 0.7362342071533203,
2414
+ "mean_token_accuracy": 0.8066077816486359,
2415
+ "num_tokens": 32809688.0,
2416
+ "step": 5300
2417
+ },
2418
+ {
2419
+ "entropy": 0.7611633980274201,
2420
+ "epoch": 1.8433442963475852,
2421
+ "grad_norm": 2.25,
2422
+ "learning_rate": 3.221362568991293e-07,
2423
+ "loss": 0.7225106811523437,
2424
+ "mean_token_accuracy": 0.809012303352356,
2425
+ "num_tokens": 32962531.0,
2426
+ "step": 5325
2427
+ },
2428
+ {
2429
+ "entropy": 0.7749582767486572,
2430
+ "epoch": 1.8519993075991,
2431
+ "grad_norm": 2.28125,
2432
+ "learning_rate": 2.8780200874370165e-07,
2433
+ "loss": 0.7392816925048828,
2434
+ "mean_token_accuracy": 0.803524352312088,
2435
+ "num_tokens": 33122116.0,
2436
+ "step": 5350
2437
+ },
2438
+ {
2439
+ "entropy": 0.7610144579410553,
2440
+ "epoch": 1.8606543188506146,
2441
+ "grad_norm": 2.28125,
2442
+ "learning_rate": 2.5537538964151676e-07,
2443
+ "loss": 0.7229084014892578,
2444
+ "mean_token_accuracy": 0.8083787798881531,
2445
+ "num_tokens": 33273413.0,
2446
+ "step": 5375
2447
+ },
2448
+ {
2449
+ "entropy": 0.791879060268402,
2450
+ "epoch": 1.869309330102129,
2451
+ "grad_norm": 2.234375,
2452
+ "learning_rate": 2.248627686926219e-07,
2453
+ "loss": 0.7606393432617188,
2454
+ "mean_token_accuracy": 0.8025156807899475,
2455
+ "num_tokens": 33421728.0,
2456
+ "step": 5400
2457
+ },
2458
+ {
2459
+ "epoch": 1.869309330102129,
2460
+ "eval_entropy": 0.770491715630547,
2461
+ "eval_loss": 0.7991881370544434,
2462
+ "eval_mean_token_accuracy": 0.7935233919346919,
2463
+ "eval_num_tokens": 33421728.0,
2464
+ "eval_runtime": 16.2639,
2465
+ "eval_samples_per_second": 299.128,
2466
+ "eval_steps_per_second": 18.753,
2467
+ "step": 5400
2468
+ },
2469
+ {
2470
+ "entropy": 0.7707629156112671,
2471
+ "epoch": 1.8779643413536438,
2472
+ "grad_norm": 2.265625,
2473
+ "learning_rate": 1.9627013905760605e-07,
2474
+ "loss": 0.7430142974853515,
2475
+ "mean_token_accuracy": 0.8047948408126832,
2476
+ "num_tokens": 33576359.0,
2477
+ "step": 5425
2478
+ },
2479
+ {
2480
+ "entropy": 0.7577464747428894,
2481
+ "epoch": 1.8866193526051585,
2482
+ "grad_norm": 2.171875,
2483
+ "learning_rate": 1.6960311678044018e-07,
2484
+ "loss": 0.7168014526367188,
2485
+ "mean_token_accuracy": 0.8094685411453247,
2486
+ "num_tokens": 33723770.0,
2487
+ "step": 5450
2488
+ },
2489
+ {
2490
+ "entropy": 0.7747404301166534,
2491
+ "epoch": 1.895274363856673,
2492
+ "grad_norm": 2.0625,
2493
+ "learning_rate": 1.4486693968540743e-07,
2494
+ "loss": 0.7389591979980469,
2495
+ "mean_token_accuracy": 0.8045553112030029,
2496
+ "num_tokens": 33879362.0,
2497
+ "step": 5475
2498
+ },
2499
+ {
2500
+ "entropy": 0.7680295896530152,
2501
+ "epoch": 1.9039293751081876,
2502
+ "grad_norm": 2.046875,
2503
+ "learning_rate": 1.22066466348304e-07,
2504
+ "loss": 0.7351342010498046,
2505
+ "mean_token_accuracy": 0.8047092247009278,
2506
+ "num_tokens": 34035862.0,
2507
+ "step": 5500
2508
+ },
2509
+ {
2510
+ "entropy": 0.7560386550426483,
2511
+ "epoch": 1.9125843863597023,
2512
+ "grad_norm": 2.15625,
2513
+ "learning_rate": 1.0120617514214115e-07,
2514
+ "loss": 0.7233296966552735,
2515
+ "mean_token_accuracy": 0.8088453054428101,
2516
+ "num_tokens": 34192367.0,
2517
+ "step": 5525
2518
+ },
2519
+ {
2520
+ "entropy": 0.749894061088562,
2521
+ "epoch": 1.9212393976112168,
2522
+ "grad_norm": 2.234375,
2523
+ "learning_rate": 8.229016335752238e-08,
2524
+ "loss": 0.721561279296875,
2525
+ "mean_token_accuracy": 0.8098326814174652,
2526
+ "num_tokens": 34346639.0,
2527
+ "step": 5550
2528
+ },
2529
+ {
2530
+ "entropy": 0.7512237560749054,
2531
+ "epoch": 1.9298944088627317,
2532
+ "grad_norm": 2.203125,
2533
+ "learning_rate": 6.532214639786926e-08,
2534
+ "loss": 0.7133358001708985,
2535
+ "mean_token_accuracy": 0.8117510068416596,
2536
+ "num_tokens": 34501969.0,
2537
+ "step": 5575
2538
+ },
2539
+ {
2540
+ "entropy": 0.7577315938472747,
2541
+ "epoch": 1.9385494201142461,
2542
+ "grad_norm": 1.984375,
2543
+ "learning_rate": 5.03054570496575e-08,
2544
+ "loss": 0.7302328491210938,
2545
+ "mean_token_accuracy": 0.8082369923591614,
2546
+ "num_tokens": 34658561.0,
2547
+ "step": 5600
2548
+ },
2549
+ {
2550
+ "epoch": 1.9385494201142461,
2551
+ "eval_entropy": 0.7700900946484237,
2552
+ "eval_loss": 0.7992120981216431,
2553
+ "eval_mean_token_accuracy": 0.7935370582049011,
2554
+ "eval_num_tokens": 34658561.0,
2555
+ "eval_runtime": 16.284,
2556
+ "eval_samples_per_second": 298.76,
2557
+ "eval_steps_per_second": 18.73,
2558
+ "step": 5600
2559
+ }
2560
+ ],
2561
+ "logging_steps": 25,
2562
+ "max_steps": 5778,
2563
+ "num_input_tokens_seen": 0,
2564
+ "num_train_epochs": 2,
2565
+ "save_steps": 200,
2566
+ "stateful_callbacks": {
2567
+ "TrainerControl": {
2568
+ "args": {
2569
+ "should_epoch_stop": false,
2570
+ "should_evaluate": false,
2571
+ "should_log": false,
2572
+ "should_save": true,
2573
+ "should_training_stop": false
2574
+ },
2575
+ "attributes": {}
2576
+ }
2577
+ },
2578
+ "total_flos": 1.4156455922319667e+17,
2579
+ "train_batch_size": 16,
2580
+ "trial_name": null,
2581
+ "trial_params": null
2582
+ }
checkpoint-5600/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fe1d1ac135572b0e564d6340370a449cb9281b7839efca5df924e60c2ca5dbe
3
+ size 5585
checkpoint-5778/chat_template.jinja ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set keep_past_thinking = keep_past_thinking | default(false) -%}
3
+ {%- set ns = namespace(system_prompt="") -%}
4
+ {%- if messages[0]["role"] == "system" -%}
5
+ {%- set sys_content = messages[0]["content"] -%}
6
+ {%- if sys_content is not string -%}
7
+ {%- for item in sys_content -%}
8
+ {%- if item["type"] == "text" -%}
9
+ {%- set ns.system_prompt = ns.system_prompt + item["text"] -%}
10
+ {%- endif -%}
11
+ {%- endfor -%}
12
+ {%- else -%}
13
+ {%- set ns.system_prompt = sys_content -%}
14
+ {%- endif -%}
15
+ {%- set messages = messages[1:] -%}
16
+ {%- endif -%}
17
+ {%- if tools -%}
18
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
19
+ {%- for tool in tools -%}
20
+ {%- if tool is not string -%}
21
+ {%- set tool = tool | tojson -%}
22
+ {%- endif -%}
23
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
24
+ {%- if not loop.last -%}
25
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- set ns.system_prompt = ns.system_prompt + "]" -%}
29
+ {%- endif -%}
30
+ {%- if ns.system_prompt -%}
31
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
32
+ {%- endif -%}
33
+ {%- set ns.last_assistant_index = -1 -%}
34
+ {%- for message in messages -%}
35
+ {%- if message["role"] == "assistant" -%}
36
+ {%- set ns.last_assistant_index = loop.index0 -%}
37
+ {%- endif -%}
38
+ {%- endfor -%}
39
+ {%- for message in messages -%}
40
+ {{- "<|im_start|>" + message["role"] + "\n" -}}
41
+ {%- set content = message["content"] -%}
42
+ {%- if content is not string -%}
43
+ {%- set ns.content = "" -%}
44
+ {%- for item in content -%}
45
+ {%- if item["type"] == "image" -%}
46
+ {%- set ns.content = ns.content + "<image>" -%}
47
+ {%- elif item["type"] == "text" -%}
48
+ {%- set ns.content = ns.content + item["text"] -%}
49
+ {%- else -%}
50
+ {%- set ns.content = ns.content + item | tojson -%}
51
+ {%- endif -%}
52
+ {%- endfor -%}
53
+ {%- set content = ns.content -%}
54
+ {%- endif -%}
55
+ {%- if message["role"] == "assistant" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}
56
+ {%- if "</think>" in content -%}
57
+ {%- set content = content.split("</think>")[-1] | trim -%}
58
+ {%- endif -%}
59
+ {%- endif -%}
60
+ {{- content + "<|im_end|>\n" -}}
61
+ {%- endfor -%}
62
+ {%- if add_generation_prompt -%}
63
+ {{- "<|im_start|>assistant\n" -}}
64
+ {%- endif -%}
checkpoint-5778/config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2ForCausalLM"
4
+ ],
5
+ "block_auto_adjust_ff_dim": true,
6
+ "block_dim": 1024,
7
+ "block_ffn_dim_multiplier": 1.0,
8
+ "block_mlp_init_scale": 1.0,
9
+ "block_multiple_of": 256,
10
+ "block_norm_eps": 1e-05,
11
+ "block_out_init_scale": 1.0,
12
+ "block_use_swiglu": true,
13
+ "block_use_xavier_init": true,
14
+ "bos_token_id": 1,
15
+ "conv_L_cache": 3,
16
+ "conv_bias": false,
17
+ "conv_dim": 1024,
18
+ "conv_use_xavier_init": true,
19
+ "dtype": "bfloat16",
20
+ "eos_token_id": 7,
21
+ "full_attn_idxs": null,
22
+ "hidden_size": 1024,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 6656,
25
+ "layer_types": [
26
+ "conv",
27
+ "conv",
28
+ "full_attention",
29
+ "conv",
30
+ "conv",
31
+ "full_attention",
32
+ "conv",
33
+ "conv",
34
+ "full_attention",
35
+ "conv",
36
+ "full_attention",
37
+ "conv",
38
+ "full_attention",
39
+ "conv",
40
+ "full_attention",
41
+ "conv"
42
+ ],
43
+ "max_position_embeddings": 128000,
44
+ "model_type": "lfm2",
45
+ "norm_eps": 1e-05,
46
+ "num_attention_heads": 16,
47
+ "num_heads": 16,
48
+ "num_hidden_layers": 16,
49
+ "num_key_value_heads": 8,
50
+ "pad_token_id": 0,
51
+ "rope_parameters": {
52
+ "rope_theta": 1000000.0,
53
+ "rope_type": "default"
54
+ },
55
+ "tie_word_embeddings": true,
56
+ "transformers_version": "5.5.0",
57
+ "use_cache": false,
58
+ "use_pos_enc": true,
59
+ "vocab_size": 65536
60
+ }
checkpoint-5778/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 7
6
+ ],
7
+ "pad_token_id": 0,
8
+ "transformers_version": "5.5.0"
9
+ }
checkpoint-5778/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36971c587161cf2d7d641ff2c68ad3db01a3ce0d22255d5072f4ee436a400015
3
+ size 708984464
checkpoint-5778/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0bc7fc6b77dda23b205c2d0ecddc098145df98f28a6fd72d2c1cbc753fc1b30
3
+ size 1418061003
checkpoint-5778/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:081b902a1cc1d7c04ce34bfbbb677ba64777c8e7d0d114a6cad2aaf4dff7c81d
3
+ size 14645
checkpoint-5778/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9168ea808b8833b03293e21d97fb9f31c83b2a6385e1dc2b915463678d40557a
3
+ size 1465
checkpoint-5778/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-5778/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|im_end|>",
6
+ "extra_special_tokens": [],
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "model_input_names": [
10
+ "input_ids",
11
+ "attention_mask"
12
+ ],
13
+ "model_max_length": 1000000000000000019884624838656,
14
+ "model_specific_special_tokens": {},
15
+ "pad_token": "<|pad|>",
16
+ "sp_model_kwargs": {},
17
+ "spaces_between_special_tokens": false,
18
+ "tokenizer_class": "TokenizersBackend",
19
+ "use_default_system_prompt": false,
20
+ "use_fast": true
21
+ }
checkpoint-5778/trainer_state.json ADDED
@@ -0,0 +1,2663 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 4600,
3
+ "best_metric": 0.7991446852684021,
4
+ "best_model_checkpoint": "lfm2.5-350m-multilang-code-math/checkpoint-4600",
5
+ "epoch": 2.0,
6
+ "eval_steps": 200,
7
+ "global_step": 5778,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2043220043182372,
14
+ "epoch": 0.008655011251514627,
15
+ "grad_norm": 23.5,
16
+ "learning_rate": 2.7586206896551725e-06,
17
+ "loss": 3.0389138793945314,
18
+ "mean_token_accuracy": 0.5589780753850937,
19
+ "num_tokens": 151155.0,
20
+ "step": 25
21
+ },
22
+ {
23
+ "entropy": 1.11516796708107,
24
+ "epoch": 0.017310022503029255,
25
+ "grad_norm": 7.34375,
26
+ "learning_rate": 5.6321839080459775e-06,
27
+ "loss": 1.8928765869140625,
28
+ "mean_token_accuracy": 0.6762128543853759,
29
+ "num_tokens": 310957.0,
30
+ "step": 50
31
+ },
32
+ {
33
+ "entropy": 1.2031074380874633,
34
+ "epoch": 0.02596503375454388,
35
+ "grad_norm": 3.78125,
36
+ "learning_rate": 8.505747126436782e-06,
37
+ "loss": 1.2897039794921874,
38
+ "mean_token_accuracy": 0.7282541489601135,
39
+ "num_tokens": 461068.0,
40
+ "step": 75
41
+ },
42
+ {
43
+ "entropy": 1.0323714792728425,
44
+ "epoch": 0.03462004500605851,
45
+ "grad_norm": 3.71875,
46
+ "learning_rate": 1.1379310344827587e-05,
47
+ "loss": 1.0183975982666016,
48
+ "mean_token_accuracy": 0.757768098115921,
49
+ "num_tokens": 615118.0,
50
+ "step": 100
51
+ },
52
+ {
53
+ "entropy": 0.9247142732143402,
54
+ "epoch": 0.043275056257573136,
55
+ "grad_norm": 2.578125,
56
+ "learning_rate": 1.4252873563218392e-05,
57
+ "loss": 0.8757442474365235,
58
+ "mean_token_accuracy": 0.7792941844463348,
59
+ "num_tokens": 762906.0,
60
+ "step": 125
61
+ },
62
+ {
63
+ "entropy": 0.9190187740325928,
64
+ "epoch": 0.05193006750908776,
65
+ "grad_norm": 2.296875,
66
+ "learning_rate": 1.7126436781609197e-05,
67
+ "loss": 0.891803970336914,
68
+ "mean_token_accuracy": 0.7753661525249481,
69
+ "num_tokens": 921356.0,
70
+ "step": 150
71
+ },
72
+ {
73
+ "entropy": 0.886636551618576,
74
+ "epoch": 0.06058507876060239,
75
+ "grad_norm": 2.296875,
76
+ "learning_rate": 2e-05,
77
+ "loss": 0.861456298828125,
78
+ "mean_token_accuracy": 0.7826532816886902,
79
+ "num_tokens": 1076183.0,
80
+ "step": 175
81
+ },
82
+ {
83
+ "entropy": 0.9015009069442749,
84
+ "epoch": 0.06924009001211702,
85
+ "grad_norm": 2.5625,
86
+ "learning_rate": 1.9999017921043748e-05,
87
+ "loss": 0.8748048400878906,
88
+ "mean_token_accuracy": 0.7792416322231293,
89
+ "num_tokens": 1229170.0,
90
+ "step": 200
91
+ },
92
+ {
93
+ "epoch": 0.06924009001211702,
94
+ "eval_entropy": 0.8920288547140653,
95
+ "eval_loss": 0.8666635155677795,
96
+ "eval_mean_token_accuracy": 0.7797926357535065,
97
+ "eval_num_tokens": 1229170.0,
98
+ "eval_runtime": 16.3134,
99
+ "eval_samples_per_second": 298.222,
100
+ "eval_steps_per_second": 18.696,
101
+ "step": 200
102
+ },
103
+ {
104
+ "entropy": 0.9062747704982758,
105
+ "epoch": 0.07789510126363164,
106
+ "grad_norm": 2.40625,
107
+ "learning_rate": 1.999607187707079e-05,
108
+ "loss": 0.8789938354492187,
109
+ "mean_token_accuracy": 0.7778702569007874,
110
+ "num_tokens": 1381380.0,
111
+ "step": 225
112
+ },
113
+ {
114
+ "entropy": 0.8776320171356201,
115
+ "epoch": 0.08655011251514627,
116
+ "grad_norm": 2.46875,
117
+ "learning_rate": 1.9991162446730694e-05,
118
+ "loss": 0.8517158508300782,
119
+ "mean_token_accuracy": 0.7819743084907532,
120
+ "num_tokens": 1538186.0,
121
+ "step": 250
122
+ },
123
+ {
124
+ "entropy": 0.8657179689407348,
125
+ "epoch": 0.09520512376666089,
126
+ "grad_norm": 2.25,
127
+ "learning_rate": 1.9984290594313102e-05,
128
+ "loss": 0.8408647155761719,
129
+ "mean_token_accuracy": 0.7840179252624512,
130
+ "num_tokens": 1690067.0,
131
+ "step": 275
132
+ },
133
+ {
134
+ "entropy": 0.8955290937423706,
135
+ "epoch": 0.10386013501817552,
136
+ "grad_norm": 2.328125,
137
+ "learning_rate": 1.9975457669558346e-05,
138
+ "loss": 0.8683735656738282,
139
+ "mean_token_accuracy": 0.7763688516616821,
140
+ "num_tokens": 1845254.0,
141
+ "step": 300
142
+ },
143
+ {
144
+ "entropy": 0.8744900381565094,
145
+ "epoch": 0.11251514626969016,
146
+ "grad_norm": 2.34375,
147
+ "learning_rate": 1.996466540739233e-05,
148
+ "loss": 0.8470248413085938,
149
+ "mean_token_accuracy": 0.7828968441486359,
150
+ "num_tokens": 2000855.0,
151
+ "step": 325
152
+ },
153
+ {
154
+ "entropy": 0.8875207018852234,
155
+ "epoch": 0.12117015752120477,
156
+ "grad_norm": 2.34375,
157
+ "learning_rate": 1.9951915927585762e-05,
158
+ "loss": 0.8633122253417969,
159
+ "mean_token_accuracy": 0.7807920384407043,
160
+ "num_tokens": 2159255.0,
161
+ "step": 350
162
+ },
163
+ {
164
+ "entropy": 0.8727393555641174,
165
+ "epoch": 0.1298251687727194,
166
+ "grad_norm": 2.40625,
167
+ "learning_rate": 1.9937211734337812e-05,
168
+ "loss": 0.8484575653076172,
169
+ "mean_token_accuracy": 0.7856597316265106,
170
+ "num_tokens": 2313980.0,
171
+ "step": 375
172
+ },
173
+ {
174
+ "entropy": 0.8450514304637909,
175
+ "epoch": 0.13848018002423404,
176
+ "grad_norm": 2.390625,
177
+ "learning_rate": 1.992055571578423e-05,
178
+ "loss": 0.8231178283691406,
179
+ "mean_token_accuracy": 0.7884540009498596,
180
+ "num_tokens": 2468392.0,
181
+ "step": 400
182
+ },
183
+ {
184
+ "epoch": 0.13848018002423404,
185
+ "eval_entropy": 0.8820633643963298,
186
+ "eval_loss": 0.8462674021720886,
187
+ "eval_mean_token_accuracy": 0.7830698802822926,
188
+ "eval_num_tokens": 2468392.0,
189
+ "eval_runtime": 16.2902,
190
+ "eval_samples_per_second": 298.645,
191
+ "eval_steps_per_second": 18.723,
192
+ "step": 400
193
+ },
194
+ {
195
+ "entropy": 0.8852161896228791,
196
+ "epoch": 0.14713519127574864,
197
+ "grad_norm": 2.25,
198
+ "learning_rate": 1.990195114343008e-05,
199
+ "loss": 0.8604666137695313,
200
+ "mean_token_accuracy": 0.7796429479122162,
201
+ "num_tokens": 2626504.0,
202
+ "step": 425
203
+ },
204
+ {
205
+ "entropy": 0.8595891773700715,
206
+ "epoch": 0.15579020252726328,
207
+ "grad_norm": 2.109375,
208
+ "learning_rate": 1.988140167150716e-05,
209
+ "loss": 0.8375725555419922,
210
+ "mean_token_accuracy": 0.7837563192844391,
211
+ "num_tokens": 2785992.0,
212
+ "step": 450
213
+ },
214
+ {
215
+ "entropy": 0.8787437427043915,
216
+ "epoch": 0.1644452137787779,
217
+ "grad_norm": 2.21875,
218
+ "learning_rate": 1.9858911336256258e-05,
219
+ "loss": 0.8602984619140625,
220
+ "mean_token_accuracy": 0.7801311421394348,
221
+ "num_tokens": 2943389.0,
222
+ "step": 475
223
+ },
224
+ {
225
+ "entropy": 0.8887673962116241,
226
+ "epoch": 0.17310022503029254,
227
+ "grad_norm": 2.34375,
228
+ "learning_rate": 1.983448455513437e-05,
229
+ "loss": 0.8559275054931641,
230
+ "mean_token_accuracy": 0.7799410057067871,
231
+ "num_tokens": 3098910.0,
232
+ "step": 500
233
+ },
234
+ {
235
+ "entropy": 0.8632562351226807,
236
+ "epoch": 0.18175523628180718,
237
+ "grad_norm": 2.359375,
238
+ "learning_rate": 1.9808126125947035e-05,
239
+ "loss": 0.8414566040039062,
240
+ "mean_token_accuracy": 0.7838216722011566,
241
+ "num_tokens": 3259790.0,
242
+ "step": 525
243
+ },
244
+ {
245
+ "entropy": 0.8468562078475952,
246
+ "epoch": 0.19041024753332178,
247
+ "grad_norm": 2.921875,
248
+ "learning_rate": 1.977984122590598e-05,
249
+ "loss": 0.8214312744140625,
250
+ "mean_token_accuracy": 0.7904518485069275,
251
+ "num_tokens": 3408921.0,
252
+ "step": 550
253
+ },
254
+ {
255
+ "entropy": 0.862267234325409,
256
+ "epoch": 0.1990652587848364,
257
+ "grad_norm": 2.21875,
258
+ "learning_rate": 1.9749635410612222e-05,
259
+ "loss": 0.8381607818603516,
260
+ "mean_token_accuracy": 0.7850737714767456,
261
+ "num_tokens": 3564021.0,
262
+ "step": 575
263
+ },
264
+ {
265
+ "entropy": 0.8673117184638977,
266
+ "epoch": 0.20772027003635105,
267
+ "grad_norm": 2.28125,
268
+ "learning_rate": 1.971751461296488e-05,
269
+ "loss": 0.8422865295410156,
270
+ "mean_token_accuracy": 0.7839160299301148,
271
+ "num_tokens": 3720222.0,
272
+ "step": 600
273
+ },
274
+ {
275
+ "epoch": 0.20772027003635105,
276
+ "eval_entropy": 0.8581681211463741,
277
+ "eval_loss": 0.8380036950111389,
278
+ "eval_mean_token_accuracy": 0.7849426593936858,
279
+ "eval_num_tokens": 3720222.0,
280
+ "eval_runtime": 16.2659,
281
+ "eval_samples_per_second": 299.091,
282
+ "eval_steps_per_second": 18.751,
283
+ "step": 600
284
+ },
285
+ {
286
+ "entropy": 0.896509393453598,
287
+ "epoch": 0.21637528128786568,
288
+ "grad_norm": 2.25,
289
+ "learning_rate": 1.9683485141995832e-05,
290
+ "loss": 0.8751831817626953,
291
+ "mean_token_accuracy": 0.778223477602005,
292
+ "num_tokens": 3868288.0,
293
+ "step": 625
294
+ },
295
+ {
296
+ "entropy": 0.8858666527271271,
297
+ "epoch": 0.2250302925393803,
298
+ "grad_norm": 2.4375,
299
+ "learning_rate": 1.964755368163055e-05,
300
+ "loss": 0.8597821807861328,
301
+ "mean_token_accuracy": 0.7796040976047516,
302
+ "num_tokens": 4020387.0,
303
+ "step": 650
304
+ },
305
+ {
306
+ "entropy": 0.8457015335559845,
307
+ "epoch": 0.23368530379089492,
308
+ "grad_norm": 2.0625,
309
+ "learning_rate": 1.9609727289375254e-05,
310
+ "loss": 0.8197212982177734,
311
+ "mean_token_accuracy": 0.7878750622272491,
312
+ "num_tokens": 4175769.0,
313
+ "step": 675
314
+ },
315
+ {
316
+ "entropy": 0.8535351765155792,
317
+ "epoch": 0.24234031504240955,
318
+ "grad_norm": 2.328125,
319
+ "learning_rate": 1.9570013394930704e-05,
320
+ "loss": 0.8300620269775391,
321
+ "mean_token_accuracy": 0.786250706911087,
322
+ "num_tokens": 4331114.0,
323
+ "step": 700
324
+ },
325
+ {
326
+ "entropy": 0.8614680600166321,
327
+ "epoch": 0.2509953262939242,
328
+ "grad_norm": 2.0625,
329
+ "learning_rate": 1.9528419798732902e-05,
330
+ "loss": 0.8407713317871094,
331
+ "mean_token_accuracy": 0.7833027255535125,
332
+ "num_tokens": 4488237.0,
333
+ "step": 725
334
+ },
335
+ {
336
+ "entropy": 0.8501963889598847,
337
+ "epoch": 0.2596503375454388,
338
+ "grad_norm": 2.328125,
339
+ "learning_rate": 1.948495467042095e-05,
340
+ "loss": 0.8218775177001953,
341
+ "mean_token_accuracy": 0.788744968175888,
342
+ "num_tokens": 4642068.0,
343
+ "step": 750
344
+ },
345
+ {
346
+ "entropy": 0.8383999621868133,
347
+ "epoch": 0.26830534879695345,
348
+ "grad_norm": 2.296875,
349
+ "learning_rate": 1.9439626547232433e-05,
350
+ "loss": 0.80978515625,
351
+ "mean_token_accuracy": 0.7903480684757233,
352
+ "num_tokens": 4788314.0,
353
+ "step": 775
354
+ },
355
+ {
356
+ "entropy": 0.8568655729293824,
357
+ "epoch": 0.2769603600484681,
358
+ "grad_norm": 2.53125,
359
+ "learning_rate": 1.939244433232652e-05,
360
+ "loss": 0.8343601226806641,
361
+ "mean_token_accuracy": 0.7855056190490722,
362
+ "num_tokens": 4941131.0,
363
+ "step": 800
364
+ },
365
+ {
366
+ "epoch": 0.2769603600484681,
367
+ "eval_entropy": 0.859573404906226,
368
+ "eval_loss": 0.8307726383209229,
369
+ "eval_mean_token_accuracy": 0.7864908138259512,
370
+ "eval_num_tokens": 4941131.0,
371
+ "eval_runtime": 16.2516,
372
+ "eval_samples_per_second": 299.355,
373
+ "eval_steps_per_second": 18.767,
374
+ "step": 800
375
+ },
376
+ {
377
+ "entropy": 0.8670335674285888,
378
+ "epoch": 0.2856153712999827,
379
+ "grad_norm": 1.9765625,
380
+ "learning_rate": 1.934341729303529e-05,
381
+ "loss": 0.8394245910644531,
382
+ "mean_token_accuracy": 0.7841019070148468,
383
+ "num_tokens": 5096940.0,
384
+ "step": 825
385
+ },
386
+ {
387
+ "entropy": 0.867160736322403,
388
+ "epoch": 0.2942703825514973,
389
+ "grad_norm": 2.21875,
390
+ "learning_rate": 1.9292555059043457e-05,
391
+ "loss": 0.8409418487548828,
392
+ "mean_token_accuracy": 0.7830280113220215,
393
+ "num_tokens": 5250616.0,
394
+ "step": 850
395
+ },
396
+ {
397
+ "entropy": 0.8582142984867096,
398
+ "epoch": 0.3029253938030119,
399
+ "grad_norm": 2.15625,
400
+ "learning_rate": 1.9239867620496952e-05,
401
+ "loss": 0.8346880340576172,
402
+ "mean_token_accuracy": 0.783586630821228,
403
+ "num_tokens": 5408253.0,
404
+ "step": 875
405
+ },
406
+ {
407
+ "entropy": 0.84959406375885,
408
+ "epoch": 0.31158040505452655,
409
+ "grad_norm": 2.1875,
410
+ "learning_rate": 1.9185365326040713e-05,
411
+ "loss": 0.826163330078125,
412
+ "mean_token_accuracy": 0.7881129395961761,
413
+ "num_tokens": 5564921.0,
414
+ "step": 900
415
+ },
416
+ {
417
+ "entropy": 0.8895779585838318,
418
+ "epoch": 0.3202354163060412,
419
+ "grad_norm": 2.1875,
420
+ "learning_rate": 1.9129058880786026e-05,
421
+ "loss": 0.866309585571289,
422
+ "mean_token_accuracy": 0.7782393181324005,
423
+ "num_tokens": 5722704.0,
424
+ "step": 925
425
+ },
426
+ {
427
+ "entropy": 0.8372803354263305,
428
+ "epoch": 0.3288904275575558,
429
+ "grad_norm": 1.890625,
430
+ "learning_rate": 1.907095934420789e-05,
431
+ "loss": 0.8083054351806641,
432
+ "mean_token_accuracy": 0.7903852713108063,
433
+ "num_tokens": 5881153.0,
434
+ "step": 950
435
+ },
436
+ {
437
+ "entropy": 0.8762626206874847,
438
+ "epoch": 0.33754543880907045,
439
+ "grad_norm": 1.96875,
440
+ "learning_rate": 1.9011078127972748e-05,
441
+ "loss": 0.8495908355712891,
442
+ "mean_token_accuracy": 0.7841223049163818,
443
+ "num_tokens": 6039323.0,
444
+ "step": 975
445
+ },
446
+ {
447
+ "entropy": 0.8670505321025849,
448
+ "epoch": 0.3462004500605851,
449
+ "grad_norm": 2.25,
450
+ "learning_rate": 1.8949426993697074e-05,
451
+ "loss": 0.8448488616943359,
452
+ "mean_token_accuracy": 0.785959769487381,
453
+ "num_tokens": 6197683.0,
454
+ "step": 1000
455
+ },
456
+ {
457
+ "epoch": 0.3462004500605851,
458
+ "eval_entropy": 0.8536441159052927,
459
+ "eval_loss": 0.8251546025276184,
460
+ "eval_mean_token_accuracy": 0.7873384729760592,
461
+ "eval_num_tokens": 6197683.0,
462
+ "eval_runtime": 16.2591,
463
+ "eval_samples_per_second": 299.217,
464
+ "eval_steps_per_second": 18.759,
465
+ "step": 1000
466
+ },
467
+ {
468
+ "entropy": 0.8527129173278809,
469
+ "epoch": 0.3548554613120997,
470
+ "grad_norm": 2.171875,
471
+ "learning_rate": 1.888601805063719e-05,
472
+ "loss": 0.8221394348144532,
473
+ "mean_token_accuracy": 0.7878592002391815,
474
+ "num_tokens": 6353591.0,
475
+ "step": 1025
476
+ },
477
+ {
478
+ "entropy": 0.8309027600288391,
479
+ "epoch": 0.36351047256361435,
480
+ "grad_norm": 2.1875,
481
+ "learning_rate": 1.8820863753310812e-05,
482
+ "loss": 0.804949951171875,
483
+ "mean_token_accuracy": 0.7907659840583802,
484
+ "num_tokens": 6503522.0,
485
+ "step": 1050
486
+ },
487
+ {
488
+ "entropy": 0.8664905667304993,
489
+ "epoch": 0.372165483815129,
490
+ "grad_norm": 2.234375,
491
+ "learning_rate": 1.875397689905081e-05,
492
+ "loss": 0.8386592102050782,
493
+ "mean_token_accuracy": 0.783461629152298,
494
+ "num_tokens": 6657038.0,
495
+ "step": 1075
496
+ },
497
+ {
498
+ "entropy": 0.8521995890140533,
499
+ "epoch": 0.38082049506664356,
500
+ "grad_norm": 2.140625,
501
+ "learning_rate": 1.8685370625491582e-05,
502
+ "loss": 0.8274839782714843,
503
+ "mean_token_accuracy": 0.7857646262645721,
504
+ "num_tokens": 6813403.0,
505
+ "step": 1100
506
+ },
507
+ {
508
+ "entropy": 0.85324840426445,
509
+ "epoch": 0.3894755063181582,
510
+ "grad_norm": 2.03125,
511
+ "learning_rate": 1.861505840798864e-05,
512
+ "loss": 0.8251450347900391,
513
+ "mean_token_accuracy": 0.7872388279438018,
514
+ "num_tokens": 6969337.0,
515
+ "step": 1125
516
+ },
517
+ {
518
+ "entropy": 0.8578063642978668,
519
+ "epoch": 0.3981305175696728,
520
+ "grad_norm": 2.09375,
521
+ "learning_rate": 1.8543054056971815e-05,
522
+ "loss": 0.834039077758789,
523
+ "mean_token_accuracy": 0.7848770570755005,
524
+ "num_tokens": 7124334.0,
525
+ "step": 1150
526
+ },
527
+ {
528
+ "entropy": 0.8491312241554261,
529
+ "epoch": 0.40678552882118746,
530
+ "grad_norm": 2.0625,
531
+ "learning_rate": 1.8469371715232686e-05,
532
+ "loss": 0.8220222473144532,
533
+ "mean_token_accuracy": 0.7871887254714965,
534
+ "num_tokens": 7277475.0,
535
+ "step": 1175
536
+ },
537
+ {
538
+ "entropy": 0.8443890881538391,
539
+ "epoch": 0.4154405400727021,
540
+ "grad_norm": 2.203125,
541
+ "learning_rate": 1.839402585514671e-05,
542
+ "loss": 0.8166568756103516,
543
+ "mean_token_accuracy": 0.7897883796691895,
544
+ "num_tokens": 7428616.0,
545
+ "step": 1200
546
+ },
547
+ {
548
+ "epoch": 0.4154405400727021,
549
+ "eval_entropy": 0.8290919256014903,
550
+ "eval_loss": 0.8211944103240967,
551
+ "eval_mean_token_accuracy": 0.788363546621604,
552
+ "eval_num_tokens": 7428616.0,
553
+ "eval_runtime": 16.2451,
554
+ "eval_samples_per_second": 299.474,
555
+ "eval_steps_per_second": 18.775,
556
+ "step": 1200
557
+ },
558
+ {
559
+ "entropy": 0.8553554141521453,
560
+ "epoch": 0.4240955513242167,
561
+ "grad_norm": 2.140625,
562
+ "learning_rate": 1.8317031275830607e-05,
563
+ "loss": 0.8335649108886719,
564
+ "mean_token_accuracy": 0.7848791778087616,
565
+ "num_tokens": 7586653.0,
566
+ "step": 1225
567
+ },
568
+ {
569
+ "entropy": 0.8513217878341675,
570
+ "epoch": 0.43275056257573136,
571
+ "grad_norm": 2.078125,
572
+ "learning_rate": 1.8238403100235602e-05,
573
+ "loss": 0.8226930999755859,
574
+ "mean_token_accuracy": 0.7892848730087281,
575
+ "num_tokens": 7736619.0,
576
+ "step": 1250
577
+ },
578
+ {
579
+ "entropy": 0.8567298305034637,
580
+ "epoch": 0.441405573827246,
581
+ "grad_norm": 2.25,
582
+ "learning_rate": 1.8158156772177015e-05,
583
+ "loss": 0.8301739501953125,
584
+ "mean_token_accuracy": 0.7857687485218048,
585
+ "num_tokens": 7895674.0,
586
+ "step": 1275
587
+ },
588
+ {
589
+ "entropy": 0.8383160126209259,
590
+ "epoch": 0.4500605850787606,
591
+ "grad_norm": 2.28125,
592
+ "learning_rate": 1.807630805330087e-05,
593
+ "loss": 0.8143462371826172,
594
+ "mean_token_accuracy": 0.7904347920417786,
595
+ "num_tokens": 8052368.0,
596
+ "step": 1300
597
+ },
598
+ {
599
+ "entropy": 0.858016027212143,
600
+ "epoch": 0.45871559633027525,
601
+ "grad_norm": 2.328125,
602
+ "learning_rate": 1.7992873019988044e-05,
603
+ "loss": 0.8361105346679687,
604
+ "mean_token_accuracy": 0.7837494373321533,
605
+ "num_tokens": 8209037.0,
606
+ "step": 1325
607
+ },
608
+ {
609
+ "entropy": 0.8485116994380951,
610
+ "epoch": 0.46737060758178983,
611
+ "grad_norm": 2.078125,
612
+ "learning_rate": 1.7907868060196627e-05,
613
+ "loss": 0.8189002990722656,
614
+ "mean_token_accuracy": 0.7860724258422852,
615
+ "num_tokens": 8366536.0,
616
+ "step": 1350
617
+ },
618
+ {
619
+ "entropy": 0.8689926111698151,
620
+ "epoch": 0.47602561883330446,
621
+ "grad_norm": 2.28125,
622
+ "learning_rate": 1.7821309870243056e-05,
623
+ "loss": 0.8450764465332031,
624
+ "mean_token_accuracy": 0.7817527639865876,
625
+ "num_tokens": 8524698.0,
626
+ "step": 1375
627
+ },
628
+ {
629
+ "entropy": 0.8510251712799072,
630
+ "epoch": 0.4846806300848191,
631
+ "grad_norm": 2.03125,
632
+ "learning_rate": 1.77332154515227e-05,
633
+ "loss": 0.8352996063232422,
634
+ "mean_token_accuracy": 0.786122510433197,
635
+ "num_tokens": 8680719.0,
636
+ "step": 1400
637
+ },
638
+ {
639
+ "epoch": 0.4846806300848191,
640
+ "eval_entropy": 0.8555903234442727,
641
+ "eval_loss": 0.8170819878578186,
642
+ "eval_mean_token_accuracy": 0.7890570513537672,
643
+ "eval_num_tokens": 8680719.0,
644
+ "eval_runtime": 16.2534,
645
+ "eval_samples_per_second": 299.322,
646
+ "eval_steps_per_second": 18.765,
647
+ "step": 1400
648
+ },
649
+ {
650
+ "entropy": 0.8443090248107911,
651
+ "epoch": 0.49333564133633373,
652
+ "grad_norm": 2.34375,
653
+ "learning_rate": 1.764360210717051e-05,
654
+ "loss": 0.8160585784912109,
655
+ "mean_token_accuracy": 0.7888865935802459,
656
+ "num_tokens": 8839183.0,
657
+ "step": 1425
658
+ },
659
+ {
660
+ "entropy": 0.8397502684593201,
661
+ "epoch": 0.5019906525878484,
662
+ "grad_norm": 2.015625,
663
+ "learning_rate": 1.7552487438662432e-05,
664
+ "loss": 0.8148053741455078,
665
+ "mean_token_accuracy": 0.7885977244377136,
666
+ "num_tokens": 8995586.0,
667
+ "step": 1450
668
+ },
669
+ {
670
+ "entropy": 0.8441075325012207,
671
+ "epoch": 0.5106456638393629,
672
+ "grad_norm": 2.34375,
673
+ "learning_rate": 1.7459889342358176e-05,
674
+ "loss": 0.8217481994628906,
675
+ "mean_token_accuracy": 0.7888561117649079,
676
+ "num_tokens": 9146595.0,
677
+ "step": 1475
678
+ },
679
+ {
680
+ "entropy": 0.8394048309326172,
681
+ "epoch": 0.5193006750908776,
682
+ "grad_norm": 2.296875,
683
+ "learning_rate": 1.736582600598609e-05,
684
+ "loss": 0.8105857849121094,
685
+ "mean_token_accuracy": 0.7910154783725738,
686
+ "num_tokens": 9297789.0,
687
+ "step": 1500
688
+ },
689
+ {
690
+ "entropy": 0.8256420779228211,
691
+ "epoch": 0.5279556863423922,
692
+ "grad_norm": 2.140625,
693
+ "learning_rate": 1.7270315905070822e-05,
694
+ "loss": 0.7967173004150391,
695
+ "mean_token_accuracy": 0.7917508590221405,
696
+ "num_tokens": 9450532.0,
697
+ "step": 1525
698
+ },
699
+ {
700
+ "entropy": 0.817123122215271,
701
+ "epoch": 0.5366106975939069,
702
+ "grad_norm": 1.8828125,
703
+ "learning_rate": 1.717337779930441e-05,
704
+ "loss": 0.7964045715332031,
705
+ "mean_token_accuracy": 0.793145546913147,
706
+ "num_tokens": 9605286.0,
707
+ "step": 1550
708
+ },
709
+ {
710
+ "entropy": 0.8218163585662842,
711
+ "epoch": 0.5452657088454215,
712
+ "grad_norm": 2.25,
713
+ "learning_rate": 1.7075030728861602e-05,
714
+ "loss": 0.7938800048828125,
715
+ "mean_token_accuracy": 0.7939867222309113,
716
+ "num_tokens": 9755402.0,
717
+ "step": 1575
718
+ },
719
+ {
720
+ "entropy": 0.8473324692249298,
721
+ "epoch": 0.5539207200969362,
722
+ "grad_norm": 2.171875,
723
+ "learning_rate": 1.697529401066006e-05,
724
+ "loss": 0.8210481262207031,
725
+ "mean_token_accuracy": 0.7879379904270172,
726
+ "num_tokens": 9915397.0,
727
+ "step": 1600
728
+ },
729
+ {
730
+ "epoch": 0.5539207200969362,
731
+ "eval_entropy": 0.8279588664164309,
732
+ "eval_loss": 0.8136925101280212,
733
+ "eval_mean_token_accuracy": 0.7897968792524495,
734
+ "eval_num_tokens": 9915397.0,
735
+ "eval_runtime": 16.2401,
736
+ "eval_samples_per_second": 299.567,
737
+ "eval_steps_per_second": 18.781,
738
+ "step": 1600
739
+ },
740
+ {
741
+ "entropy": 0.83288947224617,
742
+ "epoch": 0.5625757313484507,
743
+ "grad_norm": 2.125,
744
+ "learning_rate": 1.6874187234566198e-05,
745
+ "loss": 0.8104652404785156,
746
+ "mean_token_accuracy": 0.7881759464740753,
747
+ "num_tokens": 10066176.0,
748
+ "step": 1625
749
+ },
750
+ {
751
+ "entropy": 0.8406240797042847,
752
+ "epoch": 0.5712307425999654,
753
+ "grad_norm": 2.21875,
754
+ "learning_rate": 1.677173025954745e-05,
755
+ "loss": 0.8155978393554687,
756
+ "mean_token_accuracy": 0.7885338914394379,
757
+ "num_tokens": 10216630.0,
758
+ "step": 1650
759
+ },
760
+ {
761
+ "entropy": 0.8361802196502686,
762
+ "epoch": 0.57988575385148,
763
+ "grad_norm": 2.078125,
764
+ "learning_rate": 1.6667943209771634e-05,
765
+ "loss": 0.8124593353271484,
766
+ "mean_token_accuracy": 0.7909375095367431,
767
+ "num_tokens": 10368775.0,
768
+ "step": 1675
769
+ },
770
+ {
771
+ "entropy": 0.8546354687213897,
772
+ "epoch": 0.5885407651029946,
773
+ "grad_norm": 2.390625,
774
+ "learning_rate": 1.6562846470654253e-05,
775
+ "loss": 0.8320878601074219,
776
+ "mean_token_accuracy": 0.7862278306484223,
777
+ "num_tokens": 10522306.0,
778
+ "step": 1700
779
+ },
780
+ {
781
+ "entropy": 0.8271597623825073,
782
+ "epoch": 0.5971957763545093,
783
+ "grad_norm": 2.359375,
784
+ "learning_rate": 1.6456460684854474e-05,
785
+ "loss": 0.802950210571289,
786
+ "mean_token_accuracy": 0.7914960932731628,
787
+ "num_tokens": 10677278.0,
788
+ "step": 1725
789
+ },
790
+ {
791
+ "entropy": 0.8680684280395508,
792
+ "epoch": 0.6058507876060238,
793
+ "grad_norm": 2.21875,
794
+ "learning_rate": 1.6348806748220596e-05,
795
+ "loss": 0.8468040466308594,
796
+ "mean_token_accuracy": 0.7825022864341736,
797
+ "num_tokens": 10832759.0,
798
+ "step": 1750
799
+ },
800
+ {
801
+ "entropy": 0.8460379076004029,
802
+ "epoch": 0.6145057988575385,
803
+ "grad_norm": 1.90625,
804
+ "learning_rate": 1.6239905805685766e-05,
805
+ "loss": 0.81728515625,
806
+ "mean_token_accuracy": 0.7885628819465638,
807
+ "num_tokens": 10987982.0,
808
+ "step": 1775
809
+ },
810
+ {
811
+ "entropy": 0.8258041095733643,
812
+ "epoch": 0.6231608101090531,
813
+ "grad_norm": 1.9609375,
814
+ "learning_rate": 1.6129779247114776e-05,
815
+ "loss": 0.8052913665771484,
816
+ "mean_token_accuracy": 0.7923011314868927,
817
+ "num_tokens": 11142409.0,
818
+ "step": 1800
819
+ },
820
+ {
821
+ "epoch": 0.6231608101090531,
822
+ "eval_entropy": 0.8428774887421092,
823
+ "eval_loss": 0.8103926181793213,
824
+ "eval_mean_token_accuracy": 0.790385250967057,
825
+ "eval_num_tokens": 11142409.0,
826
+ "eval_runtime": 16.2616,
827
+ "eval_samples_per_second": 299.17,
828
+ "eval_steps_per_second": 18.756,
829
+ "step": 1800
830
+ },
831
+ {
832
+ "entropy": 0.8314215159416198,
833
+ "epoch": 0.6318158213605678,
834
+ "grad_norm": 2.09375,
835
+ "learning_rate": 1.601844870310277e-05,
836
+ "loss": 0.8011739349365234,
837
+ "mean_token_accuracy": 0.7905190980434418,
838
+ "num_tokens": 11295629.0,
839
+ "step": 1825
840
+ },
841
+ {
842
+ "entropy": 0.8174971842765808,
843
+ "epoch": 0.6404708326120824,
844
+ "grad_norm": 2.03125,
845
+ "learning_rate": 1.5905936040726636e-05,
846
+ "loss": 0.7869554138183594,
847
+ "mean_token_accuracy": 0.7940086150169372,
848
+ "num_tokens": 11449328.0,
849
+ "step": 1850
850
+ },
851
+ {
852
+ "entropy": 0.8386989748477935,
853
+ "epoch": 0.6491258438635971,
854
+ "grad_norm": 2.03125,
855
+ "learning_rate": 1.579226335924999e-05,
856
+ "loss": 0.8210865783691407,
857
+ "mean_token_accuracy": 0.7893707096576691,
858
+ "num_tokens": 11605530.0,
859
+ "step": 1875
860
+ },
861
+ {
862
+ "entropy": 0.8110369992256165,
863
+ "epoch": 0.6577808551151116,
864
+ "grad_norm": 2.3125,
865
+ "learning_rate": 1.5677452985782493e-05,
866
+ "loss": 0.7894587707519531,
867
+ "mean_token_accuracy": 0.7973484849929809,
868
+ "num_tokens": 11760213.0,
869
+ "step": 1900
870
+ },
871
+ {
872
+ "entropy": 0.8280953919887543,
873
+ "epoch": 0.6664358663666263,
874
+ "grad_norm": 2.203125,
875
+ "learning_rate": 1.5561527470894503e-05,
876
+ "loss": 0.7997794342041016,
877
+ "mean_token_accuracy": 0.7926359629631042,
878
+ "num_tokens": 11913380.0,
879
+ "step": 1925
880
+ },
881
+ {
882
+ "entropy": 0.8196328341960907,
883
+ "epoch": 0.6750908776181409,
884
+ "grad_norm": 2.171875,
885
+ "learning_rate": 1.544450958418775e-05,
886
+ "loss": 0.7956825256347656,
887
+ "mean_token_accuracy": 0.7946672177314759,
888
+ "num_tokens": 12065793.0,
889
+ "step": 1950
890
+ },
891
+ {
892
+ "entropy": 0.8513234782218934,
893
+ "epoch": 0.6837458888696555,
894
+ "grad_norm": 1.8671875,
895
+ "learning_rate": 1.5326422309823046e-05,
896
+ "loss": 0.8301435089111329,
897
+ "mean_token_accuracy": 0.7851076364517212,
898
+ "num_tokens": 12229815.0,
899
+ "step": 1975
900
+ },
901
+ {
902
+ "entropy": 0.8259239780902863,
903
+ "epoch": 0.6924009001211702,
904
+ "grad_norm": 2.078125,
905
+ "learning_rate": 1.5207288842005813e-05,
906
+ "loss": 0.8005783081054687,
907
+ "mean_token_accuracy": 0.79151895403862,
908
+ "num_tokens": 12391391.0,
909
+ "step": 2000
910
+ },
911
+ {
912
+ "epoch": 0.6924009001211702,
913
+ "eval_entropy": 0.828711828149733,
914
+ "eval_loss": 0.8071080446243286,
915
+ "eval_mean_token_accuracy": 0.7911858433582744,
916
+ "eval_num_tokens": 12391391.0,
917
+ "eval_runtime": 16.2568,
918
+ "eval_samples_per_second": 299.259,
919
+ "eval_steps_per_second": 18.761,
920
+ "step": 2000
921
+ },
922
+ {
923
+ "entropy": 0.8361215782165528,
924
+ "epoch": 0.7010559113726847,
925
+ "grad_norm": 2.109375,
926
+ "learning_rate": 1.508713258043041e-05,
927
+ "loss": 0.8149121856689453,
928
+ "mean_token_accuracy": 0.7890836715698242,
929
+ "num_tokens": 12548530.0,
930
+ "step": 2025
931
+ },
932
+ {
933
+ "entropy": 0.8404046738147736,
934
+ "epoch": 0.7097109226241994,
935
+ "grad_norm": 2.0,
936
+ "learning_rate": 1.4965977125684018e-05,
937
+ "loss": 0.814455337524414,
938
+ "mean_token_accuracy": 0.7903144693374634,
939
+ "num_tokens": 12703495.0,
940
+ "step": 2050
941
+ },
942
+ {
943
+ "entropy": 0.8286141097545624,
944
+ "epoch": 0.718365933875714,
945
+ "grad_norm": 2.296875,
946
+ "learning_rate": 1.484384627461115e-05,
947
+ "loss": 0.8082135009765625,
948
+ "mean_token_accuracy": 0.7908019495010375,
949
+ "num_tokens": 12856585.0,
950
+ "step": 2075
951
+ },
952
+ {
953
+ "entropy": 0.8540136122703552,
954
+ "epoch": 0.7270209451272287,
955
+ "grad_norm": 2.09375,
956
+ "learning_rate": 1.4720764015639555e-05,
957
+ "loss": 0.8297102355957031,
958
+ "mean_token_accuracy": 0.7875827658176422,
959
+ "num_tokens": 13011641.0,
960
+ "step": 2100
961
+ },
962
+ {
963
+ "entropy": 0.8180252170562744,
964
+ "epoch": 0.7356759563787433,
965
+ "grad_norm": 2.171875,
966
+ "learning_rate": 1.4596754524068515e-05,
967
+ "loss": 0.791978988647461,
968
+ "mean_token_accuracy": 0.7945426023006439,
969
+ "num_tokens": 13167258.0,
970
+ "step": 2125
971
+ },
972
+ {
973
+ "entropy": 0.8130170261859894,
974
+ "epoch": 0.744330967630258,
975
+ "grad_norm": 2.1875,
976
+ "learning_rate": 1.4471842157320443e-05,
977
+ "loss": 0.7898551940917968,
978
+ "mean_token_accuracy": 0.7971277523040772,
979
+ "num_tokens": 13315266.0,
980
+ "step": 2150
981
+ },
982
+ {
983
+ "entropy": 0.8508910453319549,
984
+ "epoch": 0.7529859788817725,
985
+ "grad_norm": 2.140625,
986
+ "learning_rate": 1.4346051450156689e-05,
987
+ "loss": 0.8204324340820313,
988
+ "mean_token_accuracy": 0.7856499540805817,
989
+ "num_tokens": 13473686.0,
990
+ "step": 2175
991
+ },
992
+ {
993
+ "entropy": 0.8430261933803558,
994
+ "epoch": 0.7616409901332871,
995
+ "grad_norm": 2.125,
996
+ "learning_rate": 1.4219407109858532e-05,
997
+ "loss": 0.8170342254638672,
998
+ "mean_token_accuracy": 0.7883759355545044,
999
+ "num_tokens": 13627766.0,
1000
+ "step": 2200
1001
+ },
1002
+ {
1003
+ "epoch": 0.7616409901332871,
1004
+ "eval_entropy": 0.8279429495334625,
1005
+ "eval_loss": 0.8048078417778015,
1006
+ "eval_mean_token_accuracy": 0.7913855454960808,
1007
+ "eval_num_tokens": 13627766.0,
1008
+ "eval_runtime": 16.2774,
1009
+ "eval_samples_per_second": 298.88,
1010
+ "eval_steps_per_second": 18.738,
1011
+ "step": 2200
1012
+ },
1013
+ {
1014
+ "entropy": 0.8365470349788666,
1015
+ "epoch": 0.7702960013848018,
1016
+ "grad_norm": 2.3125,
1017
+ "learning_rate": 1.4091934011374281e-05,
1018
+ "loss": 0.8083656311035157,
1019
+ "mean_token_accuracy": 0.7891746735572815,
1020
+ "num_tokens": 13781809.0,
1021
+ "step": 2225
1022
+ },
1023
+ {
1024
+ "entropy": 0.8262781822681426,
1025
+ "epoch": 0.7789510126363164,
1026
+ "grad_norm": 2.265625,
1027
+ "learning_rate": 1.396365719243344e-05,
1028
+ "loss": 0.8065071868896484,
1029
+ "mean_token_accuracy": 0.7918438529968261,
1030
+ "num_tokens": 13937124.0,
1031
+ "step": 2250
1032
+ },
1033
+ {
1034
+ "entropy": 0.8354354929924012,
1035
+ "epoch": 0.7876060238878311,
1036
+ "grad_norm": 2.03125,
1037
+ "learning_rate": 1.3834601848628897e-05,
1038
+ "loss": 0.8083207702636719,
1039
+ "mean_token_accuracy": 0.7916932463645935,
1040
+ "num_tokens": 14088345.0,
1041
+ "step": 2275
1042
+ },
1043
+ {
1044
+ "entropy": 0.8217170929908753,
1045
+ "epoch": 0.7962610351393457,
1046
+ "grad_norm": 2.203125,
1047
+ "learning_rate": 1.3704793328468122e-05,
1048
+ "loss": 0.7991712951660156,
1049
+ "mean_token_accuracy": 0.7924394917488098,
1050
+ "num_tokens": 14241089.0,
1051
+ "step": 2300
1052
+ },
1053
+ {
1054
+ "entropy": 0.8267798566818237,
1055
+ "epoch": 0.8049160463908603,
1056
+ "grad_norm": 2.09375,
1057
+ "learning_rate": 1.3574257128394315e-05,
1058
+ "loss": 0.8018778228759765,
1059
+ "mean_token_accuracy": 0.7922985780239106,
1060
+ "num_tokens": 14392337.0,
1061
+ "step": 2325
1062
+ },
1063
+ {
1064
+ "entropy": 0.8261617541313171,
1065
+ "epoch": 0.8135710576423749,
1066
+ "grad_norm": 2.171875,
1067
+ "learning_rate": 1.3443018887778494e-05,
1068
+ "loss": 0.8028769683837891,
1069
+ "mean_token_accuracy": 0.7926994669437408,
1070
+ "num_tokens": 14544382.0,
1071
+ "step": 2350
1072
+ },
1073
+ {
1074
+ "entropy": 0.8380094921588898,
1075
+ "epoch": 0.8222260688938896,
1076
+ "grad_norm": 2.3125,
1077
+ "learning_rate": 1.3311104383883537e-05,
1078
+ "loss": 0.8226547241210938,
1079
+ "mean_token_accuracy": 0.7886342179775238,
1080
+ "num_tokens": 14703460.0,
1081
+ "step": 2375
1082
+ },
1083
+ {
1084
+ "entropy": 0.8281042277812958,
1085
+ "epoch": 0.8308810801454042,
1086
+ "grad_norm": 2.296875,
1087
+ "learning_rate": 1.3178539526801106e-05,
1088
+ "loss": 0.8025864410400391,
1089
+ "mean_token_accuracy": 0.7910430765151978,
1090
+ "num_tokens": 14860294.0,
1091
+ "step": 2400
1092
+ },
1093
+ {
1094
+ "epoch": 0.8308810801454042,
1095
+ "eval_entropy": 0.8327872855741469,
1096
+ "eval_loss": 0.8026647567749023,
1097
+ "eval_mean_token_accuracy": 0.7919463300314106,
1098
+ "eval_num_tokens": 14860294.0,
1099
+ "eval_runtime": 16.2486,
1100
+ "eval_samples_per_second": 299.41,
1101
+ "eval_steps_per_second": 18.771,
1102
+ "step": 2400
1103
+ },
1104
+ {
1105
+ "entropy": 0.811096111536026,
1106
+ "epoch": 0.8395360913969189,
1107
+ "grad_norm": 2.4375,
1108
+ "learning_rate": 1.3045350354362495e-05,
1109
+ "loss": 0.7822922515869141,
1110
+ "mean_token_accuracy": 0.7949872577190399,
1111
+ "num_tokens": 15015328.0,
1112
+ "step": 2425
1113
+ },
1114
+ {
1115
+ "entropy": 0.8189793074131012,
1116
+ "epoch": 0.8481911026484334,
1117
+ "grad_norm": 2.3125,
1118
+ "learning_rate": 1.2911563027024392e-05,
1119
+ "loss": 0.8003229522705078,
1120
+ "mean_token_accuracy": 0.7943521749973297,
1121
+ "num_tokens": 15170350.0,
1122
+ "step": 2450
1123
+ },
1124
+ {
1125
+ "entropy": 0.8213041162490845,
1126
+ "epoch": 0.856846113899948,
1127
+ "grad_norm": 2.21875,
1128
+ "learning_rate": 1.2777203822730557e-05,
1129
+ "loss": 0.7970082092285157,
1130
+ "mean_token_accuracy": 0.7927951776981353,
1131
+ "num_tokens": 15324153.0,
1132
+ "step": 2475
1133
+ },
1134
+ {
1135
+ "entropy": 0.824720960855484,
1136
+ "epoch": 0.8655011251514627,
1137
+ "grad_norm": 2.125,
1138
+ "learning_rate": 1.2642299131750417e-05,
1139
+ "loss": 0.7908332824707032,
1140
+ "mean_token_accuracy": 0.7957222282886505,
1141
+ "num_tokens": 15473303.0,
1142
+ "step": 2500
1143
+ },
1144
+ {
1145
+ "entropy": 0.838714919090271,
1146
+ "epoch": 0.8741561364029773,
1147
+ "grad_norm": 2.09375,
1148
+ "learning_rate": 1.250687545149559e-05,
1149
+ "loss": 0.8137520599365234,
1150
+ "mean_token_accuracy": 0.7890112113952636,
1151
+ "num_tokens": 15629017.0,
1152
+ "step": 2525
1153
+ },
1154
+ {
1155
+ "entropy": 0.8311066961288452,
1156
+ "epoch": 0.882811147654492,
1157
+ "grad_norm": 2.1875,
1158
+ "learning_rate": 1.2370959381315393e-05,
1159
+ "loss": 0.8034115600585937,
1160
+ "mean_token_accuracy": 0.7926496362686157,
1161
+ "num_tokens": 15777876.0,
1162
+ "step": 2550
1163
+ },
1164
+ {
1165
+ "entropy": 0.8097344720363617,
1166
+ "epoch": 0.8914661589060066,
1167
+ "grad_norm": 2.203125,
1168
+ "learning_rate": 1.2234577617272282e-05,
1169
+ "loss": 0.786445083618164,
1170
+ "mean_token_accuracy": 0.7966820681095124,
1171
+ "num_tokens": 15929184.0,
1172
+ "step": 2575
1173
+ },
1174
+ {
1175
+ "entropy": 0.8254410099983215,
1176
+ "epoch": 0.9001211701575212,
1177
+ "grad_norm": 2.296875,
1178
+ "learning_rate": 1.209775694689837e-05,
1179
+ "loss": 0.8023792266845703,
1180
+ "mean_token_accuracy": 0.7923547208309174,
1181
+ "num_tokens": 16087411.0,
1182
+ "step": 2600
1183
+ },
1184
+ {
1185
+ "epoch": 0.9001211701575212,
1186
+ "eval_entropy": 0.8213896471945966,
1187
+ "eval_loss": 0.8008216619491577,
1188
+ "eval_mean_token_accuracy": 0.7924871415388389,
1189
+ "eval_num_tokens": 16087411.0,
1190
+ "eval_runtime": 16.2722,
1191
+ "eval_samples_per_second": 298.976,
1192
+ "eval_steps_per_second": 18.744,
1193
+ "step": 2600
1194
+ },
1195
+ {
1196
+ "entropy": 0.8255444514751434,
1197
+ "epoch": 0.9087761814090358,
1198
+ "grad_norm": 1.9140625,
1199
+ "learning_rate": 1.1960524243933875e-05,
1200
+ "loss": 0.8017765808105469,
1201
+ "mean_token_accuracy": 0.7921687912940979,
1202
+ "num_tokens": 16244042.0,
1203
+ "step": 2625
1204
+ },
1205
+ {
1206
+ "entropy": 0.8252915942668915,
1207
+ "epoch": 0.9174311926605505,
1208
+ "grad_norm": 2.21875,
1209
+ "learning_rate": 1.182290646304874e-05,
1210
+ "loss": 0.8029595947265625,
1211
+ "mean_token_accuracy": 0.7924495422840119,
1212
+ "num_tokens": 16399768.0,
1213
+ "step": 2650
1214
+ },
1215
+ {
1216
+ "entropy": 0.8625216126441956,
1217
+ "epoch": 0.9260862039120651,
1218
+ "grad_norm": 2.171875,
1219
+ "learning_rate": 1.1684930634548287e-05,
1220
+ "loss": 0.838686752319336,
1221
+ "mean_token_accuracy": 0.7851159811019898,
1222
+ "num_tokens": 16557371.0,
1223
+ "step": 2675
1224
+ },
1225
+ {
1226
+ "entropy": 0.8215836095809936,
1227
+ "epoch": 0.9347412151635797,
1228
+ "grad_norm": 2.265625,
1229
+ "learning_rate": 1.1546623859064053e-05,
1230
+ "loss": 0.7977464294433594,
1231
+ "mean_token_accuracy": 0.7929444241523743,
1232
+ "num_tokens": 16713045.0,
1233
+ "step": 2700
1234
+ },
1235
+ {
1236
+ "entropy": 0.8286124646663666,
1237
+ "epoch": 0.9433962264150944,
1238
+ "grad_norm": 2.078125,
1239
+ "learning_rate": 1.1408013302230765e-05,
1240
+ "loss": 0.802153091430664,
1241
+ "mean_token_accuracy": 0.7924544358253479,
1242
+ "num_tokens": 16867145.0,
1243
+ "step": 2725
1244
+ },
1245
+ {
1246
+ "entropy": 0.8237406086921691,
1247
+ "epoch": 0.9520512376666089,
1248
+ "grad_norm": 2.15625,
1249
+ "learning_rate": 1.126912618935063e-05,
1250
+ "loss": 0.8021310424804687,
1251
+ "mean_token_accuracy": 0.7944400310516357,
1252
+ "num_tokens": 17014455.0,
1253
+ "step": 2750
1254
+ },
1255
+ {
1256
+ "entropy": 0.826482676267624,
1257
+ "epoch": 0.9607062489181236,
1258
+ "grad_norm": 2.140625,
1259
+ "learning_rate": 1.1129989800045817e-05,
1260
+ "loss": 0.8052169036865234,
1261
+ "mean_token_accuracy": 0.7911915636062622,
1262
+ "num_tokens": 17175484.0,
1263
+ "step": 2775
1264
+ },
1265
+ {
1266
+ "entropy": 0.8201354122161866,
1267
+ "epoch": 0.9693612601696382,
1268
+ "grad_norm": 2.421875,
1269
+ "learning_rate": 1.099063146290032e-05,
1270
+ "loss": 0.7985061645507813,
1271
+ "mean_token_accuracy": 0.7938509380817413,
1272
+ "num_tokens": 17334068.0,
1273
+ "step": 2800
1274
+ },
1275
+ {
1276
+ "epoch": 0.9693612601696382,
1277
+ "eval_entropy": 0.8281321289109402,
1278
+ "eval_loss": 0.7994962334632874,
1279
+ "eval_mean_token_accuracy": 0.792740264681519,
1280
+ "eval_num_tokens": 17334068.0,
1281
+ "eval_runtime": 16.2806,
1282
+ "eval_samples_per_second": 298.822,
1283
+ "eval_steps_per_second": 18.734,
1284
+ "step": 2800
1285
+ },
1286
+ {
1287
+ "entropy": 0.8046394038200378,
1288
+ "epoch": 0.9780162714211529,
1289
+ "grad_norm": 2.46875,
1290
+ "learning_rate": 1.08510785500922e-05,
1291
+ "loss": 0.7775811004638672,
1292
+ "mean_token_accuracy": 0.7966744971275329,
1293
+ "num_tokens": 17483883.0,
1294
+ "step": 2825
1295
+ },
1296
+ {
1297
+ "entropy": 0.8306651365756988,
1298
+ "epoch": 0.9866712826726675,
1299
+ "grad_norm": 2.359375,
1300
+ "learning_rate": 1.0711358472017244e-05,
1301
+ "loss": 0.7954608154296875,
1302
+ "mean_token_accuracy": 0.7919278919696808,
1303
+ "num_tokens": 17639972.0,
1304
+ "step": 2850
1305
+ },
1306
+ {
1307
+ "entropy": 0.8295222818851471,
1308
+ "epoch": 0.9953262939241821,
1309
+ "grad_norm": 2.125,
1310
+ "learning_rate": 1.0571498671905146e-05,
1311
+ "loss": 0.8072916412353516,
1312
+ "mean_token_accuracy": 0.7907999694347382,
1313
+ "num_tokens": 17794430.0,
1314
+ "step": 2875
1315
+ },
1316
+ {
1317
+ "entropy": 0.8070371369926297,
1318
+ "epoch": 1.0038082049506665,
1319
+ "grad_norm": 2.09375,
1320
+ "learning_rate": 1.0431526620429207e-05,
1321
+ "loss": 0.760282211303711,
1322
+ "mean_token_accuracy": 0.7995899867038337,
1323
+ "num_tokens": 17946140.0,
1324
+ "step": 2900
1325
+ },
1326
+ {
1327
+ "entropy": 0.7756562912464142,
1328
+ "epoch": 1.0124632162021812,
1329
+ "grad_norm": 2.140625,
1330
+ "learning_rate": 1.0291469810310669e-05,
1331
+ "loss": 0.741202621459961,
1332
+ "mean_token_accuracy": 0.803013744354248,
1333
+ "num_tokens": 18100839.0,
1334
+ "step": 2925
1335
+ },
1336
+ {
1337
+ "entropy": 0.7452013885974884,
1338
+ "epoch": 1.0211182274536956,
1339
+ "grad_norm": 2.15625,
1340
+ "learning_rate": 1.0151355750918712e-05,
1341
+ "loss": 0.7064004516601563,
1342
+ "mean_token_accuracy": 0.8126378917694091,
1343
+ "num_tokens": 18256691.0,
1344
+ "step": 2950
1345
+ },
1346
+ {
1347
+ "entropy": 0.7718643927574158,
1348
+ "epoch": 1.0297732387052103,
1349
+ "grad_norm": 1.8671875,
1350
+ "learning_rate": 1.001121196286718e-05,
1351
+ "loss": 0.7311404418945312,
1352
+ "mean_token_accuracy": 0.8063716220855713,
1353
+ "num_tokens": 18418483.0,
1354
+ "step": 2975
1355
+ },
1356
+ {
1357
+ "entropy": 0.765803085565567,
1358
+ "epoch": 1.038428249956725,
1359
+ "grad_norm": 2.0,
1360
+ "learning_rate": 9.871065972609089e-06,
1361
+ "loss": 0.7332243347167968,
1362
+ "mean_token_accuracy": 0.8049838674068451,
1363
+ "num_tokens": 18576253.0,
1364
+ "step": 3000
1365
+ },
1366
+ {
1367
+ "epoch": 1.038428249956725,
1368
+ "eval_entropy": 0.7786700289757525,
1369
+ "eval_loss": 0.8012996912002563,
1370
+ "eval_mean_token_accuracy": 0.792937566022404,
1371
+ "eval_num_tokens": 18576253.0,
1372
+ "eval_runtime": 16.277,
1373
+ "eval_samples_per_second": 298.888,
1374
+ "eval_steps_per_second": 18.738,
1375
+ "step": 3000
1376
+ },
1377
+ {
1378
+ "entropy": 0.7836507785320282,
1379
+ "epoch": 1.0470832612082395,
1380
+ "grad_norm": 2.0625,
1381
+ "learning_rate": 9.730945307030009e-06,
1382
+ "loss": 0.7418560028076172,
1383
+ "mean_token_accuracy": 0.8028053915500641,
1384
+ "num_tokens": 18729564.0,
1385
+ "step": 3025
1386
+ },
1387
+ {
1388
+ "entropy": 0.7653721117973328,
1389
+ "epoch": 1.0557382724597542,
1390
+ "grad_norm": 2.03125,
1391
+ "learning_rate": 9.590877488041336e-06,
1392
+ "loss": 0.7263675689697265,
1393
+ "mean_token_accuracy": 0.807190809249878,
1394
+ "num_tokens": 18891538.0,
1395
+ "step": 3050
1396
+ },
1397
+ {
1398
+ "entropy": 0.7604654181003571,
1399
+ "epoch": 1.0643932837112688,
1400
+ "grad_norm": 2.296875,
1401
+ "learning_rate": 9.45089002717457e-06,
1402
+ "loss": 0.72638916015625,
1403
+ "mean_token_accuracy": 0.8084564554691315,
1404
+ "num_tokens": 19050606.0,
1405
+ "step": 3075
1406
+ },
1407
+ {
1408
+ "entropy": 0.7386153995990753,
1409
+ "epoch": 1.0730482949627835,
1410
+ "grad_norm": 1.9765625,
1411
+ "learning_rate": 9.311010420177597e-06,
1412
+ "loss": 0.706187973022461,
1413
+ "mean_token_accuracy": 0.8120154666900635,
1414
+ "num_tokens": 19208707.0,
1415
+ "step": 3100
1416
+ },
1417
+ {
1418
+ "entropy": 0.7525534296035766,
1419
+ "epoch": 1.081703306214298,
1420
+ "grad_norm": 2.171875,
1421
+ "learning_rate": 9.171266141614112e-06,
1422
+ "loss": 0.7135391235351562,
1423
+ "mean_token_accuracy": 0.8123032712936401,
1424
+ "num_tokens": 19362737.0,
1425
+ "step": 3125
1426
+ },
1427
+ {
1428
+ "entropy": 0.7687537455558777,
1429
+ "epoch": 1.0903583174658127,
1430
+ "grad_norm": 2.109375,
1431
+ "learning_rate": 9.031684639467159e-06,
1432
+ "loss": 0.734495849609375,
1433
+ "mean_token_accuracy": 0.8068291318416595,
1434
+ "num_tokens": 19521058.0,
1435
+ "step": 3150
1436
+ },
1437
+ {
1438
+ "entropy": 0.7806685280799865,
1439
+ "epoch": 1.0990133287173274,
1440
+ "grad_norm": 2.046875,
1441
+ "learning_rate": 8.892293329747923e-06,
1442
+ "loss": 0.7469673156738281,
1443
+ "mean_token_accuracy": 0.8041970551013946,
1444
+ "num_tokens": 19675603.0,
1445
+ "step": 3175
1446
+ },
1447
+ {
1448
+ "entropy": 0.7709089267253876,
1449
+ "epoch": 1.1076683399688418,
1450
+ "grad_norm": 2.15625,
1451
+ "learning_rate": 8.753119591110802e-06,
1452
+ "loss": 0.7363986206054688,
1453
+ "mean_token_accuracy": 0.8061197555065155,
1454
+ "num_tokens": 19828898.0,
1455
+ "step": 3200
1456
+ },
1457
+ {
1458
+ "epoch": 1.1076683399688418,
1459
+ "eval_entropy": 0.7721671974072691,
1460
+ "eval_loss": 0.8010231852531433,
1461
+ "eval_mean_token_accuracy": 0.7929807965872717,
1462
+ "eval_num_tokens": 19828898.0,
1463
+ "eval_runtime": 16.2529,
1464
+ "eval_samples_per_second": 299.331,
1465
+ "eval_steps_per_second": 18.766,
1466
+ "step": 3200
1467
+ },
1468
+ {
1469
+ "entropy": 0.7211595928668976,
1470
+ "epoch": 1.1163233512203565,
1471
+ "grad_norm": 2.046875,
1472
+ "learning_rate": 8.614190759475789e-06,
1473
+ "loss": 0.6826175689697266,
1474
+ "mean_token_accuracy": 0.8186909162998199,
1475
+ "num_tokens": 19982072.0,
1476
+ "step": 3225
1477
+ },
1478
+ {
1479
+ "entropy": 0.7708668720722198,
1480
+ "epoch": 1.1249783624718712,
1481
+ "grad_norm": 2.15625,
1482
+ "learning_rate": 8.475534122659275e-06,
1483
+ "loss": 0.7388634490966797,
1484
+ "mean_token_accuracy": 0.8042692136764527,
1485
+ "num_tokens": 20137744.0,
1486
+ "step": 3250
1487
+ },
1488
+ {
1489
+ "entropy": 0.7538685095310211,
1490
+ "epoch": 1.133633373723386,
1491
+ "grad_norm": 2.171875,
1492
+ "learning_rate": 8.337176915014293e-06,
1493
+ "loss": 0.7132722473144532,
1494
+ "mean_token_accuracy": 0.8105318355560303,
1495
+ "num_tokens": 20290524.0,
1496
+ "step": 3275
1497
+ },
1498
+ {
1499
+ "entropy": 0.7487189078330994,
1500
+ "epoch": 1.1422883849749004,
1501
+ "grad_norm": 2.046875,
1502
+ "learning_rate": 8.199146312081265e-06,
1503
+ "loss": 0.7089384460449218,
1504
+ "mean_token_accuracy": 0.8108018398284912,
1505
+ "num_tokens": 20443485.0,
1506
+ "step": 3300
1507
+ },
1508
+ {
1509
+ "entropy": 0.7543841946125031,
1510
+ "epoch": 1.150943396226415,
1511
+ "grad_norm": 2.0625,
1512
+ "learning_rate": 8.061469425250273e-06,
1513
+ "loss": 0.7227223205566407,
1514
+ "mean_token_accuracy": 0.8090525925159454,
1515
+ "num_tokens": 20595797.0,
1516
+ "step": 3325
1517
+ },
1518
+ {
1519
+ "entropy": 0.7593919610977173,
1520
+ "epoch": 1.1595984074779297,
1521
+ "grad_norm": 2.015625,
1522
+ "learning_rate": 7.924173296435987e-06,
1523
+ "loss": 0.7215187072753906,
1524
+ "mean_token_accuracy": 0.8093890881538391,
1525
+ "num_tokens": 20749554.0,
1526
+ "step": 3350
1527
+ },
1528
+ {
1529
+ "entropy": 0.7829481947422028,
1530
+ "epoch": 1.1682534187294444,
1531
+ "grad_norm": 2.171875,
1532
+ "learning_rate": 7.78728489276618e-06,
1533
+ "loss": 0.7491289520263672,
1534
+ "mean_token_accuracy": 0.8015105819702149,
1535
+ "num_tokens": 20909534.0,
1536
+ "step": 3375
1537
+ },
1538
+ {
1539
+ "entropy": 0.7392001152038574,
1540
+ "epoch": 1.176908429980959,
1541
+ "grad_norm": 2.1875,
1542
+ "learning_rate": 7.650831101284977e-06,
1543
+ "loss": 0.7020195007324219,
1544
+ "mean_token_accuracy": 0.8150424540042878,
1545
+ "num_tokens": 21059158.0,
1546
+ "step": 3400
1547
+ },
1548
+ {
1549
+ "epoch": 1.176908429980959,
1550
+ "eval_entropy": 0.7698653085309951,
1551
+ "eval_loss": 0.8008679747581482,
1552
+ "eval_mean_token_accuracy": 0.793300522155449,
1553
+ "eval_num_tokens": 21059158.0,
1554
+ "eval_runtime": 16.2799,
1555
+ "eval_samples_per_second": 298.835,
1556
+ "eval_steps_per_second": 18.735,
1557
+ "step": 3400
1558
+ },
1559
+ {
1560
+ "entropy": 0.768788868188858,
1561
+ "epoch": 1.1855634412324736,
1562
+ "grad_norm": 2.046875,
1563
+ "learning_rate": 7.514838723671799e-06,
1564
+ "loss": 0.7372692108154297,
1565
+ "mean_token_accuracy": 0.8063059449195862,
1566
+ "num_tokens": 21208688.0,
1567
+ "step": 3425
1568
+ },
1569
+ {
1570
+ "entropy": 0.7443365502357483,
1571
+ "epoch": 1.1942184524839883,
1572
+ "grad_norm": 1.875,
1573
+ "learning_rate": 7.379334470977093e-06,
1574
+ "loss": 0.710152587890625,
1575
+ "mean_token_accuracy": 0.811426283121109,
1576
+ "num_tokens": 21365731.0,
1577
+ "step": 3450
1578
+ },
1579
+ {
1580
+ "entropy": 0.7438737845420837,
1581
+ "epoch": 1.2028734637355027,
1582
+ "grad_norm": 1.96875,
1583
+ "learning_rate": 7.244344958375881e-06,
1584
+ "loss": 0.710167465209961,
1585
+ "mean_token_accuracy": 0.8120112872123718,
1586
+ "num_tokens": 21519688.0,
1587
+ "step": 3475
1588
+ },
1589
+ {
1590
+ "entropy": 0.7728407967090607,
1591
+ "epoch": 1.2115284749870174,
1592
+ "grad_norm": 2.046875,
1593
+ "learning_rate": 7.109896699940084e-06,
1594
+ "loss": 0.7461888885498047,
1595
+ "mean_token_accuracy": 0.8037091171741486,
1596
+ "num_tokens": 21676493.0,
1597
+ "step": 3500
1598
+ },
1599
+ {
1600
+ "entropy": 0.7640272963047028,
1601
+ "epoch": 1.2201834862385321,
1602
+ "grad_norm": 2.203125,
1603
+ "learning_rate": 6.976016103430765e-06,
1604
+ "loss": 0.7235647583007813,
1605
+ "mean_token_accuracy": 0.8083086097240448,
1606
+ "num_tokens": 21825664.0,
1607
+ "step": 3525
1608
+ },
1609
+ {
1610
+ "entropy": 0.7460694527626037,
1611
+ "epoch": 1.2288384974900468,
1612
+ "grad_norm": 2.03125,
1613
+ "learning_rate": 6.842729465111224e-06,
1614
+ "loss": 0.7100708770751953,
1615
+ "mean_token_accuracy": 0.8116568970680237,
1616
+ "num_tokens": 21976504.0,
1617
+ "step": 3550
1618
+ },
1619
+ {
1620
+ "entropy": 0.759437519311905,
1621
+ "epoch": 1.2374935087415613,
1622
+ "grad_norm": 2.609375,
1623
+ "learning_rate": 6.710062964581988e-06,
1624
+ "loss": 0.722847900390625,
1625
+ "mean_token_accuracy": 0.807286411523819,
1626
+ "num_tokens": 22133165.0,
1627
+ "step": 3575
1628
+ },
1629
+ {
1630
+ "entropy": 0.7708945345878601,
1631
+ "epoch": 1.246148519993076,
1632
+ "grad_norm": 2.015625,
1633
+ "learning_rate": 6.578042659638726e-06,
1634
+ "loss": 0.7426469421386719,
1635
+ "mean_token_accuracy": 0.8035933816432953,
1636
+ "num_tokens": 22290456.0,
1637
+ "step": 3600
1638
+ },
1639
+ {
1640
+ "epoch": 1.246148519993076,
1641
+ "eval_entropy": 0.7726592179204597,
1642
+ "eval_loss": 0.8001708984375,
1643
+ "eval_mean_token_accuracy": 0.7932597899046101,
1644
+ "eval_num_tokens": 22290456.0,
1645
+ "eval_runtime": 16.2656,
1646
+ "eval_samples_per_second": 299.097,
1647
+ "eval_steps_per_second": 18.751,
1648
+ "step": 3600
1649
+ },
1650
+ {
1651
+ "entropy": 0.7578576147556305,
1652
+ "epoch": 1.2548035312445907,
1653
+ "grad_norm": 2.078125,
1654
+ "learning_rate": 6.446694481154104e-06,
1655
+ "loss": 0.7225240325927734,
1656
+ "mean_token_accuracy": 0.8097064507007599,
1657
+ "num_tokens": 22446824.0,
1658
+ "step": 3625
1659
+ },
1660
+ {
1661
+ "entropy": 0.7762425947189331,
1662
+ "epoch": 1.2634585424961053,
1663
+ "grad_norm": 2.265625,
1664
+ "learning_rate": 6.3160442279845206e-06,
1665
+ "loss": 0.7412718200683593,
1666
+ "mean_token_accuracy": 0.805392199754715,
1667
+ "num_tokens": 22602588.0,
1668
+ "step": 3650
1669
+ },
1670
+ {
1671
+ "entropy": 0.7637201595306397,
1672
+ "epoch": 1.2721135537476198,
1673
+ "grad_norm": 2.25,
1674
+ "learning_rate": 6.186117561902836e-06,
1675
+ "loss": 0.7297930908203125,
1676
+ "mean_token_accuracy": 0.8078333747386932,
1677
+ "num_tokens": 22752707.0,
1678
+ "step": 3675
1679
+ },
1680
+ {
1681
+ "entropy": 0.7704725599288941,
1682
+ "epoch": 1.2807685649991345,
1683
+ "grad_norm": 2.125,
1684
+ "learning_rate": 6.056940002557969e-06,
1685
+ "loss": 0.7372798919677734,
1686
+ "mean_token_accuracy": 0.8056825184822083,
1687
+ "num_tokens": 22907803.0,
1688
+ "step": 3700
1689
+ },
1690
+ {
1691
+ "entropy": 0.7552802407741547,
1692
+ "epoch": 1.2894235762506492,
1693
+ "grad_norm": 2.15625,
1694
+ "learning_rate": 5.928536922462449e-06,
1695
+ "loss": 0.7166344451904297,
1696
+ "mean_token_accuracy": 0.8109099209308624,
1697
+ "num_tokens": 23059819.0,
1698
+ "step": 3725
1699
+ },
1700
+ {
1701
+ "entropy": 0.7541340529918671,
1702
+ "epoch": 1.2980785875021636,
1703
+ "grad_norm": 2.3125,
1704
+ "learning_rate": 5.80093354200886e-06,
1705
+ "loss": 0.7238423919677734,
1706
+ "mean_token_accuracy": 0.809191288948059,
1707
+ "num_tokens": 23217392.0,
1708
+ "step": 3750
1709
+ },
1710
+ {
1711
+ "entropy": 0.7676697385311126,
1712
+ "epoch": 1.3067335987536783,
1713
+ "grad_norm": 2.21875,
1714
+ "learning_rate": 5.674154924516129e-06,
1715
+ "loss": 0.7366065979003906,
1716
+ "mean_token_accuracy": 0.8054041469097137,
1717
+ "num_tokens": 23371073.0,
1718
+ "step": 3775
1719
+ },
1720
+ {
1721
+ "entropy": 0.7785192513465882,
1722
+ "epoch": 1.315388610005193,
1723
+ "grad_norm": 1.9140625,
1724
+ "learning_rate": 5.548225971306731e-06,
1725
+ "loss": 0.7439189147949219,
1726
+ "mean_token_accuracy": 0.8034746491909027,
1727
+ "num_tokens": 23524592.0,
1728
+ "step": 3800
1729
+ },
1730
+ {
1731
+ "epoch": 1.315388610005193,
1732
+ "eval_entropy": 0.7713183484116538,
1733
+ "eval_loss": 0.7998276352882385,
1734
+ "eval_mean_token_accuracy": 0.7933589818047695,
1735
+ "eval_num_tokens": 23524592.0,
1736
+ "eval_runtime": 16.2844,
1737
+ "eval_samples_per_second": 298.753,
1738
+ "eval_steps_per_second": 18.73,
1739
+ "step": 3800
1740
+ },
1741
+ {
1742
+ "entropy": 0.7570435631275177,
1743
+ "epoch": 1.3240436212567077,
1744
+ "grad_norm": 2.078125,
1745
+ "learning_rate": 5.423171416815651e-06,
1746
+ "loss": 0.7266172790527343,
1747
+ "mean_token_accuracy": 0.8094574224948883,
1748
+ "num_tokens": 23682141.0,
1749
+ "step": 3825
1750
+ },
1751
+ {
1752
+ "entropy": 0.76507066488266,
1753
+ "epoch": 1.3326986325082222,
1754
+ "grad_norm": 2.15625,
1755
+ "learning_rate": 5.299015823732163e-06,
1756
+ "loss": 0.7294864654541016,
1757
+ "mean_token_accuracy": 0.8064786922931672,
1758
+ "num_tokens": 23839050.0,
1759
+ "step": 3850
1760
+ },
1761
+ {
1762
+ "entropy": 0.7562107217311859,
1763
+ "epoch": 1.3413536437597369,
1764
+ "grad_norm": 2.0625,
1765
+ "learning_rate": 5.175783578175309e-06,
1766
+ "loss": 0.719031982421875,
1767
+ "mean_token_accuracy": 0.8091187167167664,
1768
+ "num_tokens": 23992205.0,
1769
+ "step": 3875
1770
+ },
1771
+ {
1772
+ "entropy": 0.7576862835884094,
1773
+ "epoch": 1.3500086550112516,
1774
+ "grad_norm": 2.109375,
1775
+ "learning_rate": 5.053498884904122e-06,
1776
+ "loss": 0.7270169830322266,
1777
+ "mean_token_accuracy": 0.8094841420650483,
1778
+ "num_tokens": 24139859.0,
1779
+ "step": 3900
1780
+ },
1781
+ {
1782
+ "entropy": 0.7541490840911865,
1783
+ "epoch": 1.3586636662627662,
1784
+ "grad_norm": 2.109375,
1785
+ "learning_rate": 4.932185762563381e-06,
1786
+ "loss": 0.7113311004638672,
1787
+ "mean_token_accuracy": 0.8112999868392944,
1788
+ "num_tokens": 24289196.0,
1789
+ "step": 3925
1790
+ },
1791
+ {
1792
+ "entropy": 0.7688007843494415,
1793
+ "epoch": 1.3673186775142807,
1794
+ "grad_norm": 2.1875,
1795
+ "learning_rate": 4.811868038965999e-06,
1796
+ "loss": 0.745551986694336,
1797
+ "mean_token_accuracy": 0.8050249302387238,
1798
+ "num_tokens": 24447938.0,
1799
+ "step": 3950
1800
+ },
1801
+ {
1802
+ "entropy": 0.7551292729377747,
1803
+ "epoch": 1.3759736887657954,
1804
+ "grad_norm": 2.1875,
1805
+ "learning_rate": 4.692569346412864e-06,
1806
+ "loss": 0.7206728363037109,
1807
+ "mean_token_accuracy": 0.8098892831802368,
1808
+ "num_tokens": 24598742.0,
1809
+ "step": 3975
1810
+ },
1811
+ {
1812
+ "entropy": 0.7587357807159424,
1813
+ "epoch": 1.38462870001731,
1814
+ "grad_norm": 2.125,
1815
+ "learning_rate": 4.574313117051059e-06,
1816
+ "loss": 0.7276847076416015,
1817
+ "mean_token_accuracy": 0.8077455127239227,
1818
+ "num_tokens": 24750406.0,
1819
+ "step": 4000
1820
+ },
1821
+ {
1822
+ "epoch": 1.38462870001731,
1823
+ "eval_entropy": 0.7717335020909544,
1824
+ "eval_loss": 0.7996781468391418,
1825
+ "eval_mean_token_accuracy": 0.7934077425081222,
1826
+ "eval_num_tokens": 24750406.0,
1827
+ "eval_runtime": 16.2676,
1828
+ "eval_samples_per_second": 299.061,
1829
+ "eval_steps_per_second": 18.749,
1830
+ "step": 4000
1831
+ },
1832
+ {
1833
+ "entropy": 0.7805033266544342,
1834
+ "epoch": 1.3932837112688246,
1835
+ "grad_norm": 2.078125,
1836
+ "learning_rate": 4.457122578271457e-06,
1837
+ "loss": 0.7509593200683594,
1838
+ "mean_token_accuracy": 0.8028152561187745,
1839
+ "num_tokens": 24906659.0,
1840
+ "step": 4025
1841
+ },
1842
+ {
1843
+ "entropy": 0.7705829620361329,
1844
+ "epoch": 1.4019387225203392,
1845
+ "grad_norm": 2.125,
1846
+ "learning_rate": 4.341020748146449e-06,
1847
+ "loss": 0.7392497253417969,
1848
+ "mean_token_accuracy": 0.8049503290653228,
1849
+ "num_tokens": 25066654.0,
1850
+ "step": 4050
1851
+ },
1852
+ {
1853
+ "entropy": 0.7581047600507737,
1854
+ "epoch": 1.410593733771854,
1855
+ "grad_norm": 2.0,
1856
+ "learning_rate": 4.226030430908869e-06,
1857
+ "loss": 0.7285484313964844,
1858
+ "mean_token_accuracy": 0.8085175144672394,
1859
+ "num_tokens": 25220415.0,
1860
+ "step": 4075
1861
+ },
1862
+ {
1863
+ "entropy": 0.7656786620616913,
1864
+ "epoch": 1.4192487450233684,
1865
+ "grad_norm": 2.21875,
1866
+ "learning_rate": 4.112174212472864e-06,
1867
+ "loss": 0.7323892211914063,
1868
+ "mean_token_accuracy": 0.8051603281497955,
1869
+ "num_tokens": 25374458.0,
1870
+ "step": 4100
1871
+ },
1872
+ {
1873
+ "entropy": 0.7522096228599549,
1874
+ "epoch": 1.427903756274883,
1875
+ "grad_norm": 2.171875,
1876
+ "learning_rate": 3.99947445599767e-06,
1877
+ "loss": 0.7212752532958985,
1878
+ "mean_token_accuracy": 0.8096666383743286,
1879
+ "num_tokens": 25525969.0,
1880
+ "step": 4125
1881
+ },
1882
+ {
1883
+ "entropy": 0.7608710610866547,
1884
+ "epoch": 1.4365587675263978,
1885
+ "grad_norm": 2.40625,
1886
+ "learning_rate": 3.887953297495121e-06,
1887
+ "loss": 0.7316089630126953,
1888
+ "mean_token_accuracy": 0.8074719154834747,
1889
+ "num_tokens": 25681452.0,
1890
+ "step": 4150
1891
+ },
1892
+ {
1893
+ "entropy": 0.7639850771427155,
1894
+ "epoch": 1.4452137787779125,
1895
+ "grad_norm": 2.140625,
1896
+ "learning_rate": 3.77763264148181e-06,
1897
+ "loss": 0.7320518493652344,
1898
+ "mean_token_accuracy": 0.809201009273529,
1899
+ "num_tokens": 25838398.0,
1900
+ "step": 4175
1901
+ },
1902
+ {
1903
+ "entropy": 0.7513358175754548,
1904
+ "epoch": 1.4538687900294271,
1905
+ "grad_norm": 2.21875,
1906
+ "learning_rate": 3.6685341566766785e-06,
1907
+ "loss": 0.7206896209716797,
1908
+ "mean_token_accuracy": 0.8095373523235321,
1909
+ "num_tokens": 25992640.0,
1910
+ "step": 4200
1911
+ },
1912
+ {
1913
+ "epoch": 1.4538687900294271,
1914
+ "eval_entropy": 0.7707103953986871,
1915
+ "eval_loss": 0.7995045185089111,
1916
+ "eval_mean_token_accuracy": 0.7933821330305005,
1917
+ "eval_num_tokens": 25992640.0,
1918
+ "eval_runtime": 16.2677,
1919
+ "eval_samples_per_second": 299.058,
1920
+ "eval_steps_per_second": 18.749,
1921
+ "step": 4200
1922
+ },
1923
+ {
1924
+ "entropy": 0.7595930647850037,
1925
+ "epoch": 1.4625238012809416,
1926
+ "grad_norm": 2.171875,
1927
+ "learning_rate": 3.5606792717449455e-06,
1928
+ "loss": 0.7264796447753906,
1929
+ "mean_token_accuracy": 0.8094517874717713,
1930
+ "num_tokens": 26141304.0,
1931
+ "step": 4225
1932
+ },
1933
+ {
1934
+ "entropy": 0.7630014491081237,
1935
+ "epoch": 1.4711788125324563,
1936
+ "grad_norm": 2.15625,
1937
+ "learning_rate": 3.4540891710891768e-06,
1938
+ "loss": 0.7287496185302734,
1939
+ "mean_token_accuracy": 0.808629115819931,
1940
+ "num_tokens": 26295315.0,
1941
+ "step": 4250
1942
+ },
1943
+ {
1944
+ "entropy": 0.7566706609725952,
1945
+ "epoch": 1.479833823783971,
1946
+ "grad_norm": 2.34375,
1947
+ "learning_rate": 3.3487847906883232e-06,
1948
+ "loss": 0.7267755126953125,
1949
+ "mean_token_accuracy": 0.8091002118587494,
1950
+ "num_tokens": 26450821.0,
1951
+ "step": 4275
1952
+ },
1953
+ {
1954
+ "entropy": 0.7702415359020233,
1955
+ "epoch": 1.4884888350354855,
1956
+ "grad_norm": 2.09375,
1957
+ "learning_rate": 3.2447868139855954e-06,
1958
+ "loss": 0.74352294921875,
1959
+ "mean_token_accuracy": 0.8064157390594482,
1960
+ "num_tokens": 26610364.0,
1961
+ "step": 4300
1962
+ },
1963
+ {
1964
+ "entropy": 0.78652991771698,
1965
+ "epoch": 1.4971438462870001,
1966
+ "grad_norm": 2.359375,
1967
+ "learning_rate": 3.142115667825868e-06,
1968
+ "loss": 0.7553633117675781,
1969
+ "mean_token_accuracy": 0.8014207696914672,
1970
+ "num_tokens": 26766393.0,
1971
+ "step": 4325
1972
+ },
1973
+ {
1974
+ "entropy": 0.7758943355083465,
1975
+ "epoch": 1.5057988575385148,
1976
+ "grad_norm": 2.0,
1977
+ "learning_rate": 3.0407915184435544e-06,
1978
+ "loss": 0.7414107513427735,
1979
+ "mean_token_accuracy": 0.8034427273273468,
1980
+ "num_tokens": 26928821.0,
1981
+ "step": 4350
1982
+ },
1983
+ {
1984
+ "entropy": 0.7746455824375152,
1985
+ "epoch": 1.5144538687900293,
1986
+ "grad_norm": 2.15625,
1987
+ "learning_rate": 2.9408342675016288e-06,
1988
+ "loss": 0.7338740539550781,
1989
+ "mean_token_accuracy": 0.8053066456317901,
1990
+ "num_tokens": 27081955.0,
1991
+ "step": 4375
1992
+ },
1993
+ {
1994
+ "entropy": 0.7544065153598786,
1995
+ "epoch": 1.5231088800415442,
1996
+ "grad_norm": 2.125,
1997
+ "learning_rate": 2.84226354818263e-06,
1998
+ "loss": 0.7168594360351562,
1999
+ "mean_token_accuracy": 0.8104126238822937,
2000
+ "num_tokens": 27233223.0,
2001
+ "step": 4400
2002
+ },
2003
+ {
2004
+ "epoch": 1.5231088800415442,
2005
+ "eval_entropy": 0.770115079156688,
2006
+ "eval_loss": 0.7993313074111938,
2007
+ "eval_mean_token_accuracy": 0.7934993093131019,
2008
+ "eval_num_tokens": 27233223.0,
2009
+ "eval_runtime": 16.2903,
2010
+ "eval_samples_per_second": 298.644,
2011
+ "eval_steps_per_second": 18.723,
2012
+ "step": 4400
2013
+ },
2014
+ {
2015
+ "entropy": 0.7756598269939423,
2016
+ "epoch": 1.5317638912930587,
2017
+ "grad_norm": 2.15625,
2018
+ "learning_rate": 2.74509872133238e-06,
2019
+ "loss": 0.750801010131836,
2020
+ "mean_token_accuracy": 0.8043915629386902,
2021
+ "num_tokens": 27385671.0,
2022
+ "step": 4425
2023
+ },
2024
+ {
2025
+ "entropy": 0.7870103931427002,
2026
+ "epoch": 1.5404189025445731,
2027
+ "grad_norm": 2.46875,
2028
+ "learning_rate": 2.6493588716572306e-06,
2029
+ "loss": 0.7561086273193359,
2030
+ "mean_token_accuracy": 0.8031144332885742,
2031
+ "num_tokens": 27542323.0,
2032
+ "step": 4450
2033
+ },
2034
+ {
2035
+ "entropy": 0.7758941638469696,
2036
+ "epoch": 1.549073913796088,
2037
+ "grad_norm": 2.359375,
2038
+ "learning_rate": 2.555062803975512e-06,
2039
+ "loss": 0.7479061889648437,
2040
+ "mean_token_accuracy": 0.8041390895843505,
2041
+ "num_tokens": 27695755.0,
2042
+ "step": 4475
2043
+ },
2044
+ {
2045
+ "entropy": 0.754768637418747,
2046
+ "epoch": 1.5577289250476025,
2047
+ "grad_norm": 2.4375,
2048
+ "learning_rate": 2.4622290395239613e-06,
2049
+ "loss": 0.7247459411621093,
2050
+ "mean_token_accuracy": 0.8090137577056885,
2051
+ "num_tokens": 27847694.0,
2052
+ "step": 4500
2053
+ },
2054
+ {
2055
+ "entropy": 0.7636888045072555,
2056
+ "epoch": 1.5663839362991172,
2057
+ "grad_norm": 2.140625,
2058
+ "learning_rate": 2.3708758123198904e-06,
2059
+ "loss": 0.7294938659667969,
2060
+ "mean_token_accuracy": 0.8078456258773804,
2061
+ "num_tokens": 28001389.0,
2062
+ "step": 4525
2063
+ },
2064
+ {
2065
+ "entropy": 0.7723946702480317,
2066
+ "epoch": 1.575038947550632,
2067
+ "grad_norm": 1.9453125,
2068
+ "learning_rate": 2.2810210655796915e-06,
2069
+ "loss": 0.7421239471435547,
2070
+ "mean_token_accuracy": 0.8051141202449799,
2071
+ "num_tokens": 28155633.0,
2072
+ "step": 4550
2073
+ },
2074
+ {
2075
+ "entropy": 0.7729098165035247,
2076
+ "epoch": 1.5836939588021464,
2077
+ "grad_norm": 1.9375,
2078
+ "learning_rate": 2.192682448194555e-06,
2079
+ "loss": 0.7385528564453125,
2080
+ "mean_token_accuracy": 0.8061604249477387,
2081
+ "num_tokens": 28316815.0,
2082
+ "step": 4575
2083
+ },
2084
+ {
2085
+ "entropy": 0.7528582894802094,
2086
+ "epoch": 1.592348970053661,
2087
+ "grad_norm": 2.234375,
2088
+ "learning_rate": 2.1058773112639054e-06,
2089
+ "loss": 0.719280014038086,
2090
+ "mean_token_accuracy": 0.8096978998184204,
2091
+ "num_tokens": 28473438.0,
2092
+ "step": 4600
2093
+ },
2094
+ {
2095
+ "epoch": 1.592348970053661,
2096
+ "eval_entropy": 0.7718443285246365,
2097
+ "eval_loss": 0.7991446852684021,
2098
+ "eval_mean_token_accuracy": 0.7934910416603088,
2099
+ "eval_num_tokens": 28473438.0,
2100
+ "eval_runtime": 16.2845,
2101
+ "eval_samples_per_second": 298.751,
2102
+ "eval_steps_per_second": 18.729,
2103
+ "step": 4600
2104
+ },
2105
+ {
2106
+ "entropy": 0.7752782785892487,
2107
+ "epoch": 1.6010039813051757,
2108
+ "grad_norm": 2.203125,
2109
+ "learning_rate": 2.020622704687398e-06,
2110
+ "loss": 0.7455803680419922,
2111
+ "mean_token_accuracy": 0.8029440557956695,
2112
+ "num_tokens": 28629933.0,
2113
+ "step": 4625
2114
+ },
2115
+ {
2116
+ "entropy": 0.7682564723491668,
2117
+ "epoch": 1.6096589925566902,
2118
+ "grad_norm": 2.28125,
2119
+ "learning_rate": 1.936935373816046e-06,
2120
+ "loss": 0.7349263000488281,
2121
+ "mean_token_accuracy": 0.8075251269340515,
2122
+ "num_tokens": 28779447.0,
2123
+ "step": 4650
2124
+ },
2125
+ {
2126
+ "entropy": 0.7602540409564972,
2127
+ "epoch": 1.6183140038082051,
2128
+ "grad_norm": 2.15625,
2129
+ "learning_rate": 1.8548317561631534e-06,
2130
+ "loss": 0.722344741821289,
2131
+ "mean_token_accuracy": 0.809203851222992,
2132
+ "num_tokens": 28935999.0,
2133
+ "step": 4675
2134
+ },
2135
+ {
2136
+ "entropy": 0.7502141880989075,
2137
+ "epoch": 1.6269690150597196,
2138
+ "grad_norm": 1.984375,
2139
+ "learning_rate": 1.7743279781757493e-06,
2140
+ "loss": 0.718406982421875,
2141
+ "mean_token_accuracy": 0.8091218245029449,
2142
+ "num_tokens": 29092219.0,
2143
+ "step": 4700
2144
+ },
2145
+ {
2146
+ "entropy": 0.7485624659061432,
2147
+ "epoch": 1.635624026311234,
2148
+ "grad_norm": 1.9375,
2149
+ "learning_rate": 1.6954398520670867e-06,
2150
+ "loss": 0.7138198852539063,
2151
+ "mean_token_accuracy": 0.8104638087749482,
2152
+ "num_tokens": 29249386.0,
2153
+ "step": 4725
2154
+ },
2155
+ {
2156
+ "entropy": 0.7689664399623871,
2157
+ "epoch": 1.644279037562749,
2158
+ "grad_norm": 2.0625,
2159
+ "learning_rate": 1.6181828727108772e-06,
2160
+ "loss": 0.7358683776855469,
2161
+ "mean_token_accuracy": 0.8066857266426086,
2162
+ "num_tokens": 29406880.0,
2163
+ "step": 4750
2164
+ },
2165
+ {
2166
+ "entropy": 0.7672972738742828,
2167
+ "epoch": 1.6529340488142634,
2168
+ "grad_norm": 2.40625,
2169
+ "learning_rate": 1.542572214597844e-06,
2170
+ "loss": 0.7319490051269532,
2171
+ "mean_token_accuracy": 0.8055110704898835,
2172
+ "num_tokens": 29558251.0,
2173
+ "step": 4775
2174
+ },
2175
+ {
2176
+ "entropy": 0.7510581874847412,
2177
+ "epoch": 1.6615890600657781,
2178
+ "grad_norm": 2.078125,
2179
+ "learning_rate": 1.4686227288552357e-06,
2180
+ "loss": 0.7089730072021484,
2181
+ "mean_token_accuracy": 0.811128660440445,
2182
+ "num_tokens": 29713889.0,
2183
+ "step": 4800
2184
+ },
2185
+ {
2186
+ "epoch": 1.6615890600657781,
2187
+ "eval_entropy": 0.7707234706057877,
2188
+ "eval_loss": 0.7992180585861206,
2189
+ "eval_mean_token_accuracy": 0.7935016180648178,
2190
+ "eval_num_tokens": 29713889.0,
2191
+ "eval_runtime": 16.3186,
2192
+ "eval_samples_per_second": 298.126,
2193
+ "eval_steps_per_second": 18.69,
2194
+ "step": 4800
2195
+ },
2196
+ {
2197
+ "entropy": 0.7817390990257264,
2198
+ "epoch": 1.6702440713172928,
2199
+ "grad_norm": 2.25,
2200
+ "learning_rate": 1.3963489403297992e-06,
2201
+ "loss": 0.7543077087402343,
2202
+ "mean_token_accuracy": 0.8011208462715149,
2203
+ "num_tokens": 29870315.0,
2204
+ "step": 4825
2205
+ },
2206
+ {
2207
+ "entropy": 0.7862332105636597,
2208
+ "epoch": 1.6788990825688073,
2209
+ "grad_norm": 2.125,
2210
+ "learning_rate": 1.3257650447349035e-06,
2211
+ "loss": 0.7527214050292969,
2212
+ "mean_token_accuracy": 0.8031284868717193,
2213
+ "num_tokens": 30023428.0,
2214
+ "step": 4850
2215
+ },
2216
+ {
2217
+ "entropy": 0.7391238558292389,
2218
+ "epoch": 1.687554093820322,
2219
+ "grad_norm": 2.046875,
2220
+ "learning_rate": 1.2568849058622435e-06,
2221
+ "loss": 0.7039041137695312,
2222
+ "mean_token_accuracy": 0.811707170009613,
2223
+ "num_tokens": 30181568.0,
2224
+ "step": 4875
2225
+ },
2226
+ {
2227
+ "entropy": 0.7737973690032959,
2228
+ "epoch": 1.6962091050718366,
2229
+ "grad_norm": 2.125,
2230
+ "learning_rate": 1.1897220528588005e-06,
2231
+ "loss": 0.7400728607177735,
2232
+ "mean_token_accuracy": 0.8040676033496856,
2233
+ "num_tokens": 30334041.0,
2234
+ "step": 4900
2235
+ },
2236
+ {
2237
+ "entropy": 0.7637208473682403,
2238
+ "epoch": 1.704864116323351,
2239
+ "grad_norm": 2.265625,
2240
+ "learning_rate": 1.1242896775694923e-06,
2241
+ "loss": 0.7268269348144532,
2242
+ "mean_token_accuracy": 0.8077472651004791,
2243
+ "num_tokens": 30488073.0,
2244
+ "step": 4925
2245
+ },
2246
+ {
2247
+ "entropy": 0.7567976391315461,
2248
+ "epoch": 1.7135191275748658,
2249
+ "grad_norm": 1.984375,
2250
+ "learning_rate": 1.0606006319460794e-06,
2251
+ "loss": 0.7226065826416016,
2252
+ "mean_token_accuracy": 0.8086667931079865,
2253
+ "num_tokens": 30643659.0,
2254
+ "step": 4950
2255
+ },
2256
+ {
2257
+ "entropy": 0.7528193962574005,
2258
+ "epoch": 1.7221741388263805,
2259
+ "grad_norm": 2.09375,
2260
+ "learning_rate": 9.986674255228546e-07,
2261
+ "loss": 0.7181925964355469,
2262
+ "mean_token_accuracy": 0.8110329484939576,
2263
+ "num_tokens": 30794462.0,
2264
+ "step": 4975
2265
+ },
2266
+ {
2267
+ "entropy": 0.7771501529216767,
2268
+ "epoch": 1.730829150077895,
2269
+ "grad_norm": 2.203125,
2270
+ "learning_rate": 9.385022229595631e-07,
2271
+ "loss": 0.7418533325195312,
2272
+ "mean_token_accuracy": 0.8045794427394867,
2273
+ "num_tokens": 30945283.0,
2274
+ "step": 5000
2275
+ },
2276
+ {
2277
+ "epoch": 1.730829150077895,
2278
+ "eval_entropy": 0.7709188702653667,
2279
+ "eval_loss": 0.7991884350776672,
2280
+ "eval_mean_token_accuracy": 0.7934240538565839,
2281
+ "eval_num_tokens": 30945283.0,
2282
+ "eval_runtime": 16.2662,
2283
+ "eval_samples_per_second": 299.086,
2284
+ "eval_steps_per_second": 18.751,
2285
+ "step": 5000
2286
+ },
2287
+ {
2288
+ "entropy": 0.7722408485412597,
2289
+ "epoch": 1.7394841613294099,
2290
+ "grad_norm": 2.0625,
2291
+ "learning_rate": 8.801168416520733e-07,
2292
+ "loss": 0.7394268035888671,
2293
+ "mean_token_accuracy": 0.806780993938446,
2294
+ "num_tokens": 31105352.0,
2295
+ "step": 5025
2296
+ },
2297
+ {
2298
+ "entropy": 0.7758919060230255,
2299
+ "epoch": 1.7481391725809243,
2300
+ "grad_norm": 2.0625,
2301
+ "learning_rate": 8.235227494112463e-07,
2302
+ "loss": 0.7348842620849609,
2303
+ "mean_token_accuracy": 0.8069607448577881,
2304
+ "num_tokens": 31253216.0,
2305
+ "step": 5050
2306
+ },
2307
+ {
2308
+ "entropy": 0.7573688685894012,
2309
+ "epoch": 1.756794183832439,
2310
+ "grad_norm": 2.0,
2311
+ "learning_rate": 7.687310622104982e-07,
2312
+ "loss": 0.7183684539794922,
2313
+ "mean_token_accuracy": 0.8097042238712311,
2314
+ "num_tokens": 31404175.0,
2315
+ "step": 5075
2316
+ },
2317
+ {
2318
+ "entropy": 0.7699829995632171,
2319
+ "epoch": 1.7654491950839537,
2320
+ "grad_norm": 2.21875,
2321
+ "learning_rate": 7.157525420024181e-07,
2322
+ "loss": 0.7363372802734375,
2323
+ "mean_token_accuracy": 0.8065412914752961,
2324
+ "num_tokens": 31553043.0,
2325
+ "step": 5100
2326
+ },
2327
+ {
2328
+ "entropy": 0.7569133055210113,
2329
+ "epoch": 1.7741042063354682,
2330
+ "grad_norm": 1.9609375,
2331
+ "learning_rate": 6.645975946049732e-07,
2332
+ "loss": 0.7242645263671875,
2333
+ "mean_token_accuracy": 0.8096802568435669,
2334
+ "num_tokens": 31708719.0,
2335
+ "step": 5125
2336
+ },
2337
+ {
2338
+ "entropy": 0.763712112903595,
2339
+ "epoch": 1.7827592175869829,
2340
+ "grad_norm": 2.125,
2341
+ "learning_rate": 6.152762676576352e-07,
2342
+ "loss": 0.7330033111572266,
2343
+ "mean_token_accuracy": 0.8064149105548859,
2344
+ "num_tokens": 31869942.0,
2345
+ "step": 5150
2346
+ },
2347
+ {
2348
+ "entropy": 0.7759097361564636,
2349
+ "epoch": 1.7914142288384975,
2350
+ "grad_norm": 2.203125,
2351
+ "learning_rate": 5.677982486478595e-07,
2352
+ "loss": 0.7459344482421875,
2353
+ "mean_token_accuracy": 0.8050018990039826,
2354
+ "num_tokens": 32024066.0,
2355
+ "step": 5175
2356
+ },
2357
+ {
2358
+ "entropy": 0.7608749055862427,
2359
+ "epoch": 1.800069240090012,
2360
+ "grad_norm": 2.09375,
2361
+ "learning_rate": 5.221728630083201e-07,
2362
+ "loss": 0.7280139923095703,
2363
+ "mean_token_accuracy": 0.8076766312122345,
2364
+ "num_tokens": 32180495.0,
2365
+ "step": 5200
2366
+ },
2367
+ {
2368
+ "epoch": 1.800069240090012,
2369
+ "eval_entropy": 0.7705254261610938,
2370
+ "eval_loss": 0.7991676330566406,
2371
+ "eval_mean_token_accuracy": 0.7934689746528375,
2372
+ "eval_num_tokens": 32180495.0,
2373
+ "eval_runtime": 16.2695,
2374
+ "eval_samples_per_second": 299.026,
2375
+ "eval_steps_per_second": 18.747,
2376
+ "step": 5200
2377
+ },
2378
+ {
2379
+ "entropy": 0.7587528014183045,
2380
+ "epoch": 1.8087242513415267,
2381
+ "grad_norm": 2.3125,
2382
+ "learning_rate": 4.784090722852353e-07,
2383
+ "loss": 0.7158480834960937,
2384
+ "mean_token_accuracy": 0.809622370004654,
2385
+ "num_tokens": 32338906.0,
2386
+ "step": 5225
2387
+ },
2388
+ {
2389
+ "entropy": 0.7616716659069062,
2390
+ "epoch": 1.8173792625930414,
2391
+ "grad_norm": 2.109375,
2392
+ "learning_rate": 4.365154723781895e-07,
2393
+ "loss": 0.7293673706054687,
2394
+ "mean_token_accuracy": 0.8077958106994629,
2395
+ "num_tokens": 32495631.0,
2396
+ "step": 5250
2397
+ },
2398
+ {
2399
+ "entropy": 0.737936383485794,
2400
+ "epoch": 1.8260342738445559,
2401
+ "grad_norm": 2.046875,
2402
+ "learning_rate": 3.965002918517569e-07,
2403
+ "loss": 0.7038777160644532,
2404
+ "mean_token_accuracy": 0.8131058704853058,
2405
+ "num_tokens": 32651647.0,
2406
+ "step": 5275
2407
+ },
2408
+ {
2409
+ "entropy": 0.7654093158245087,
2410
+ "epoch": 1.8346892850960708,
2411
+ "grad_norm": 2.21875,
2412
+ "learning_rate": 3.5837139031928315e-07,
2413
+ "loss": 0.7362342071533203,
2414
+ "mean_token_accuracy": 0.8066077816486359,
2415
+ "num_tokens": 32809688.0,
2416
+ "step": 5300
2417
+ },
2418
+ {
2419
+ "entropy": 0.7611633980274201,
2420
+ "epoch": 1.8433442963475852,
2421
+ "grad_norm": 2.25,
2422
+ "learning_rate": 3.221362568991293e-07,
2423
+ "loss": 0.7225106811523437,
2424
+ "mean_token_accuracy": 0.809012303352356,
2425
+ "num_tokens": 32962531.0,
2426
+ "step": 5325
2427
+ },
2428
+ {
2429
+ "entropy": 0.7749582767486572,
2430
+ "epoch": 1.8519993075991,
2431
+ "grad_norm": 2.28125,
2432
+ "learning_rate": 2.8780200874370165e-07,
2433
+ "loss": 0.7392816925048828,
2434
+ "mean_token_accuracy": 0.803524352312088,
2435
+ "num_tokens": 33122116.0,
2436
+ "step": 5350
2437
+ },
2438
+ {
2439
+ "entropy": 0.7610144579410553,
2440
+ "epoch": 1.8606543188506146,
2441
+ "grad_norm": 2.28125,
2442
+ "learning_rate": 2.5537538964151676e-07,
2443
+ "loss": 0.7229084014892578,
2444
+ "mean_token_accuracy": 0.8083787798881531,
2445
+ "num_tokens": 33273413.0,
2446
+ "step": 5375
2447
+ },
2448
+ {
2449
+ "entropy": 0.791879060268402,
2450
+ "epoch": 1.869309330102129,
2451
+ "grad_norm": 2.234375,
2452
+ "learning_rate": 2.248627686926219e-07,
2453
+ "loss": 0.7606393432617188,
2454
+ "mean_token_accuracy": 0.8025156807899475,
2455
+ "num_tokens": 33421728.0,
2456
+ "step": 5400
2457
+ },
2458
+ {
2459
+ "epoch": 1.869309330102129,
2460
+ "eval_entropy": 0.770491715630547,
2461
+ "eval_loss": 0.7991881370544434,
2462
+ "eval_mean_token_accuracy": 0.7935233919346919,
2463
+ "eval_num_tokens": 33421728.0,
2464
+ "eval_runtime": 16.2639,
2465
+ "eval_samples_per_second": 299.128,
2466
+ "eval_steps_per_second": 18.753,
2467
+ "step": 5400
2468
+ },
2469
+ {
2470
+ "entropy": 0.7707629156112671,
2471
+ "epoch": 1.8779643413536438,
2472
+ "grad_norm": 2.265625,
2473
+ "learning_rate": 1.9627013905760605e-07,
2474
+ "loss": 0.7430142974853515,
2475
+ "mean_token_accuracy": 0.8047948408126832,
2476
+ "num_tokens": 33576359.0,
2477
+ "step": 5425
2478
+ },
2479
+ {
2480
+ "entropy": 0.7577464747428894,
2481
+ "epoch": 1.8866193526051585,
2482
+ "grad_norm": 2.171875,
2483
+ "learning_rate": 1.6960311678044018e-07,
2484
+ "loss": 0.7168014526367188,
2485
+ "mean_token_accuracy": 0.8094685411453247,
2486
+ "num_tokens": 33723770.0,
2487
+ "step": 5450
2488
+ },
2489
+ {
2490
+ "entropy": 0.7747404301166534,
2491
+ "epoch": 1.895274363856673,
2492
+ "grad_norm": 2.0625,
2493
+ "learning_rate": 1.4486693968540743e-07,
2494
+ "loss": 0.7389591979980469,
2495
+ "mean_token_accuracy": 0.8045553112030029,
2496
+ "num_tokens": 33879362.0,
2497
+ "step": 5475
2498
+ },
2499
+ {
2500
+ "entropy": 0.7680295896530152,
2501
+ "epoch": 1.9039293751081876,
2502
+ "grad_norm": 2.046875,
2503
+ "learning_rate": 1.22066466348304e-07,
2504
+ "loss": 0.7351342010498046,
2505
+ "mean_token_accuracy": 0.8047092247009278,
2506
+ "num_tokens": 34035862.0,
2507
+ "step": 5500
2508
+ },
2509
+ {
2510
+ "entropy": 0.7560386550426483,
2511
+ "epoch": 1.9125843863597023,
2512
+ "grad_norm": 2.15625,
2513
+ "learning_rate": 1.0120617514214115e-07,
2514
+ "loss": 0.7233296966552735,
2515
+ "mean_token_accuracy": 0.8088453054428101,
2516
+ "num_tokens": 34192367.0,
2517
+ "step": 5525
2518
+ },
2519
+ {
2520
+ "entropy": 0.749894061088562,
2521
+ "epoch": 1.9212393976112168,
2522
+ "grad_norm": 2.234375,
2523
+ "learning_rate": 8.229016335752238e-08,
2524
+ "loss": 0.721561279296875,
2525
+ "mean_token_accuracy": 0.8098326814174652,
2526
+ "num_tokens": 34346639.0,
2527
+ "step": 5550
2528
+ },
2529
+ {
2530
+ "entropy": 0.7512237560749054,
2531
+ "epoch": 1.9298944088627317,
2532
+ "grad_norm": 2.203125,
2533
+ "learning_rate": 6.532214639786926e-08,
2534
+ "loss": 0.7133358001708985,
2535
+ "mean_token_accuracy": 0.8117510068416596,
2536
+ "num_tokens": 34501969.0,
2537
+ "step": 5575
2538
+ },
2539
+ {
2540
+ "entropy": 0.7577315938472747,
2541
+ "epoch": 1.9385494201142461,
2542
+ "grad_norm": 1.984375,
2543
+ "learning_rate": 5.03054570496575e-08,
2544
+ "loss": 0.7302328491210938,
2545
+ "mean_token_accuracy": 0.8082369923591614,
2546
+ "num_tokens": 34658561.0,
2547
+ "step": 5600
2548
+ },
2549
+ {
2550
+ "epoch": 1.9385494201142461,
2551
+ "eval_entropy": 0.7700900946484237,
2552
+ "eval_loss": 0.7992120981216431,
2553
+ "eval_mean_token_accuracy": 0.7935370582049011,
2554
+ "eval_num_tokens": 34658561.0,
2555
+ "eval_runtime": 16.284,
2556
+ "eval_samples_per_second": 298.76,
2557
+ "eval_steps_per_second": 18.73,
2558
+ "step": 5600
2559
+ },
2560
+ {
2561
+ "entropy": 0.7591597294807434,
2562
+ "epoch": 1.9472044313657608,
2563
+ "grad_norm": 2.203125,
2564
+ "learning_rate": 3.724304482781049e-08,
2565
+ "loss": 0.7255685424804688,
2566
+ "mean_token_accuracy": 0.808221675157547,
2567
+ "num_tokens": 34809418.0,
2568
+ "step": 5625
2569
+ },
2570
+ {
2571
+ "entropy": 0.7517114734649658,
2572
+ "epoch": 1.9558594426172755,
2573
+ "grad_norm": 2.078125,
2574
+ "learning_rate": 2.613747539635725e-08,
2575
+ "loss": 0.7198385620117187,
2576
+ "mean_token_accuracy": 0.8097265243530274,
2577
+ "num_tokens": 34958876.0,
2578
+ "step": 5650
2579
+ },
2580
+ {
2581
+ "entropy": 0.7789162063598633,
2582
+ "epoch": 1.96451445386879,
2583
+ "grad_norm": 2.15625,
2584
+ "learning_rate": 1.6990930064507694e-08,
2585
+ "loss": 0.7432637786865235,
2586
+ "mean_token_accuracy": 0.8060086870193481,
2587
+ "num_tokens": 35115995.0,
2588
+ "step": 5675
2589
+ },
2590
+ {
2591
+ "entropy": 0.7361849272251129,
2592
+ "epoch": 1.9731694651203047,
2593
+ "grad_norm": 2.015625,
2594
+ "learning_rate": 9.805205358197622e-09,
2595
+ "loss": 0.7050675201416016,
2596
+ "mean_token_accuracy": 0.8122973215579986,
2597
+ "num_tokens": 35274445.0,
2598
+ "step": 5700
2599
+ },
2600
+ {
2601
+ "entropy": 0.7350089454650879,
2602
+ "epoch": 1.9818244763718194,
2603
+ "grad_norm": 2.15625,
2604
+ "learning_rate": 4.5817126672331735e-09,
2605
+ "loss": 0.697210922241211,
2606
+ "mean_token_accuracy": 0.816340823173523,
2607
+ "num_tokens": 35428606.0,
2608
+ "step": 5725
2609
+ },
2610
+ {
2611
+ "entropy": 0.7550158965587616,
2612
+ "epoch": 1.9904794876233338,
2613
+ "grad_norm": 2.171875,
2614
+ "learning_rate": 1.3214779680637002e-09,
2615
+ "loss": 0.71933349609375,
2616
+ "mean_token_accuracy": 0.8101057040691376,
2617
+ "num_tokens": 35582057.0,
2618
+ "step": 5750
2619
+ },
2620
+ {
2621
+ "entropy": 0.7830293524265289,
2622
+ "epoch": 1.9991344988748485,
2623
+ "grad_norm": 2.171875,
2624
+ "learning_rate": 2.514162226741057e-11,
2625
+ "loss": 0.7570413970947265,
2626
+ "mean_token_accuracy": 0.8038502883911133,
2627
+ "num_tokens": 35740061.0,
2628
+ "step": 5775
2629
+ },
2630
+ {
2631
+ "epoch": 2.0,
2632
+ "eval_entropy": 0.7710395187628074,
2633
+ "eval_loss": 0.7991825342178345,
2634
+ "eval_mean_token_accuracy": 0.7934717604371367,
2635
+ "eval_num_tokens": 35756220.0,
2636
+ "eval_runtime": 16.2915,
2637
+ "eval_samples_per_second": 298.623,
2638
+ "eval_steps_per_second": 18.721,
2639
+ "step": 5778
2640
+ }
2641
+ ],
2642
+ "logging_steps": 25,
2643
+ "max_steps": 5778,
2644
+ "num_input_tokens_seen": 0,
2645
+ "num_train_epochs": 2,
2646
+ "save_steps": 200,
2647
+ "stateful_callbacks": {
2648
+ "TrainerControl": {
2649
+ "args": {
2650
+ "should_epoch_stop": false,
2651
+ "should_evaluate": false,
2652
+ "should_log": false,
2653
+ "should_save": true,
2654
+ "should_training_stop": true
2655
+ },
2656
+ "attributes": {}
2657
+ }
2658
+ },
2659
+ "total_flos": 1.4611218403146547e+17,
2660
+ "train_batch_size": 16,
2661
+ "trial_name": null,
2662
+ "trial_params": null
2663
+ }
checkpoint-5778/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fe1d1ac135572b0e564d6340370a449cb9281b7839efca5df924e60c2ca5dbe
3
+ size 5585
config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2ForCausalLM"
4
+ ],
5
+ "block_auto_adjust_ff_dim": true,
6
+ "block_dim": 1024,
7
+ "block_ffn_dim_multiplier": 1.0,
8
+ "block_mlp_init_scale": 1.0,
9
+ "block_multiple_of": 256,
10
+ "block_norm_eps": 1e-05,
11
+ "block_out_init_scale": 1.0,
12
+ "block_use_swiglu": true,
13
+ "block_use_xavier_init": true,
14
+ "bos_token_id": 1,
15
+ "conv_L_cache": 3,
16
+ "conv_bias": false,
17
+ "conv_dim": 1024,
18
+ "conv_use_xavier_init": true,
19
+ "dtype": "bfloat16",
20
+ "eos_token_id": 7,
21
+ "full_attn_idxs": null,
22
+ "hidden_size": 1024,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 6656,
25
+ "layer_types": [
26
+ "conv",
27
+ "conv",
28
+ "full_attention",
29
+ "conv",
30
+ "conv",
31
+ "full_attention",
32
+ "conv",
33
+ "conv",
34
+ "full_attention",
35
+ "conv",
36
+ "full_attention",
37
+ "conv",
38
+ "full_attention",
39
+ "conv",
40
+ "full_attention",
41
+ "conv"
42
+ ],
43
+ "max_position_embeddings": 128000,
44
+ "model_type": "lfm2",
45
+ "norm_eps": 1e-05,
46
+ "num_attention_heads": 16,
47
+ "num_heads": 16,
48
+ "num_hidden_layers": 16,
49
+ "num_key_value_heads": 8,
50
+ "pad_token_id": 0,
51
+ "rope_parameters": {
52
+ "rope_theta": 1000000.0,
53
+ "rope_type": "default"
54
+ },
55
+ "tie_word_embeddings": true,
56
+ "transformers_version": "5.5.0",
57
+ "use_cache": false,
58
+ "use_pos_enc": true,
59
+ "vocab_size": 65536
60
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 7
6
+ ],
7
+ "pad_token_id": 0,
8
+ "transformers_version": "5.5.0"
9
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3eaee205a112143b29e0fc07a362b9bd5068d377cd2d96bf537ab645b8e19098
3
+ size 708984464
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|im_end|>",
6
+ "extra_special_tokens": [],
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "model_input_names": [
10
+ "input_ids",
11
+ "attention_mask"
12
+ ],
13
+ "model_max_length": 1000000000000000019884624838656,
14
+ "model_specific_special_tokens": {},
15
+ "pad_token": "<|pad|>",
16
+ "sp_model_kwargs": {},
17
+ "spaces_between_special_tokens": false,
18
+ "tokenizer_class": "TokenizersBackend",
19
+ "use_default_system_prompt": false,
20
+ "use_fast": true
21
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fe1d1ac135572b0e564d6340370a449cb9281b7839efca5df924e60c2ca5dbe
3
+ size 5585