Hugging Face's logo

Maxtimer97
/
LLamaInstruct2Hymba_SFT_wd_lr5e5

Safetensors
hymba
custom_code
Model card Files
xet
 Community
Maxtimer97 commited on
Commit
c7c0331
·
verified ·
1 Parent(s): 58f8fa3

Initial upload of fine-tuned model

Browse files
Files changed (15) hide show
  1. .gitattributes +1 -0
  2. chat_template.jinja +21 -0
  3. config.json +133 -0
  4. configuration_hymba.py +125 -0
  5. generation_config.json +11 -0
  6. model.safetensors +3 -0
  7. modeling_hymba.py +0 -0
  8. optimizer.pt +3 -0
  9. rng_state.pth +3 -0
  10. scheduler.pt +3 -0
  11. special_tokens_map.json +17 -0
  12. tokenizer.json +3 -0
  13. tokenizer_config.json +2063 -0
  14. trainer_state.json +3346 -0
  15. training_args.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ {{- bos_token }}
3
+ {%- for message in messages %}
4
+ {%- if message['role'] == 'assistant' %}
5
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
6
+ ' }}
7
+ {%- generation %}
8
+ {{- message['content'] + '<|eot_id|>' }}
9
+ {%- endgeneration %}
10
+ {{- '
11
+ ' }}
12
+ {%- else %}
13
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
14
+ ' + message['content'] + '<|eot_id|>' + '
15
+ ' }}
16
+ {%- endif %}
17
+ {%- endfor %}
18
+ {%- if add_generation_prompt %}
19
+ {{- '<|start_header_id|>assistant<|end_header_id|>
20
+ ' }}
21
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "HymbaForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "attn_factor": 0.5,
7
+ "attn_hidden_size": 1024,
8
+ "attn_implementation": "flash_attention_2",
9
+ "attn_implementation_new": "flash_attention_2",
10
+ "attn_only_wo_proj": true,
11
+ "auto_map": {
12
+ "AutoConfig": "configuration_hymba.HymbaConfig",
13
+ "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
14
+ },
15
+ "bos_token_id": 128000,
16
+ "calc_logits_for_entire_prompt": false,
17
+ "conv_dim": {
18
+ "0": 256,
19
+ "1": 256,
20
+ "10": 256,
21
+ "11": 256,
22
+ "12": 256,
23
+ "13": 256,
24
+ "14": 256,
25
+ "15": 256,
26
+ "16": 3200,
27
+ "17": 3200,
28
+ "18": 3200,
29
+ "19": 3200,
30
+ "2": 256,
31
+ "20": 3200,
32
+ "21": 3200,
33
+ "22": 3200,
34
+ "23": 3200,
35
+ "24": 3200,
36
+ "25": 3200,
37
+ "26": 3200,
38
+ "27": 3200,
39
+ "28": 3200,
40
+ "29": 3200,
41
+ "3": 256,
42
+ "30": 3200,
43
+ "31": 3200,
44
+ "4": 256,
45
+ "5": 256,
46
+ "6": 256,
47
+ "7": 256,
48
+ "8": 256,
49
+ "9": 256
50
+ },
51
+ "eos_token_id": [
52
+ 128001,
53
+ 128008,
54
+ 128009
55
+ ],
56
+ "global_attn_idx": [
57
+ 0,
58
+ 8,
59
+ 15
60
+ ],
61
+ "head_dim": 64,
62
+ "hidden_act": "silu",
63
+ "hidden_size": 2048,
64
+ "initializer_range": 0.02,
65
+ "intermediate_size": 8192,
66
+ "kq_head_dim": -1,
67
+ "kq_norm": "none",
68
+ "kv_reuse_every_i_layer": -1,
69
+ "kv_reuse_group": null,
70
+ "kv_weight_reuse": false,
71
+ "layer_type": [
72
+ "h",
73
+ "h",
74
+ "h",
75
+ "h",
76
+ "h",
77
+ "h",
78
+ "h",
79
+ "h",
80
+ "h",
81
+ "h",
82
+ "h",
83
+ "h",
84
+ "h",
85
+ "h",
86
+ "h",
87
+ "h"
88
+ ],
89
+ "mamba_conv_bias": true,
90
+ "mamba_d_conv": 4,
91
+ "mamba_d_state": 16,
92
+ "mamba_dt_rank": 100,
93
+ "mamba_expand": 1,
94
+ "mamba_inner_layernorms": false,
95
+ "mamba_proj_bias": false,
96
+ "max_position_embeddings": 131072,
97
+ "memory_tokens_interspersed_every": 0,
98
+ "mlp_hidden_act": "silu",
99
+ "model_type": "hymba",
100
+ "num_attention_heads": 16,
101
+ "num_experts": 1,
102
+ "num_experts_per_tok": 1,
103
+ "num_hidden_layers": 16,
104
+ "num_key_value_heads": 4,
105
+ "num_mamba": 1,
106
+ "num_memory_tokens": 0,
107
+ "orig_max_position_embeddings": 2048,
108
+ "output_router_logits": false,
109
+ "pad_token_id": null,
110
+ "pure_attn": false,
111
+ "repeat_x_before_conv": true,
112
+ "rms_norm_eps": 1e-05,
113
+ "rope": true,
114
+ "rope_scaling": {
115
+ "factor": 32.0,
116
+ "high_freq_factor": 4.0,
117
+ "low_freq_factor": 1.0,
118
+ "original_max_position_embeddings": 8192,
119
+ "rope_type": "llama3"
120
+ },
121
+ "rope_theta": 500000.0,
122
+ "rope_type": "ntk",
123
+ "router_aux_loss_coef": 0.001,
124
+ "seq_length": 8192,
125
+ "sliding_window": 1024,
126
+ "tie_word_embeddings": true,
127
+ "torch_dtype": "bfloat16",
128
+ "transformers_version": "4.53.0",
129
+ "use_cache": false,
130
+ "use_mamba_kernels": true,
131
+ "v_head_dim": 64,
132
+ "vocab_size": 128256
133
+ }
configuration_hymba.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from transformers.configuration_utils import PretrainedConfig
3
+
4
+
5
+ class HymbaConfig(PretrainedConfig):
6
+
7
+ model_type = "hymba"
8
+ keys_to_ignore_at_inference = ["past_key_values"]
9
+
10
+ def __init__(
11
+ self,
12
+ vocab_size=65536,
13
+ tie_word_embeddings=False,
14
+ hidden_size=4096,
15
+ intermediate_size=14336,
16
+ num_hidden_layers=32,
17
+ num_attention_heads=32,
18
+ num_key_value_heads=8,
19
+ hidden_act="silu",
20
+ initializer_range=0.02,
21
+ rms_norm_eps=1e-6,
22
+ attn_only_wo_proj=False,
23
+ use_cache=True,
24
+ calc_logits_for_entire_prompt=False,
25
+ output_router_logits=False,
26
+ router_aux_loss_coef=0.001,
27
+ pad_token_id=0,
28
+ bos_token_id=1,
29
+ eos_token_id=2,
30
+ sliding_window=None,
31
+ max_position_embeddings=262144,
32
+ orig_max_position_embeddings=None,
33
+ attention_dropout=0.0,
34
+ num_experts_per_tok=2,
35
+ num_experts=16,
36
+ use_mamba_kernels=True,
37
+ mamba_d_state=16,
38
+ mamba_d_conv=4,
39
+ mamba_expand=2,
40
+ mamba_dt_rank="auto",
41
+ mamba_conv_bias=True,
42
+ mamba_proj_bias=False,
43
+ mamba_inner_layernorms=True,
44
+ kv_reuse_every_i_layer=-1,
45
+ kv_reuse_group=None,
46
+ kv_weight_reuse=False,
47
+ global_attn_idx=None,
48
+ num_mamba=1,
49
+ pure_attn=False,
50
+ repeat_x_before_conv=True,
51
+ attn_implementation_new='sdpa',
52
+ rope_type=None,
53
+ attn_factor=0.5,
54
+ **kwargs,
55
+ ):
56
+ self.vocab_size = vocab_size
57
+ self.tie_word_embeddings = tie_word_embeddings
58
+ self.hidden_size = hidden_size
59
+ self.intermediate_size = intermediate_size
60
+ self.num_hidden_layers = num_hidden_layers
61
+ self.num_attention_heads = num_attention_heads
62
+ self.sliding_window = sliding_window
63
+ self.max_position_embeddings = max_position_embeddings
64
+ self.orig_max_position_embeddings = orig_max_position_embeddings
65
+ self.attention_dropout = attention_dropout
66
+ self.attn_only_wo_proj = attn_only_wo_proj
67
+
68
+ if num_key_value_heads is None:
69
+ num_key_value_heads = num_attention_heads
70
+
71
+ self.num_key_value_heads = num_key_value_heads
72
+ self.hidden_act = hidden_act
73
+ self.initializer_range = initializer_range
74
+ self.rms_norm_eps = rms_norm_eps
75
+
76
+ self.use_cache = use_cache
77
+ self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
78
+ self.output_router_logits = output_router_logits
79
+ self.router_aux_loss_coef = router_aux_loss_coef
80
+
81
+ self.num_experts_per_tok = num_experts_per_tok
82
+ self.num_experts = num_experts
83
+
84
+ self.use_mamba_kernels = use_mamba_kernels
85
+ self.mamba_d_state = mamba_d_state
86
+ self.mamba_d_conv = mamba_d_conv
87
+ self.mamba_expand = mamba_expand
88
+ self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
89
+ self.mamba_conv_bias = mamba_conv_bias
90
+ self.mamba_proj_bias = mamba_proj_bias
91
+ self.mamba_inner_layernorms = mamba_inner_layernorms
92
+
93
+ self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
94
+ self.attn_factor = attn_factor
95
+ self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
96
+ self.v_head_dim = kwargs.pop("v_head_dim", -1)
97
+ self.kq_norm = kwargs.pop("kq_norm", None)
98
+ self.rope = kwargs.pop("rope", False)
99
+ self.rope_theta = kwargs.pop("rope_theta", 10000.0)
100
+ self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
101
+ self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
102
+
103
+ self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
104
+ self.kv_reuse_group = kv_reuse_group
105
+ self.kv_weight_reuse = kv_weight_reuse
106
+
107
+ self.global_attn_idx = global_attn_idx
108
+
109
+ self.num_mamba = num_mamba
110
+
111
+ self.attn_implementation_new = attn_implementation_new
112
+
113
+ self.rope_type = rope_type
114
+
115
+ self.pure_attn = pure_attn
116
+
117
+ self.repeat_x_before_conv = repeat_x_before_conv
118
+
119
+ super().__init__(
120
+ pad_token_id=pad_token_id,
121
+ bos_token_id=bos_token_id,
122
+ eos_token_id=eos_token_id,
123
+ tie_word_embeddings=tie_word_embeddings,
124
+ **kwargs,
125
+ )
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "transformers_version": "4.53.0",
10
+ "use_cache": false
11
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:110de04ca22612d8cbf8c8215217b42611c96e332fc50d9ec2007afbbc4586c1
3
+ size 2549869520
modeling_hymba.py ADDED
The diff for this file is too large to render. See raw diff
 
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:350267335cac03325274a32d4357daa4361e1e8856bcbe8f0e80ddcd81a87187
3
+ size 5099871810
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4846267e0a17ad491353c9e36f2e436fbe4fe2b2cfcb7043301852b07efd9cf
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|begin_of_text|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|eot_id|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<|eot_id|>"
17
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
3
+ size 17209920
tokenizer_config.json ADDED
@@ -0,0 +1,2063 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "128000": {
4
+ "content": "<|begin_of_text|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "128001": {
12
+ "content": "<|end_of_text|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "128002": {
20
+ "content": "<|reserved_special_token_0|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "128003": {
28
+ "content": "<|reserved_special_token_1|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "128004": {
36
+ "content": "<|finetune_right_pad_id|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "128005": {
44
+ "content": "<|reserved_special_token_2|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "128006": {
52
+ "content": "<|start_header_id|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "128007": {
60
+ "content": "<|end_header_id|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "128008": {
68
+ "content": "<|eom_id|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "128009": {
76
+ "content": "<|eot_id|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "128010": {
84
+ "content": "<|python_tag|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "128011": {
92
+ "content": "<|reserved_special_token_3|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "128012": {
100
+ "content": "<|reserved_special_token_4|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "128013": {
108
+ "content": "<|reserved_special_token_5|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "128014": {
116
+ "content": "<|reserved_special_token_6|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "128015": {
124
+ "content": "<|reserved_special_token_7|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "128016": {
132
+ "content": "<|reserved_special_token_8|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "128017": {
140
+ "content": "<|reserved_special_token_9|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "128018": {
148
+ "content": "<|reserved_special_token_10|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "128019": {
156
+ "content": "<|reserved_special_token_11|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "128020": {
164
+ "content": "<|reserved_special_token_12|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "128021": {
172
+ "content": "<|reserved_special_token_13|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "128022": {
180
+ "content": "<|reserved_special_token_14|>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "128023": {
188
+ "content": "<|reserved_special_token_15|>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "128024": {
196
+ "content": "<|reserved_special_token_16|>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "128025": {
204
+ "content": "<|reserved_special_token_17|>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "128026": {
212
+ "content": "<|reserved_special_token_18|>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "128027": {
220
+ "content": "<|reserved_special_token_19|>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "128028": {
228
+ "content": "<|reserved_special_token_20|>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "128029": {
236
+ "content": "<|reserved_special_token_21|>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "128030": {
244
+ "content": "<|reserved_special_token_22|>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "128031": {
252
+ "content": "<|reserved_special_token_23|>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "128032": {
260
+ "content": "<|reserved_special_token_24|>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "128033": {
268
+ "content": "<|reserved_special_token_25|>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "128034": {
276
+ "content": "<|reserved_special_token_26|>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "128035": {
284
+ "content": "<|reserved_special_token_27|>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "128036": {
292
+ "content": "<|reserved_special_token_28|>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "128037": {
300
+ "content": "<|reserved_special_token_29|>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "128038": {
308
+ "content": "<|reserved_special_token_30|>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "128039": {
316
+ "content": "<|reserved_special_token_31|>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "128040": {
324
+ "content": "<|reserved_special_token_32|>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "128041": {
332
+ "content": "<|reserved_special_token_33|>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "128042": {
340
+ "content": "<|reserved_special_token_34|>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "128043": {
348
+ "content": "<|reserved_special_token_35|>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "128044": {
356
+ "content": "<|reserved_special_token_36|>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "128045": {
364
+ "content": "<|reserved_special_token_37|>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "128046": {
372
+ "content": "<|reserved_special_token_38|>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "128047": {
380
+ "content": "<|reserved_special_token_39|>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "128048": {
388
+ "content": "<|reserved_special_token_40|>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "128049": {
396
+ "content": "<|reserved_special_token_41|>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "128050": {
404
+ "content": "<|reserved_special_token_42|>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "128051": {
412
+ "content": "<|reserved_special_token_43|>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "128052": {
420
+ "content": "<|reserved_special_token_44|>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "128053": {
428
+ "content": "<|reserved_special_token_45|>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "128054": {
436
+ "content": "<|reserved_special_token_46|>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "128055": {
444
+ "content": "<|reserved_special_token_47|>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "128056": {
452
+ "content": "<|reserved_special_token_48|>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "128057": {
460
+ "content": "<|reserved_special_token_49|>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "128058": {
468
+ "content": "<|reserved_special_token_50|>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "128059": {
476
+ "content": "<|reserved_special_token_51|>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "128060": {
484
+ "content": "<|reserved_special_token_52|>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "128061": {
492
+ "content": "<|reserved_special_token_53|>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "128062": {
500
+ "content": "<|reserved_special_token_54|>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "128063": {
508
+ "content": "<|reserved_special_token_55|>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "128064": {
516
+ "content": "<|reserved_special_token_56|>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "128065": {
524
+ "content": "<|reserved_special_token_57|>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "128066": {
532
+ "content": "<|reserved_special_token_58|>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "128067": {
540
+ "content": "<|reserved_special_token_59|>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "128068": {
548
+ "content": "<|reserved_special_token_60|>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "128069": {
556
+ "content": "<|reserved_special_token_61|>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "128070": {
564
+ "content": "<|reserved_special_token_62|>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "128071": {
572
+ "content": "<|reserved_special_token_63|>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "128072": {
580
+ "content": "<|reserved_special_token_64|>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "128073": {
588
+ "content": "<|reserved_special_token_65|>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "128074": {
596
+ "content": "<|reserved_special_token_66|>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "128075": {
604
+ "content": "<|reserved_special_token_67|>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "128076": {
612
+ "content": "<|reserved_special_token_68|>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "128077": {
620
+ "content": "<|reserved_special_token_69|>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "128078": {
628
+ "content": "<|reserved_special_token_70|>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "128079": {
636
+ "content": "<|reserved_special_token_71|>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "128080": {
644
+ "content": "<|reserved_special_token_72|>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "128081": {
652
+ "content": "<|reserved_special_token_73|>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "128082": {
660
+ "content": "<|reserved_special_token_74|>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "128083": {
668
+ "content": "<|reserved_special_token_75|>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "128084": {
676
+ "content": "<|reserved_special_token_76|>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "128085": {
684
+ "content": "<|reserved_special_token_77|>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "128086": {
692
+ "content": "<|reserved_special_token_78|>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "128087": {
700
+ "content": "<|reserved_special_token_79|>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "128088": {
708
+ "content": "<|reserved_special_token_80|>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "128089": {
716
+ "content": "<|reserved_special_token_81|>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "128090": {
724
+ "content": "<|reserved_special_token_82|>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "128091": {
732
+ "content": "<|reserved_special_token_83|>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "128092": {
740
+ "content": "<|reserved_special_token_84|>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "128093": {
748
+ "content": "<|reserved_special_token_85|>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "128094": {
756
+ "content": "<|reserved_special_token_86|>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "128095": {
764
+ "content": "<|reserved_special_token_87|>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "128096": {
772
+ "content": "<|reserved_special_token_88|>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "128097": {
780
+ "content": "<|reserved_special_token_89|>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "128098": {
788
+ "content": "<|reserved_special_token_90|>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "128099": {
796
+ "content": "<|reserved_special_token_91|>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "128100": {
804
+ "content": "<|reserved_special_token_92|>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "128101": {
812
+ "content": "<|reserved_special_token_93|>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "128102": {
820
+ "content": "<|reserved_special_token_94|>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "128103": {
828
+ "content": "<|reserved_special_token_95|>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "128104": {
836
+ "content": "<|reserved_special_token_96|>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "128105": {
844
+ "content": "<|reserved_special_token_97|>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "128106": {
852
+ "content": "<|reserved_special_token_98|>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "128107": {
860
+ "content": "<|reserved_special_token_99|>",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "128108": {
868
+ "content": "<|reserved_special_token_100|>",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "128109": {
876
+ "content": "<|reserved_special_token_101|>",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "128110": {
884
+ "content": "<|reserved_special_token_102|>",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "128111": {
892
+ "content": "<|reserved_special_token_103|>",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "128112": {
900
+ "content": "<|reserved_special_token_104|>",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "128113": {
908
+ "content": "<|reserved_special_token_105|>",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "128114": {
916
+ "content": "<|reserved_special_token_106|>",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "128115": {
924
+ "content": "<|reserved_special_token_107|>",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "128116": {
932
+ "content": "<|reserved_special_token_108|>",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "128117": {
940
+ "content": "<|reserved_special_token_109|>",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "128118": {
948
+ "content": "<|reserved_special_token_110|>",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "128119": {
956
+ "content": "<|reserved_special_token_111|>",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "128120": {
964
+ "content": "<|reserved_special_token_112|>",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "128121": {
972
+ "content": "<|reserved_special_token_113|>",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "128122": {
980
+ "content": "<|reserved_special_token_114|>",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "128123": {
988
+ "content": "<|reserved_special_token_115|>",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "128124": {
996
+ "content": "<|reserved_special_token_116|>",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "128125": {
1004
+ "content": "<|reserved_special_token_117|>",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "128126": {
1012
+ "content": "<|reserved_special_token_118|>",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "128127": {
1020
+ "content": "<|reserved_special_token_119|>",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ },
1027
+ "128128": {
1028
+ "content": "<|reserved_special_token_120|>",
1029
+ "lstrip": false,
1030
+ "normalized": false,
1031
+ "rstrip": false,
1032
+ "single_word": false,
1033
+ "special": true
1034
+ },
1035
+ "128129": {
1036
+ "content": "<|reserved_special_token_121|>",
1037
+ "lstrip": false,
1038
+ "normalized": false,
1039
+ "rstrip": false,
1040
+ "single_word": false,
1041
+ "special": true
1042
+ },
1043
+ "128130": {
1044
+ "content": "<|reserved_special_token_122|>",
1045
+ "lstrip": false,
1046
+ "normalized": false,
1047
+ "rstrip": false,
1048
+ "single_word": false,
1049
+ "special": true
1050
+ },
1051
+ "128131": {
1052
+ "content": "<|reserved_special_token_123|>",
1053
+ "lstrip": false,
1054
+ "normalized": false,
1055
+ "rstrip": false,
1056
+ "single_word": false,
1057
+ "special": true
1058
+ },
1059
+ "128132": {
1060
+ "content": "<|reserved_special_token_124|>",
1061
+ "lstrip": false,
1062
+ "normalized": false,
1063
+ "rstrip": false,
1064
+ "single_word": false,
1065
+ "special": true
1066
+ },
1067
+ "128133": {
1068
+ "content": "<|reserved_special_token_125|>",
1069
+ "lstrip": false,
1070
+ "normalized": false,
1071
+ "rstrip": false,
1072
+ "single_word": false,
1073
+ "special": true
1074
+ },
1075
+ "128134": {
1076
+ "content": "<|reserved_special_token_126|>",
1077
+ "lstrip": false,
1078
+ "normalized": false,
1079
+ "rstrip": false,
1080
+ "single_word": false,
1081
+ "special": true
1082
+ },
1083
+ "128135": {
1084
+ "content": "<|reserved_special_token_127|>",
1085
+ "lstrip": false,
1086
+ "normalized": false,
1087
+ "rstrip": false,
1088
+ "single_word": false,
1089
+ "special": true
1090
+ },
1091
+ "128136": {
1092
+ "content": "<|reserved_special_token_128|>",
1093
+ "lstrip": false,
1094
+ "normalized": false,
1095
+ "rstrip": false,
1096
+ "single_word": false,
1097
+ "special": true
1098
+ },
1099
+ "128137": {
1100
+ "content": "<|reserved_special_token_129|>",
1101
+ "lstrip": false,
1102
+ "normalized": false,
1103
+ "rstrip": false,
1104
+ "single_word": false,
1105
+ "special": true
1106
+ },
1107
+ "128138": {
1108
+ "content": "<|reserved_special_token_130|>",
1109
+ "lstrip": false,
1110
+ "normalized": false,
1111
+ "rstrip": false,
1112
+ "single_word": false,
1113
+ "special": true
1114
+ },
1115
+ "128139": {
1116
+ "content": "<|reserved_special_token_131|>",
1117
+ "lstrip": false,
1118
+ "normalized": false,
1119
+ "rstrip": false,
1120
+ "single_word": false,
1121
+ "special": true
1122
+ },
1123
+ "128140": {
1124
+ "content": "<|reserved_special_token_132|>",
1125
+ "lstrip": false,
1126
+ "normalized": false,
1127
+ "rstrip": false,
1128
+ "single_word": false,
1129
+ "special": true
1130
+ },
1131
+ "128141": {
1132
+ "content": "<|reserved_special_token_133|>",
1133
+ "lstrip": false,
1134
+ "normalized": false,
1135
+ "rstrip": false,
1136
+ "single_word": false,
1137
+ "special": true
1138
+ },
1139
+ "128142": {
1140
+ "content": "<|reserved_special_token_134|>",
1141
+ "lstrip": false,
1142
+ "normalized": false,
1143
+ "rstrip": false,
1144
+ "single_word": false,
1145
+ "special": true
1146
+ },
1147
+ "128143": {
1148
+ "content": "<|reserved_special_token_135|>",
1149
+ "lstrip": false,
1150
+ "normalized": false,
1151
+ "rstrip": false,
1152
+ "single_word": false,
1153
+ "special": true
1154
+ },
1155
+ "128144": {
1156
+ "content": "<|reserved_special_token_136|>",
1157
+ "lstrip": false,
1158
+ "normalized": false,
1159
+ "rstrip": false,
1160
+ "single_word": false,
1161
+ "special": true
1162
+ },
1163
+ "128145": {
1164
+ "content": "<|reserved_special_token_137|>",
1165
+ "lstrip": false,
1166
+ "normalized": false,
1167
+ "rstrip": false,
1168
+ "single_word": false,
1169
+ "special": true
1170
+ },
1171
+ "128146": {
1172
+ "content": "<|reserved_special_token_138|>",
1173
+ "lstrip": false,
1174
+ "normalized": false,
1175
+ "rstrip": false,
1176
+ "single_word": false,
1177
+ "special": true
1178
+ },
1179
+ "128147": {
1180
+ "content": "<|reserved_special_token_139|>",
1181
+ "lstrip": false,
1182
+ "normalized": false,
1183
+ "rstrip": false,
1184
+ "single_word": false,
1185
+ "special": true
1186
+ },
1187
+ "128148": {
1188
+ "content": "<|reserved_special_token_140|>",
1189
+ "lstrip": false,
1190
+ "normalized": false,
1191
+ "rstrip": false,
1192
+ "single_word": false,
1193
+ "special": true
1194
+ },
1195
+ "128149": {
1196
+ "content": "<|reserved_special_token_141|>",
1197
+ "lstrip": false,
1198
+ "normalized": false,
1199
+ "rstrip": false,
1200
+ "single_word": false,
1201
+ "special": true
1202
+ },
1203
+ "128150": {
1204
+ "content": "<|reserved_special_token_142|>",
1205
+ "lstrip": false,
1206
+ "normalized": false,
1207
+ "rstrip": false,
1208
+ "single_word": false,
1209
+ "special": true
1210
+ },
1211
+ "128151": {
1212
+ "content": "<|reserved_special_token_143|>",
1213
+ "lstrip": false,
1214
+ "normalized": false,
1215
+ "rstrip": false,
1216
+ "single_word": false,
1217
+ "special": true
1218
+ },
1219
+ "128152": {
1220
+ "content": "<|reserved_special_token_144|>",
1221
+ "lstrip": false,
1222
+ "normalized": false,
1223
+ "rstrip": false,
1224
+ "single_word": false,
1225
+ "special": true
1226
+ },
1227
+ "128153": {
1228
+ "content": "<|reserved_special_token_145|>",
1229
+ "lstrip": false,
1230
+ "normalized": false,
1231
+ "rstrip": false,
1232
+ "single_word": false,
1233
+ "special": true
1234
+ },
1235
+ "128154": {
1236
+ "content": "<|reserved_special_token_146|>",
1237
+ "lstrip": false,
1238
+ "normalized": false,
1239
+ "rstrip": false,
1240
+ "single_word": false,
1241
+ "special": true
1242
+ },
1243
+ "128155": {
1244
+ "content": "<|reserved_special_token_147|>",
1245
+ "lstrip": false,
1246
+ "normalized": false,
1247
+ "rstrip": false,
1248
+ "single_word": false,
1249
+ "special": true
1250
+ },
1251
+ "128156": {
1252
+ "content": "<|reserved_special_token_148|>",
1253
+ "lstrip": false,
1254
+ "normalized": false,
1255
+ "rstrip": false,
1256
+ "single_word": false,
1257
+ "special": true
1258
+ },
1259
+ "128157": {
1260
+ "content": "<|reserved_special_token_149|>",
1261
+ "lstrip": false,
1262
+ "normalized": false,
1263
+ "rstrip": false,
1264
+ "single_word": false,
1265
+ "special": true
1266
+ },
1267
+ "128158": {
1268
+ "content": "<|reserved_special_token_150|>",
1269
+ "lstrip": false,
1270
+ "normalized": false,
1271
+ "rstrip": false,
1272
+ "single_word": false,
1273
+ "special": true
1274
+ },
1275
+ "128159": {
1276
+ "content": "<|reserved_special_token_151|>",
1277
+ "lstrip": false,
1278
+ "normalized": false,
1279
+ "rstrip": false,
1280
+ "single_word": false,
1281
+ "special": true
1282
+ },
1283
+ "128160": {
1284
+ "content": "<|reserved_special_token_152|>",
1285
+ "lstrip": false,
1286
+ "normalized": false,
1287
+ "rstrip": false,
1288
+ "single_word": false,
1289
+ "special": true
1290
+ },
1291
+ "128161": {
1292
+ "content": "<|reserved_special_token_153|>",
1293
+ "lstrip": false,
1294
+ "normalized": false,
1295
+ "rstrip": false,
1296
+ "single_word": false,
1297
+ "special": true
1298
+ },
1299
+ "128162": {
1300
+ "content": "<|reserved_special_token_154|>",
1301
+ "lstrip": false,
1302
+ "normalized": false,
1303
+ "rstrip": false,
1304
+ "single_word": false,
1305
+ "special": true
1306
+ },
1307
+ "128163": {
1308
+ "content": "<|reserved_special_token_155|>",
1309
+ "lstrip": false,
1310
+ "normalized": false,
1311
+ "rstrip": false,
1312
+ "single_word": false,
1313
+ "special": true
1314
+ },
1315
+ "128164": {
1316
+ "content": "<|reserved_special_token_156|>",
1317
+ "lstrip": false,
1318
+ "normalized": false,
1319
+ "rstrip": false,
1320
+ "single_word": false,
1321
+ "special": true
1322
+ },
1323
+ "128165": {
1324
+ "content": "<|reserved_special_token_157|>",
1325
+ "lstrip": false,
1326
+ "normalized": false,
1327
+ "rstrip": false,
1328
+ "single_word": false,
1329
+ "special": true
1330
+ },
1331
+ "128166": {
1332
+ "content": "<|reserved_special_token_158|>",
1333
+ "lstrip": false,
1334
+ "normalized": false,
1335
+ "rstrip": false,
1336
+ "single_word": false,
1337
+ "special": true
1338
+ },
1339
+ "128167": {
1340
+ "content": "<|reserved_special_token_159|>",
1341
+ "lstrip": false,
1342
+ "normalized": false,
1343
+ "rstrip": false,
1344
+ "single_word": false,
1345
+ "special": true
1346
+ },
1347
+ "128168": {
1348
+ "content": "<|reserved_special_token_160|>",
1349
+ "lstrip": false,
1350
+ "normalized": false,
1351
+ "rstrip": false,
1352
+ "single_word": false,
1353
+ "special": true
1354
+ },
1355
+ "128169": {
1356
+ "content": "<|reserved_special_token_161|>",
1357
+ "lstrip": false,
1358
+ "normalized": false,
1359
+ "rstrip": false,
1360
+ "single_word": false,
1361
+ "special": true
1362
+ },
1363
+ "128170": {
1364
+ "content": "<|reserved_special_token_162|>",
1365
+ "lstrip": false,
1366
+ "normalized": false,
1367
+ "rstrip": false,
1368
+ "single_word": false,
1369
+ "special": true
1370
+ },
1371
+ "128171": {
1372
+ "content": "<|reserved_special_token_163|>",
1373
+ "lstrip": false,
1374
+ "normalized": false,
1375
+ "rstrip": false,
1376
+ "single_word": false,
1377
+ "special": true
1378
+ },
1379
+ "128172": {
1380
+ "content": "<|reserved_special_token_164|>",
1381
+ "lstrip": false,
1382
+ "normalized": false,
1383
+ "rstrip": false,
1384
+ "single_word": false,
1385
+ "special": true
1386
+ },
1387
+ "128173": {
1388
+ "content": "<|reserved_special_token_165|>",
1389
+ "lstrip": false,
1390
+ "normalized": false,
1391
+ "rstrip": false,
1392
+ "single_word": false,
1393
+ "special": true
1394
+ },
1395
+ "128174": {
1396
+ "content": "<|reserved_special_token_166|>",
1397
+ "lstrip": false,
1398
+ "normalized": false,
1399
+ "rstrip": false,
1400
+ "single_word": false,
1401
+ "special": true
1402
+ },
1403
+ "128175": {
1404
+ "content": "<|reserved_special_token_167|>",
1405
+ "lstrip": false,
1406
+ "normalized": false,
1407
+ "rstrip": false,
1408
+ "single_word": false,
1409
+ "special": true
1410
+ },
1411
+ "128176": {
1412
+ "content": "<|reserved_special_token_168|>",
1413
+ "lstrip": false,
1414
+ "normalized": false,
1415
+ "rstrip": false,
1416
+ "single_word": false,
1417
+ "special": true
1418
+ },
1419
+ "128177": {
1420
+ "content": "<|reserved_special_token_169|>",
1421
+ "lstrip": false,
1422
+ "normalized": false,
1423
+ "rstrip": false,
1424
+ "single_word": false,
1425
+ "special": true
1426
+ },
1427
+ "128178": {
1428
+ "content": "<|reserved_special_token_170|>",
1429
+ "lstrip": false,
1430
+ "normalized": false,
1431
+ "rstrip": false,
1432
+ "single_word": false,
1433
+ "special": true
1434
+ },
1435
+ "128179": {
1436
+ "content": "<|reserved_special_token_171|>",
1437
+ "lstrip": false,
1438
+ "normalized": false,
1439
+ "rstrip": false,
1440
+ "single_word": false,
1441
+ "special": true
1442
+ },
1443
+ "128180": {
1444
+ "content": "<|reserved_special_token_172|>",
1445
+ "lstrip": false,
1446
+ "normalized": false,
1447
+ "rstrip": false,
1448
+ "single_word": false,
1449
+ "special": true
1450
+ },
1451
+ "128181": {
1452
+ "content": "<|reserved_special_token_173|>",
1453
+ "lstrip": false,
1454
+ "normalized": false,
1455
+ "rstrip": false,
1456
+ "single_word": false,
1457
+ "special": true
1458
+ },
1459
+ "128182": {
1460
+ "content": "<|reserved_special_token_174|>",
1461
+ "lstrip": false,
1462
+ "normalized": false,
1463
+ "rstrip": false,
1464
+ "single_word": false,
1465
+ "special": true
1466
+ },
1467
+ "128183": {
1468
+ "content": "<|reserved_special_token_175|>",
1469
+ "lstrip": false,
1470
+ "normalized": false,
1471
+ "rstrip": false,
1472
+ "single_word": false,
1473
+ "special": true
1474
+ },
1475
+ "128184": {
1476
+ "content": "<|reserved_special_token_176|>",
1477
+ "lstrip": false,
1478
+ "normalized": false,
1479
+ "rstrip": false,
1480
+ "single_word": false,
1481
+ "special": true
1482
+ },
1483
+ "128185": {
1484
+ "content": "<|reserved_special_token_177|>",
1485
+ "lstrip": false,
1486
+ "normalized": false,
1487
+ "rstrip": false,
1488
+ "single_word": false,
1489
+ "special": true
1490
+ },
1491
+ "128186": {
1492
+ "content": "<|reserved_special_token_178|>",
1493
+ "lstrip": false,
1494
+ "normalized": false,
1495
+ "rstrip": false,
1496
+ "single_word": false,
1497
+ "special": true
1498
+ },
1499
+ "128187": {
1500
+ "content": "<|reserved_special_token_179|>",
1501
+ "lstrip": false,
1502
+ "normalized": false,
1503
+ "rstrip": false,
1504
+ "single_word": false,
1505
+ "special": true
1506
+ },
1507
+ "128188": {
1508
+ "content": "<|reserved_special_token_180|>",
1509
+ "lstrip": false,
1510
+ "normalized": false,
1511
+ "rstrip": false,
1512
+ "single_word": false,
1513
+ "special": true
1514
+ },
1515
+ "128189": {
1516
+ "content": "<|reserved_special_token_181|>",
1517
+ "lstrip": false,
1518
+ "normalized": false,
1519
+ "rstrip": false,
1520
+ "single_word": false,
1521
+ "special": true
1522
+ },
1523
+ "128190": {
1524
+ "content": "<|reserved_special_token_182|>",
1525
+ "lstrip": false,
1526
+ "normalized": false,
1527
+ "rstrip": false,
1528
+ "single_word": false,
1529
+ "special": true
1530
+ },
1531
+ "128191": {
1532
+ "content": "<|reserved_special_token_183|>",
1533
+ "lstrip": false,
1534
+ "normalized": false,
1535
+ "rstrip": false,
1536
+ "single_word": false,
1537
+ "special": true
1538
+ },
1539
+ "128192": {
1540
+ "content": "<|reserved_special_token_184|>",
1541
+ "lstrip": false,
1542
+ "normalized": false,
1543
+ "rstrip": false,
1544
+ "single_word": false,
1545
+ "special": true
1546
+ },
1547
+ "128193": {
1548
+ "content": "<|reserved_special_token_185|>",
1549
+ "lstrip": false,
1550
+ "normalized": false,
1551
+ "rstrip": false,
1552
+ "single_word": false,
1553
+ "special": true
1554
+ },
1555
+ "128194": {
1556
+ "content": "<|reserved_special_token_186|>",
1557
+ "lstrip": false,
1558
+ "normalized": false,
1559
+ "rstrip": false,
1560
+ "single_word": false,
1561
+ "special": true
1562
+ },
1563
+ "128195": {
1564
+ "content": "<|reserved_special_token_187|>",
1565
+ "lstrip": false,
1566
+ "normalized": false,
1567
+ "rstrip": false,
1568
+ "single_word": false,
1569
+ "special": true
1570
+ },
1571
+ "128196": {
1572
+ "content": "<|reserved_special_token_188|>",
1573
+ "lstrip": false,
1574
+ "normalized": false,
1575
+ "rstrip": false,
1576
+ "single_word": false,
1577
+ "special": true
1578
+ },
1579
+ "128197": {
1580
+ "content": "<|reserved_special_token_189|>",
1581
+ "lstrip": false,
1582
+ "normalized": false,
1583
+ "rstrip": false,
1584
+ "single_word": false,
1585
+ "special": true
1586
+ },
1587
+ "128198": {
1588
+ "content": "<|reserved_special_token_190|>",
1589
+ "lstrip": false,
1590
+ "normalized": false,
1591
+ "rstrip": false,
1592
+ "single_word": false,
1593
+ "special": true
1594
+ },
1595
+ "128199": {
1596
+ "content": "<|reserved_special_token_191|>",
1597
+ "lstrip": false,
1598
+ "normalized": false,
1599
+ "rstrip": false,
1600
+ "single_word": false,
1601
+ "special": true
1602
+ },
1603
+ "128200": {
1604
+ "content": "<|reserved_special_token_192|>",
1605
+ "lstrip": false,
1606
+ "normalized": false,
1607
+ "rstrip": false,
1608
+ "single_word": false,
1609
+ "special": true
1610
+ },
1611
+ "128201": {
1612
+ "content": "<|reserved_special_token_193|>",
1613
+ "lstrip": false,
1614
+ "normalized": false,
1615
+ "rstrip": false,
1616
+ "single_word": false,
1617
+ "special": true
1618
+ },
1619
+ "128202": {
1620
+ "content": "<|reserved_special_token_194|>",
1621
+ "lstrip": false,
1622
+ "normalized": false,
1623
+ "rstrip": false,
1624
+ "single_word": false,
1625
+ "special": true
1626
+ },
1627
+ "128203": {
1628
+ "content": "<|reserved_special_token_195|>",
1629
+ "lstrip": false,
1630
+ "normalized": false,
1631
+ "rstrip": false,
1632
+ "single_word": false,
1633
+ "special": true
1634
+ },
1635
+ "128204": {
1636
+ "content": "<|reserved_special_token_196|>",
1637
+ "lstrip": false,
1638
+ "normalized": false,
1639
+ "rstrip": false,
1640
+ "single_word": false,
1641
+ "special": true
1642
+ },
1643
+ "128205": {
1644
+ "content": "<|reserved_special_token_197|>",
1645
+ "lstrip": false,
1646
+ "normalized": false,
1647
+ "rstrip": false,
1648
+ "single_word": false,
1649
+ "special": true
1650
+ },
1651
+ "128206": {
1652
+ "content": "<|reserved_special_token_198|>",
1653
+ "lstrip": false,
1654
+ "normalized": false,
1655
+ "rstrip": false,
1656
+ "single_word": false,
1657
+ "special": true
1658
+ },
1659
+ "128207": {
1660
+ "content": "<|reserved_special_token_199|>",
1661
+ "lstrip": false,
1662
+ "normalized": false,
1663
+ "rstrip": false,
1664
+ "single_word": false,
1665
+ "special": true
1666
+ },
1667
+ "128208": {
1668
+ "content": "<|reserved_special_token_200|>",
1669
+ "lstrip": false,
1670
+ "normalized": false,
1671
+ "rstrip": false,
1672
+ "single_word": false,
1673
+ "special": true
1674
+ },
1675
+ "128209": {
1676
+ "content": "<|reserved_special_token_201|>",
1677
+ "lstrip": false,
1678
+ "normalized": false,
1679
+ "rstrip": false,
1680
+ "single_word": false,
1681
+ "special": true
1682
+ },
1683
+ "128210": {
1684
+ "content": "<|reserved_special_token_202|>",
1685
+ "lstrip": false,
1686
+ "normalized": false,
1687
+ "rstrip": false,
1688
+ "single_word": false,
1689
+ "special": true
1690
+ },
1691
+ "128211": {
1692
+ "content": "<|reserved_special_token_203|>",
1693
+ "lstrip": false,
1694
+ "normalized": false,
1695
+ "rstrip": false,
1696
+ "single_word": false,
1697
+ "special": true
1698
+ },
1699
+ "128212": {
1700
+ "content": "<|reserved_special_token_204|>",
1701
+ "lstrip": false,
1702
+ "normalized": false,
1703
+ "rstrip": false,
1704
+ "single_word": false,
1705
+ "special": true
1706
+ },
1707
+ "128213": {
1708
+ "content": "<|reserved_special_token_205|>",
1709
+ "lstrip": false,
1710
+ "normalized": false,
1711
+ "rstrip": false,
1712
+ "single_word": false,
1713
+ "special": true
1714
+ },
1715
+ "128214": {
1716
+ "content": "<|reserved_special_token_206|>",
1717
+ "lstrip": false,
1718
+ "normalized": false,
1719
+ "rstrip": false,
1720
+ "single_word": false,
1721
+ "special": true
1722
+ },
1723
+ "128215": {
1724
+ "content": "<|reserved_special_token_207|>",
1725
+ "lstrip": false,
1726
+ "normalized": false,
1727
+ "rstrip": false,
1728
+ "single_word": false,
1729
+ "special": true
1730
+ },
1731
+ "128216": {
1732
+ "content": "<|reserved_special_token_208|>",
1733
+ "lstrip": false,
1734
+ "normalized": false,
1735
+ "rstrip": false,
1736
+ "single_word": false,
1737
+ "special": true
1738
+ },
1739
+ "128217": {
1740
+ "content": "<|reserved_special_token_209|>",
1741
+ "lstrip": false,
1742
+ "normalized": false,
1743
+ "rstrip": false,
1744
+ "single_word": false,
1745
+ "special": true
1746
+ },
1747
+ "128218": {
1748
+ "content": "<|reserved_special_token_210|>",
1749
+ "lstrip": false,
1750
+ "normalized": false,
1751
+ "rstrip": false,
1752
+ "single_word": false,
1753
+ "special": true
1754
+ },
1755
+ "128219": {
1756
+ "content": "<|reserved_special_token_211|>",
1757
+ "lstrip": false,
1758
+ "normalized": false,
1759
+ "rstrip": false,
1760
+ "single_word": false,
1761
+ "special": true
1762
+ },
1763
+ "128220": {
1764
+ "content": "<|reserved_special_token_212|>",
1765
+ "lstrip": false,
1766
+ "normalized": false,
1767
+ "rstrip": false,
1768
+ "single_word": false,
1769
+ "special": true
1770
+ },
1771
+ "128221": {
1772
+ "content": "<|reserved_special_token_213|>",
1773
+ "lstrip": false,
1774
+ "normalized": false,
1775
+ "rstrip": false,
1776
+ "single_word": false,
1777
+ "special": true
1778
+ },
1779
+ "128222": {
1780
+ "content": "<|reserved_special_token_214|>",
1781
+ "lstrip": false,
1782
+ "normalized": false,
1783
+ "rstrip": false,
1784
+ "single_word": false,
1785
+ "special": true
1786
+ },
1787
+ "128223": {
1788
+ "content": "<|reserved_special_token_215|>",
1789
+ "lstrip": false,
1790
+ "normalized": false,
1791
+ "rstrip": false,
1792
+ "single_word": false,
1793
+ "special": true
1794
+ },
1795
+ "128224": {
1796
+ "content": "<|reserved_special_token_216|>",
1797
+ "lstrip": false,
1798
+ "normalized": false,
1799
+ "rstrip": false,
1800
+ "single_word": false,
1801
+ "special": true
1802
+ },
1803
+ "128225": {
1804
+ "content": "<|reserved_special_token_217|>",
1805
+ "lstrip": false,
1806
+ "normalized": false,
1807
+ "rstrip": false,
1808
+ "single_word": false,
1809
+ "special": true
1810
+ },
1811
+ "128226": {
1812
+ "content": "<|reserved_special_token_218|>",
1813
+ "lstrip": false,
1814
+ "normalized": false,
1815
+ "rstrip": false,
1816
+ "single_word": false,
1817
+ "special": true
1818
+ },
1819
+ "128227": {
1820
+ "content": "<|reserved_special_token_219|>",
1821
+ "lstrip": false,
1822
+ "normalized": false,
1823
+ "rstrip": false,
1824
+ "single_word": false,
1825
+ "special": true
1826
+ },
1827
+ "128228": {
1828
+ "content": "<|reserved_special_token_220|>",
1829
+ "lstrip": false,
1830
+ "normalized": false,
1831
+ "rstrip": false,
1832
+ "single_word": false,
1833
+ "special": true
1834
+ },
1835
+ "128229": {
1836
+ "content": "<|reserved_special_token_221|>",
1837
+ "lstrip": false,
1838
+ "normalized": false,
1839
+ "rstrip": false,
1840
+ "single_word": false,
1841
+ "special": true
1842
+ },
1843
+ "128230": {
1844
+ "content": "<|reserved_special_token_222|>",
1845
+ "lstrip": false,
1846
+ "normalized": false,
1847
+ "rstrip": false,
1848
+ "single_word": false,
1849
+ "special": true
1850
+ },
1851
+ "128231": {
1852
+ "content": "<|reserved_special_token_223|>",
1853
+ "lstrip": false,
1854
+ "normalized": false,
1855
+ "rstrip": false,
1856
+ "single_word": false,
1857
+ "special": true
1858
+ },
1859
+ "128232": {
1860
+ "content": "<|reserved_special_token_224|>",
1861
+ "lstrip": false,
1862
+ "normalized": false,
1863
+ "rstrip": false,
1864
+ "single_word": false,
1865
+ "special": true
1866
+ },
1867
+ "128233": {
1868
+ "content": "<|reserved_special_token_225|>",
1869
+ "lstrip": false,
1870
+ "normalized": false,
1871
+ "rstrip": false,
1872
+ "single_word": false,
1873
+ "special": true
1874
+ },
1875
+ "128234": {
1876
+ "content": "<|reserved_special_token_226|>",
1877
+ "lstrip": false,
1878
+ "normalized": false,
1879
+ "rstrip": false,
1880
+ "single_word": false,
1881
+ "special": true
1882
+ },
1883
+ "128235": {
1884
+ "content": "<|reserved_special_token_227|>",
1885
+ "lstrip": false,
1886
+ "normalized": false,
1887
+ "rstrip": false,
1888
+ "single_word": false,
1889
+ "special": true
1890
+ },
1891
+ "128236": {
1892
+ "content": "<|reserved_special_token_228|>",
1893
+ "lstrip": false,
1894
+ "normalized": false,
1895
+ "rstrip": false,
1896
+ "single_word": false,
1897
+ "special": true
1898
+ },
1899
+ "128237": {
1900
+ "content": "<|reserved_special_token_229|>",
1901
+ "lstrip": false,
1902
+ "normalized": false,
1903
+ "rstrip": false,
1904
+ "single_word": false,
1905
+ "special": true
1906
+ },
1907
+ "128238": {
1908
+ "content": "<|reserved_special_token_230|>",
1909
+ "lstrip": false,
1910
+ "normalized": false,
1911
+ "rstrip": false,
1912
+ "single_word": false,
1913
+ "special": true
1914
+ },
1915
+ "128239": {
1916
+ "content": "<|reserved_special_token_231|>",
1917
+ "lstrip": false,
1918
+ "normalized": false,
1919
+ "rstrip": false,
1920
+ "single_word": false,
1921
+ "special": true
1922
+ },
1923
+ "128240": {
1924
+ "content": "<|reserved_special_token_232|>",
1925
+ "lstrip": false,
1926
+ "normalized": false,
1927
+ "rstrip": false,
1928
+ "single_word": false,
1929
+ "special": true
1930
+ },
1931
+ "128241": {
1932
+ "content": "<|reserved_special_token_233|>",
1933
+ "lstrip": false,
1934
+ "normalized": false,
1935
+ "rstrip": false,
1936
+ "single_word": false,
1937
+ "special": true
1938
+ },
1939
+ "128242": {
1940
+ "content": "<|reserved_special_token_234|>",
1941
+ "lstrip": false,
1942
+ "normalized": false,
1943
+ "rstrip": false,
1944
+ "single_word": false,
1945
+ "special": true
1946
+ },
1947
+ "128243": {
1948
+ "content": "<|reserved_special_token_235|>",
1949
+ "lstrip": false,
1950
+ "normalized": false,
1951
+ "rstrip": false,
1952
+ "single_word": false,
1953
+ "special": true
1954
+ },
1955
+ "128244": {
1956
+ "content": "<|reserved_special_token_236|>",
1957
+ "lstrip": false,
1958
+ "normalized": false,
1959
+ "rstrip": false,
1960
+ "single_word": false,
1961
+ "special": true
1962
+ },
1963
+ "128245": {
1964
+ "content": "<|reserved_special_token_237|>",
1965
+ "lstrip": false,
1966
+ "normalized": false,
1967
+ "rstrip": false,
1968
+ "single_word": false,
1969
+ "special": true
1970
+ },
1971
+ "128246": {
1972
+ "content": "<|reserved_special_token_238|>",
1973
+ "lstrip": false,
1974
+ "normalized": false,
1975
+ "rstrip": false,
1976
+ "single_word": false,
1977
+ "special": true
1978
+ },
1979
+ "128247": {
1980
+ "content": "<|reserved_special_token_239|>",
1981
+ "lstrip": false,
1982
+ "normalized": false,
1983
+ "rstrip": false,
1984
+ "single_word": false,
1985
+ "special": true
1986
+ },
1987
+ "128248": {
1988
+ "content": "<|reserved_special_token_240|>",
1989
+ "lstrip": false,
1990
+ "normalized": false,
1991
+ "rstrip": false,
1992
+ "single_word": false,
1993
+ "special": true
1994
+ },
1995
+ "128249": {
1996
+ "content": "<|reserved_special_token_241|>",
1997
+ "lstrip": false,
1998
+ "normalized": false,
1999
+ "rstrip": false,
2000
+ "single_word": false,
2001
+ "special": true
2002
+ },
2003
+ "128250": {
2004
+ "content": "<|reserved_special_token_242|>",
2005
+ "lstrip": false,
2006
+ "normalized": false,
2007
+ "rstrip": false,
2008
+ "single_word": false,
2009
+ "special": true
2010
+ },
2011
+ "128251": {
2012
+ "content": "<|reserved_special_token_243|>",
2013
+ "lstrip": false,
2014
+ "normalized": false,
2015
+ "rstrip": false,
2016
+ "single_word": false,
2017
+ "special": true
2018
+ },
2019
+ "128252": {
2020
+ "content": "<|reserved_special_token_244|>",
2021
+ "lstrip": false,
2022
+ "normalized": false,
2023
+ "rstrip": false,
2024
+ "single_word": false,
2025
+ "special": true
2026
+ },
2027
+ "128253": {
2028
+ "content": "<|reserved_special_token_245|>",
2029
+ "lstrip": false,
2030
+ "normalized": false,
2031
+ "rstrip": false,
2032
+ "single_word": false,
2033
+ "special": true
2034
+ },
2035
+ "128254": {
2036
+ "content": "<|reserved_special_token_246|>",
2037
+ "lstrip": false,
2038
+ "normalized": false,
2039
+ "rstrip": false,
2040
+ "single_word": false,
2041
+ "special": true
2042
+ },
2043
+ "128255": {
2044
+ "content": "<|reserved_special_token_247|>",
2045
+ "lstrip": false,
2046
+ "normalized": false,
2047
+ "rstrip": false,
2048
+ "single_word": false,
2049
+ "special": true
2050
+ }
2051
+ },
2052
+ "bos_token": "<|begin_of_text|>",
2053
+ "clean_up_tokenization_spaces": true,
2054
+ "eos_token": "<|eot_id|>",
2055
+ "extra_special_tokens": {},
2056
+ "model_input_names": [
2057
+ "input_ids",
2058
+ "attention_mask"
2059
+ ],
2060
+ "model_max_length": 131072,
2061
+ "pad_token": "<|eot_id|>",
2062
+ "tokenizer_class": "PreTrainedTokenizerFast"
2063
+ }
trainer_state.json ADDED
@@ -0,0 +1,3346 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.0,
6
+ "eval_steps": 500,
7
+ "global_step": 3670,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.0002725306171115161,
14
+ "grad_norm": 58.5752067565918,
15
+ "learning_rate": 0.0,
16
+ "loss": 4.1988,
17
+ "mean_token_accuracy": 0.27959318878129125,
18
+ "num_tokens": 180497.0,
19
+ "step": 1
20
+ },
21
+ {
22
+ "epoch": 0.0027253061711151614,
23
+ "grad_norm": 55.061241149902344,
24
+ "learning_rate": 2.0361990950226245e-06,
25
+ "loss": 4.1578,
26
+ "mean_token_accuracy": 0.2840940863307979,
27
+ "num_tokens": 1772191.0,
28
+ "step": 10
29
+ },
30
+ {
31
+ "epoch": 0.005450612342230323,
32
+ "grad_norm": 31.145395278930664,
33
+ "learning_rate": 4.298642533936651e-06,
34
+ "loss": 3.805,
35
+ "mean_token_accuracy": 0.30883588222786784,
36
+ "num_tokens": 3528128.0,
37
+ "step": 20
38
+ },
39
+ {
40
+ "epoch": 0.008175918513345483,
41
+ "grad_norm": 12.37236213684082,
42
+ "learning_rate": 6.5610859728506795e-06,
43
+ "loss": 3.0369,
44
+ "mean_token_accuracy": 0.3894659894518554,
45
+ "num_tokens": 5321820.0,
46
+ "step": 30
47
+ },
48
+ {
49
+ "epoch": 0.010901224684460645,
50
+ "grad_norm": 4.672379016876221,
51
+ "learning_rate": 8.823529411764707e-06,
52
+ "loss": 2.3771,
53
+ "mean_token_accuracy": 0.48662197683006525,
54
+ "num_tokens": 7036353.0,
55
+ "step": 40
56
+ },
57
+ {
58
+ "epoch": 0.013626530855575806,
59
+ "grad_norm": 2.293363094329834,
60
+ "learning_rate": 1.1085972850678733e-05,
61
+ "loss": 1.9359,
62
+ "mean_token_accuracy": 0.5632315624505282,
63
+ "num_tokens": 8794502.0,
64
+ "step": 50
65
+ },
66
+ {
67
+ "epoch": 0.016351837026690966,
68
+ "grad_norm": 1.8514742851257324,
69
+ "learning_rate": 1.3348416289592761e-05,
70
+ "loss": 1.687,
71
+ "mean_token_accuracy": 0.6138357298448682,
72
+ "num_tokens": 10519144.0,
73
+ "step": 60
74
+ },
75
+ {
76
+ "epoch": 0.019077143197806127,
77
+ "grad_norm": 1.5001717805862427,
78
+ "learning_rate": 1.5610859728506788e-05,
79
+ "loss": 1.511,
80
+ "mean_token_accuracy": 0.6497381268069148,
81
+ "num_tokens": 12272638.0,
82
+ "step": 70
83
+ },
84
+ {
85
+ "epoch": 0.02180244936892129,
86
+ "grad_norm": 1.2374789714813232,
87
+ "learning_rate": 1.7873303167420814e-05,
88
+ "loss": 1.4223,
89
+ "mean_token_accuracy": 0.667938649840653,
90
+ "num_tokens": 14037595.0,
91
+ "step": 80
92
+ },
93
+ {
94
+ "epoch": 0.02452775554003645,
95
+ "grad_norm": 1.232344388961792,
96
+ "learning_rate": 2.0135746606334844e-05,
97
+ "loss": 1.3629,
98
+ "mean_token_accuracy": 0.6789748859591782,
99
+ "num_tokens": 15813413.0,
100
+ "step": 90
101
+ },
102
+ {
103
+ "epoch": 0.027253061711151612,
104
+ "grad_norm": 1.2523256540298462,
105
+ "learning_rate": 2.239819004524887e-05,
106
+ "loss": 1.3328,
107
+ "mean_token_accuracy": 0.6849433833733201,
108
+ "num_tokens": 17606667.0,
109
+ "step": 100
110
+ },
111
+ {
112
+ "epoch": 0.029978367882266772,
113
+ "grad_norm": 1.1663717031478882,
114
+ "learning_rate": 2.4660633484162897e-05,
115
+ "loss": 1.2645,
116
+ "mean_token_accuracy": 0.7004281104542315,
117
+ "num_tokens": 19387650.0,
118
+ "step": 110
119
+ },
120
+ {
121
+ "epoch": 0.03270367405338193,
122
+ "grad_norm": 1.148568868637085,
123
+ "learning_rate": 2.6923076923076923e-05,
124
+ "loss": 1.2482,
125
+ "mean_token_accuracy": 0.7018253466114401,
126
+ "num_tokens": 21111629.0,
127
+ "step": 120
128
+ },
129
+ {
130
+ "epoch": 0.03542898022449709,
131
+ "grad_norm": 1.0586553812026978,
132
+ "learning_rate": 2.9185520361990953e-05,
133
+ "loss": 1.2162,
134
+ "mean_token_accuracy": 0.7089157540351152,
135
+ "num_tokens": 22855847.0,
136
+ "step": 130
137
+ },
138
+ {
139
+ "epoch": 0.038154286395612254,
140
+ "grad_norm": 1.1304233074188232,
141
+ "learning_rate": 3.1447963800904976e-05,
142
+ "loss": 1.1918,
143
+ "mean_token_accuracy": 0.7152549306862056,
144
+ "num_tokens": 24647294.0,
145
+ "step": 140
146
+ },
147
+ {
148
+ "epoch": 0.04087959256672742,
149
+ "grad_norm": 1.1450897455215454,
150
+ "learning_rate": 3.371040723981901e-05,
151
+ "loss": 1.1655,
152
+ "mean_token_accuracy": 0.7181231830269098,
153
+ "num_tokens": 26336128.0,
154
+ "step": 150
155
+ },
156
+ {
157
+ "epoch": 0.04360489873784258,
158
+ "grad_norm": 1.0469363927841187,
159
+ "learning_rate": 3.5972850678733036e-05,
160
+ "loss": 1.2058,
161
+ "mean_token_accuracy": 0.7123475090600551,
162
+ "num_tokens": 28061528.0,
163
+ "step": 160
164
+ },
165
+ {
166
+ "epoch": 0.04633020490895774,
167
+ "grad_norm": 1.0489130020141602,
168
+ "learning_rate": 3.8235294117647055e-05,
169
+ "loss": 1.178,
170
+ "mean_token_accuracy": 0.7180261522531509,
171
+ "num_tokens": 29834651.0,
172
+ "step": 170
173
+ },
174
+ {
175
+ "epoch": 0.0490555110800729,
176
+ "grad_norm": 1.1838144063949585,
177
+ "learning_rate": 4.049773755656109e-05,
178
+ "loss": 1.1964,
179
+ "mean_token_accuracy": 0.7144321620464325,
180
+ "num_tokens": 31533806.0,
181
+ "step": 180
182
+ },
183
+ {
184
+ "epoch": 0.05178081725118806,
185
+ "grad_norm": 1.0851892232894897,
186
+ "learning_rate": 4.2760180995475115e-05,
187
+ "loss": 1.1408,
188
+ "mean_token_accuracy": 0.72315530218184,
189
+ "num_tokens": 33328644.0,
190
+ "step": 190
191
+ },
192
+ {
193
+ "epoch": 0.054506123422303224,
194
+ "grad_norm": 1.1279138326644897,
195
+ "learning_rate": 4.502262443438914e-05,
196
+ "loss": 1.1406,
197
+ "mean_token_accuracy": 0.7242222094908357,
198
+ "num_tokens": 35085202.0,
199
+ "step": 200
200
+ },
201
+ {
202
+ "epoch": 0.057231429593418384,
203
+ "grad_norm": 1.07979416847229,
204
+ "learning_rate": 4.728506787330317e-05,
205
+ "loss": 1.1566,
206
+ "mean_token_accuracy": 0.7219386071898043,
207
+ "num_tokens": 36899685.0,
208
+ "step": 210
209
+ },
210
+ {
211
+ "epoch": 0.059956735764533545,
212
+ "grad_norm": 1.1669384241104126,
213
+ "learning_rate": 4.95475113122172e-05,
214
+ "loss": 1.0724,
215
+ "mean_token_accuracy": 0.7370353294536471,
216
+ "num_tokens": 38663867.0,
217
+ "step": 220
218
+ },
219
+ {
220
+ "epoch": 0.0626820419356487,
221
+ "grad_norm": 1.0999536514282227,
222
+ "learning_rate": 4.994381233319287e-05,
223
+ "loss": 1.0878,
224
+ "mean_token_accuracy": 0.733780630491674,
225
+ "num_tokens": 40368688.0,
226
+ "step": 230
227
+ },
228
+ {
229
+ "epoch": 0.06540734810676387,
230
+ "grad_norm": 1.029019832611084,
231
+ "learning_rate": 4.9873577749683945e-05,
232
+ "loss": 1.0928,
233
+ "mean_token_accuracy": 0.7319904958829284,
234
+ "num_tokens": 42091556.0,
235
+ "step": 240
236
+ },
237
+ {
238
+ "epoch": 0.06813265427787903,
239
+ "grad_norm": 1.0037535429000854,
240
+ "learning_rate": 4.9803343166175026e-05,
241
+ "loss": 1.0902,
242
+ "mean_token_accuracy": 0.7336283173412085,
243
+ "num_tokens": 43832865.0,
244
+ "step": 250
245
+ },
246
+ {
247
+ "epoch": 0.07085796044899419,
248
+ "grad_norm": 1.0020638704299927,
249
+ "learning_rate": 4.9733108582666106e-05,
250
+ "loss": 1.0952,
251
+ "mean_token_accuracy": 0.7330724668689073,
252
+ "num_tokens": 45556421.0,
253
+ "step": 260
254
+ },
255
+ {
256
+ "epoch": 0.07358326662010935,
257
+ "grad_norm": 1.0238029956817627,
258
+ "learning_rate": 4.9662873999157186e-05,
259
+ "loss": 1.1225,
260
+ "mean_token_accuracy": 0.7294999814592302,
261
+ "num_tokens": 47284838.0,
262
+ "step": 270
263
+ },
264
+ {
265
+ "epoch": 0.07630857279122451,
266
+ "grad_norm": 1.0471445322036743,
267
+ "learning_rate": 4.9592639415648266e-05,
268
+ "loss": 1.0805,
269
+ "mean_token_accuracy": 0.734851144067943,
270
+ "num_tokens": 49025592.0,
271
+ "step": 280
272
+ },
273
+ {
274
+ "epoch": 0.07903387896233968,
275
+ "grad_norm": 1.1734944581985474,
276
+ "learning_rate": 4.9522404832139346e-05,
277
+ "loss": 1.1459,
278
+ "mean_token_accuracy": 0.7227287207730114,
279
+ "num_tokens": 50771353.0,
280
+ "step": 290
281
+ },
282
+ {
283
+ "epoch": 0.08175918513345484,
284
+ "grad_norm": 1.0051774978637695,
285
+ "learning_rate": 4.945217024863043e-05,
286
+ "loss": 1.0595,
287
+ "mean_token_accuracy": 0.7396277584135532,
288
+ "num_tokens": 52521467.0,
289
+ "step": 300
290
+ },
291
+ {
292
+ "epoch": 0.08448449130457,
293
+ "grad_norm": 1.1502430438995361,
294
+ "learning_rate": 4.938193566512151e-05,
295
+ "loss": 1.0856,
296
+ "mean_token_accuracy": 0.7348609266802668,
297
+ "num_tokens": 54338858.0,
298
+ "step": 310
299
+ },
300
+ {
301
+ "epoch": 0.08720979747568516,
302
+ "grad_norm": 1.079554796218872,
303
+ "learning_rate": 4.931170108161259e-05,
304
+ "loss": 1.0876,
305
+ "mean_token_accuracy": 0.7352117918431759,
306
+ "num_tokens": 56014081.0,
307
+ "step": 320
308
+ },
309
+ {
310
+ "epoch": 0.08993510364680032,
311
+ "grad_norm": 1.0328309535980225,
312
+ "learning_rate": 4.924146649810367e-05,
313
+ "loss": 1.0622,
314
+ "mean_token_accuracy": 0.7386171711608768,
315
+ "num_tokens": 57778337.0,
316
+ "step": 330
317
+ },
318
+ {
319
+ "epoch": 0.09266040981791548,
320
+ "grad_norm": 0.9490402936935425,
321
+ "learning_rate": 4.9171231914594754e-05,
322
+ "loss": 1.0549,
323
+ "mean_token_accuracy": 0.74089086484164,
324
+ "num_tokens": 59528977.0,
325
+ "step": 340
326
+ },
327
+ {
328
+ "epoch": 0.09538571598903064,
329
+ "grad_norm": 0.9298028349876404,
330
+ "learning_rate": 4.910099733108583e-05,
331
+ "loss": 1.0704,
332
+ "mean_token_accuracy": 0.7380094730295241,
333
+ "num_tokens": 61295985.0,
334
+ "step": 350
335
+ },
336
+ {
337
+ "epoch": 0.0981110221601458,
338
+ "grad_norm": 1.0823702812194824,
339
+ "learning_rate": 4.903076274757691e-05,
340
+ "loss": 1.0535,
341
+ "mean_token_accuracy": 0.7405891207046806,
342
+ "num_tokens": 63026897.0,
343
+ "step": 360
344
+ },
345
+ {
346
+ "epoch": 0.10083632833126097,
347
+ "grad_norm": 0.9523594379425049,
348
+ "learning_rate": 4.896052816406799e-05,
349
+ "loss": 1.0444,
350
+ "mean_token_accuracy": 0.7437008136883378,
351
+ "num_tokens": 64770711.0,
352
+ "step": 370
353
+ },
354
+ {
355
+ "epoch": 0.10356163450237613,
356
+ "grad_norm": 1.1027802228927612,
357
+ "learning_rate": 4.889029358055907e-05,
358
+ "loss": 1.0446,
359
+ "mean_token_accuracy": 0.7440237908624112,
360
+ "num_tokens": 66551008.0,
361
+ "step": 380
362
+ },
363
+ {
364
+ "epoch": 0.10628694067349129,
365
+ "grad_norm": 0.9772301912307739,
366
+ "learning_rate": 4.882005899705015e-05,
367
+ "loss": 1.0598,
368
+ "mean_token_accuracy": 0.7391570524312556,
369
+ "num_tokens": 68320690.0,
370
+ "step": 390
371
+ },
372
+ {
373
+ "epoch": 0.10901224684460645,
374
+ "grad_norm": 0.8989792466163635,
375
+ "learning_rate": 4.874982441354123e-05,
376
+ "loss": 1.0231,
377
+ "mean_token_accuracy": 0.7470484189689159,
378
+ "num_tokens": 70095284.0,
379
+ "step": 400
380
+ },
381
+ {
382
+ "epoch": 0.11173755301572161,
383
+ "grad_norm": 0.9571840167045593,
384
+ "learning_rate": 4.8679589830032316e-05,
385
+ "loss": 1.0416,
386
+ "mean_token_accuracy": 0.7444285966455937,
387
+ "num_tokens": 71803511.0,
388
+ "step": 410
389
+ },
390
+ {
391
+ "epoch": 0.11446285918683677,
392
+ "grad_norm": 0.9428116679191589,
393
+ "learning_rate": 4.860935524652339e-05,
394
+ "loss": 0.9991,
395
+ "mean_token_accuracy": 0.7514733706600965,
396
+ "num_tokens": 73528297.0,
397
+ "step": 420
398
+ },
399
+ {
400
+ "epoch": 0.11718816535795193,
401
+ "grad_norm": 0.9464392066001892,
402
+ "learning_rate": 4.853912066301447e-05,
403
+ "loss": 1.0022,
404
+ "mean_token_accuracy": 0.7517835930921137,
405
+ "num_tokens": 75274289.0,
406
+ "step": 430
407
+ },
408
+ {
409
+ "epoch": 0.11991347152906709,
410
+ "grad_norm": 1.0934405326843262,
411
+ "learning_rate": 4.846888607950555e-05,
412
+ "loss": 1.0466,
413
+ "mean_token_accuracy": 0.7423655292019248,
414
+ "num_tokens": 77077403.0,
415
+ "step": 440
416
+ },
417
+ {
418
+ "epoch": 0.12263877770018225,
419
+ "grad_norm": 0.9798712134361267,
420
+ "learning_rate": 4.8398651495996636e-05,
421
+ "loss": 1.0416,
422
+ "mean_token_accuracy": 0.7423399617895484,
423
+ "num_tokens": 78859882.0,
424
+ "step": 450
425
+ },
426
+ {
427
+ "epoch": 0.1253640838712974,
428
+ "grad_norm": 0.9301265478134155,
429
+ "learning_rate": 4.832841691248771e-05,
430
+ "loss": 1.0564,
431
+ "mean_token_accuracy": 0.7402447827160359,
432
+ "num_tokens": 80590976.0,
433
+ "step": 460
434
+ },
435
+ {
436
+ "epoch": 0.12808939004241257,
437
+ "grad_norm": 1.028621792793274,
438
+ "learning_rate": 4.825818232897879e-05,
439
+ "loss": 1.0135,
440
+ "mean_token_accuracy": 0.7484846420586109,
441
+ "num_tokens": 82275126.0,
442
+ "step": 470
443
+ },
444
+ {
445
+ "epoch": 0.13081469621352773,
446
+ "grad_norm": 0.9343350529670715,
447
+ "learning_rate": 4.818794774546987e-05,
448
+ "loss": 1.0102,
449
+ "mean_token_accuracy": 0.7489786158315838,
450
+ "num_tokens": 84096910.0,
451
+ "step": 480
452
+ },
453
+ {
454
+ "epoch": 0.1335400023846429,
455
+ "grad_norm": 0.9360990524291992,
456
+ "learning_rate": 4.811771316196095e-05,
457
+ "loss": 1.0402,
458
+ "mean_token_accuracy": 0.7446358595974744,
459
+ "num_tokens": 85876326.0,
460
+ "step": 490
461
+ },
462
+ {
463
+ "epoch": 0.13626530855575805,
464
+ "grad_norm": 0.9260503053665161,
465
+ "learning_rate": 4.804747857845203e-05,
466
+ "loss": 1.0412,
467
+ "mean_token_accuracy": 0.7441607388667763,
468
+ "num_tokens": 87607478.0,
469
+ "step": 500
470
+ },
471
+ {
472
+ "epoch": 0.1389906147268732,
473
+ "grad_norm": 0.9006381034851074,
474
+ "learning_rate": 4.797724399494311e-05,
475
+ "loss": 1.0014,
476
+ "mean_token_accuracy": 0.7509601839818061,
477
+ "num_tokens": 89350066.0,
478
+ "step": 510
479
+ },
480
+ {
481
+ "epoch": 0.14171592089798837,
482
+ "grad_norm": 0.9362842440605164,
483
+ "learning_rate": 4.79070094114342e-05,
484
+ "loss": 1.0138,
485
+ "mean_token_accuracy": 0.7497716393321753,
486
+ "num_tokens": 91067910.0,
487
+ "step": 520
488
+ },
489
+ {
490
+ "epoch": 0.14444122706910353,
491
+ "grad_norm": 0.9452476501464844,
492
+ "learning_rate": 4.783677482792527e-05,
493
+ "loss": 1.0052,
494
+ "mean_token_accuracy": 0.7499888489954174,
495
+ "num_tokens": 92797017.0,
496
+ "step": 530
497
+ },
498
+ {
499
+ "epoch": 0.1471665332402187,
500
+ "grad_norm": 0.9116583466529846,
501
+ "learning_rate": 4.776654024441635e-05,
502
+ "loss": 1.0263,
503
+ "mean_token_accuracy": 0.7451268086209893,
504
+ "num_tokens": 94606329.0,
505
+ "step": 540
506
+ },
507
+ {
508
+ "epoch": 0.14989183941133385,
509
+ "grad_norm": 0.9305568933486938,
510
+ "learning_rate": 4.769630566090743e-05,
511
+ "loss": 1.0039,
512
+ "mean_token_accuracy": 0.7499175570905209,
513
+ "num_tokens": 96331087.0,
514
+ "step": 550
515
+ },
516
+ {
517
+ "epoch": 0.15261714558244902,
518
+ "grad_norm": 0.9302889704704285,
519
+ "learning_rate": 4.762607107739852e-05,
520
+ "loss": 1.0239,
521
+ "mean_token_accuracy": 0.7466340310871601,
522
+ "num_tokens": 98138711.0,
523
+ "step": 560
524
+ },
525
+ {
526
+ "epoch": 0.15534245175356418,
527
+ "grad_norm": 0.9294123649597168,
528
+ "learning_rate": 4.755583649388959e-05,
529
+ "loss": 1.0285,
530
+ "mean_token_accuracy": 0.7461859748698771,
531
+ "num_tokens": 99885005.0,
532
+ "step": 570
533
+ },
534
+ {
535
+ "epoch": 0.15806775792467936,
536
+ "grad_norm": 0.8810741901397705,
537
+ "learning_rate": 4.748560191038067e-05,
538
+ "loss": 0.9361,
539
+ "mean_token_accuracy": 0.7639201901853084,
540
+ "num_tokens": 101636075.0,
541
+ "step": 580
542
+ },
543
+ {
544
+ "epoch": 0.16079306409579452,
545
+ "grad_norm": 0.987924337387085,
546
+ "learning_rate": 4.741536732687175e-05,
547
+ "loss": 0.9949,
548
+ "mean_token_accuracy": 0.7529382834210991,
549
+ "num_tokens": 103349118.0,
550
+ "step": 590
551
+ },
552
+ {
553
+ "epoch": 0.16351837026690969,
554
+ "grad_norm": 0.9978275895118713,
555
+ "learning_rate": 4.734513274336283e-05,
556
+ "loss": 1.0056,
557
+ "mean_token_accuracy": 0.7498491807840765,
558
+ "num_tokens": 105033010.0,
559
+ "step": 600
560
+ },
561
+ {
562
+ "epoch": 0.16624367643802485,
563
+ "grad_norm": 0.9176990985870361,
564
+ "learning_rate": 4.727489815985391e-05,
565
+ "loss": 0.9731,
566
+ "mean_token_accuracy": 0.7562306389212609,
567
+ "num_tokens": 106723283.0,
568
+ "step": 610
569
+ },
570
+ {
571
+ "epoch": 0.16896898260914,
572
+ "grad_norm": 0.8888489007949829,
573
+ "learning_rate": 4.720466357634499e-05,
574
+ "loss": 0.9907,
575
+ "mean_token_accuracy": 0.7524073977023363,
576
+ "num_tokens": 108436878.0,
577
+ "step": 620
578
+ },
579
+ {
580
+ "epoch": 0.17169428878025517,
581
+ "grad_norm": 0.9666376709938049,
582
+ "learning_rate": 4.713442899283608e-05,
583
+ "loss": 0.9752,
584
+ "mean_token_accuracy": 0.7562545623630286,
585
+ "num_tokens": 110203157.0,
586
+ "step": 630
587
+ },
588
+ {
589
+ "epoch": 0.17441959495137033,
590
+ "grad_norm": 1.034400463104248,
591
+ "learning_rate": 4.706419440932715e-05,
592
+ "loss": 1.0029,
593
+ "mean_token_accuracy": 0.7515896405093372,
594
+ "num_tokens": 111949130.0,
595
+ "step": 640
596
+ },
597
+ {
598
+ "epoch": 0.1771449011224855,
599
+ "grad_norm": 1.044718623161316,
600
+ "learning_rate": 4.6993959825818233e-05,
601
+ "loss": 1.0037,
602
+ "mean_token_accuracy": 0.7487363073974848,
603
+ "num_tokens": 113652926.0,
604
+ "step": 650
605
+ },
606
+ {
607
+ "epoch": 0.17987020729360065,
608
+ "grad_norm": 0.906299352645874,
609
+ "learning_rate": 4.6923725242309314e-05,
610
+ "loss": 0.9903,
611
+ "mean_token_accuracy": 0.7514964304864407,
612
+ "num_tokens": 115334647.0,
613
+ "step": 660
614
+ },
615
+ {
616
+ "epoch": 0.1825955134647158,
617
+ "grad_norm": 0.8646354675292969,
618
+ "learning_rate": 4.68534906588004e-05,
619
+ "loss": 0.9501,
620
+ "mean_token_accuracy": 0.7617973797954619,
621
+ "num_tokens": 117032381.0,
622
+ "step": 670
623
+ },
624
+ {
625
+ "epoch": 0.18532081963583097,
626
+ "grad_norm": 0.8861644864082336,
627
+ "learning_rate": 4.6783256075291474e-05,
628
+ "loss": 0.9859,
629
+ "mean_token_accuracy": 0.7530626423656941,
630
+ "num_tokens": 118801553.0,
631
+ "step": 680
632
+ },
633
+ {
634
+ "epoch": 0.18804612580694613,
635
+ "grad_norm": 0.9357079863548279,
636
+ "learning_rate": 4.6713021491782554e-05,
637
+ "loss": 0.9989,
638
+ "mean_token_accuracy": 0.7504322250373662,
639
+ "num_tokens": 120530271.0,
640
+ "step": 690
641
+ },
642
+ {
643
+ "epoch": 0.1907714319780613,
644
+ "grad_norm": 0.8540539741516113,
645
+ "learning_rate": 4.6642786908273634e-05,
646
+ "loss": 0.9904,
647
+ "mean_token_accuracy": 0.7538057889789342,
648
+ "num_tokens": 122314411.0,
649
+ "step": 700
650
+ },
651
+ {
652
+ "epoch": 0.19349673814917645,
653
+ "grad_norm": 0.8792332410812378,
654
+ "learning_rate": 4.6572552324764715e-05,
655
+ "loss": 0.9839,
656
+ "mean_token_accuracy": 0.7542797921225428,
657
+ "num_tokens": 124054113.0,
658
+ "step": 710
659
+ },
660
+ {
661
+ "epoch": 0.1962220443202916,
662
+ "grad_norm": 0.9250332713127136,
663
+ "learning_rate": 4.6502317741255795e-05,
664
+ "loss": 0.9988,
665
+ "mean_token_accuracy": 0.7507336236536503,
666
+ "num_tokens": 125786705.0,
667
+ "step": 720
668
+ },
669
+ {
670
+ "epoch": 0.19894735049140677,
671
+ "grad_norm": 0.9268140196800232,
672
+ "learning_rate": 4.6432083157746875e-05,
673
+ "loss": 0.9998,
674
+ "mean_token_accuracy": 0.7502107596024871,
675
+ "num_tokens": 127510112.0,
676
+ "step": 730
677
+ },
678
+ {
679
+ "epoch": 0.20167265666252193,
680
+ "grad_norm": 0.9151081442832947,
681
+ "learning_rate": 4.636184857423796e-05,
682
+ "loss": 0.9637,
683
+ "mean_token_accuracy": 0.7585853973403573,
684
+ "num_tokens": 129321733.0,
685
+ "step": 740
686
+ },
687
+ {
688
+ "epoch": 0.2043979628336371,
689
+ "grad_norm": 0.8551493287086487,
690
+ "learning_rate": 4.6291613990729035e-05,
691
+ "loss": 1.0006,
692
+ "mean_token_accuracy": 0.7512839620932936,
693
+ "num_tokens": 131068939.0,
694
+ "step": 750
695
+ },
696
+ {
697
+ "epoch": 0.20712326900475225,
698
+ "grad_norm": 1.0179383754730225,
699
+ "learning_rate": 4.6221379407220116e-05,
700
+ "loss": 1.0366,
701
+ "mean_token_accuracy": 0.7441357178613544,
702
+ "num_tokens": 132800192.0,
703
+ "step": 760
704
+ },
705
+ {
706
+ "epoch": 0.2098485751758674,
707
+ "grad_norm": 0.8894006013870239,
708
+ "learning_rate": 4.6151144823711196e-05,
709
+ "loss": 0.9706,
710
+ "mean_token_accuracy": 0.7571743190288543,
711
+ "num_tokens": 134501880.0,
712
+ "step": 770
713
+ },
714
+ {
715
+ "epoch": 0.21257388134698257,
716
+ "grad_norm": 0.9344801306724548,
717
+ "learning_rate": 4.608091024020228e-05,
718
+ "loss": 0.9495,
719
+ "mean_token_accuracy": 0.7609304752200842,
720
+ "num_tokens": 136227230.0,
721
+ "step": 780
722
+ },
723
+ {
724
+ "epoch": 0.21529918751809773,
725
+ "grad_norm": 0.9688650965690613,
726
+ "learning_rate": 4.6010675656693356e-05,
727
+ "loss": 0.9634,
728
+ "mean_token_accuracy": 0.7590550154447555,
729
+ "num_tokens": 137938433.0,
730
+ "step": 790
731
+ },
732
+ {
733
+ "epoch": 0.2180244936892129,
734
+ "grad_norm": 0.8858566880226135,
735
+ "learning_rate": 4.5940441073184436e-05,
736
+ "loss": 0.9736,
737
+ "mean_token_accuracy": 0.7575385818257928,
738
+ "num_tokens": 139667163.0,
739
+ "step": 800
740
+ },
741
+ {
742
+ "epoch": 0.22074979986032806,
743
+ "grad_norm": 0.7869423031806946,
744
+ "learning_rate": 4.5870206489675517e-05,
745
+ "loss": 0.9494,
746
+ "mean_token_accuracy": 0.7619166353717446,
747
+ "num_tokens": 141481099.0,
748
+ "step": 810
749
+ },
750
+ {
751
+ "epoch": 0.22347510603144322,
752
+ "grad_norm": 0.889818549156189,
753
+ "learning_rate": 4.57999719061666e-05,
754
+ "loss": 1.0118,
755
+ "mean_token_accuracy": 0.7489442000165581,
756
+ "num_tokens": 143204243.0,
757
+ "step": 820
758
+ },
759
+ {
760
+ "epoch": 0.22620041220255838,
761
+ "grad_norm": 0.8601938486099243,
762
+ "learning_rate": 4.572973732265768e-05,
763
+ "loss": 0.9731,
764
+ "mean_token_accuracy": 0.7579358568415046,
765
+ "num_tokens": 144995581.0,
766
+ "step": 830
767
+ },
768
+ {
769
+ "epoch": 0.22892571837367354,
770
+ "grad_norm": 0.868138313293457,
771
+ "learning_rate": 4.565950273914876e-05,
772
+ "loss": 0.9795,
773
+ "mean_token_accuracy": 0.7540082458406687,
774
+ "num_tokens": 146711076.0,
775
+ "step": 840
776
+ },
777
+ {
778
+ "epoch": 0.2316510245447887,
779
+ "grad_norm": 0.8250023722648621,
780
+ "learning_rate": 4.5589268155639844e-05,
781
+ "loss": 0.971,
782
+ "mean_token_accuracy": 0.7573945553973317,
783
+ "num_tokens": 148463902.0,
784
+ "step": 850
785
+ },
786
+ {
787
+ "epoch": 0.23437633071590386,
788
+ "grad_norm": 0.8885928392410278,
789
+ "learning_rate": 4.551903357213092e-05,
790
+ "loss": 0.9964,
791
+ "mean_token_accuracy": 0.7519168628379702,
792
+ "num_tokens": 150212801.0,
793
+ "step": 860
794
+ },
795
+ {
796
+ "epoch": 0.23710163688701902,
797
+ "grad_norm": 0.8809767961502075,
798
+ "learning_rate": 4.5448798988622e-05,
799
+ "loss": 0.945,
800
+ "mean_token_accuracy": 0.7604282542131842,
801
+ "num_tokens": 151950016.0,
802
+ "step": 870
803
+ },
804
+ {
805
+ "epoch": 0.23982694305813418,
806
+ "grad_norm": 1.013855218887329,
807
+ "learning_rate": 4.537856440511308e-05,
808
+ "loss": 0.9567,
809
+ "mean_token_accuracy": 0.7587636118754745,
810
+ "num_tokens": 153686341.0,
811
+ "step": 880
812
+ },
813
+ {
814
+ "epoch": 0.24255224922924934,
815
+ "grad_norm": 0.8054437637329102,
816
+ "learning_rate": 4.5308329821604165e-05,
817
+ "loss": 0.9852,
818
+ "mean_token_accuracy": 0.753642223123461,
819
+ "num_tokens": 155442220.0,
820
+ "step": 890
821
+ },
822
+ {
823
+ "epoch": 0.2452775554003645,
824
+ "grad_norm": 0.9395073056221008,
825
+ "learning_rate": 4.523809523809524e-05,
826
+ "loss": 0.9552,
827
+ "mean_token_accuracy": 0.7594983844086528,
828
+ "num_tokens": 157222278.0,
829
+ "step": 900
830
+ },
831
+ {
832
+ "epoch": 0.24800286157147966,
833
+ "grad_norm": 0.8815314173698425,
834
+ "learning_rate": 4.516786065458632e-05,
835
+ "loss": 0.9484,
836
+ "mean_token_accuracy": 0.7606731666252017,
837
+ "num_tokens": 158989873.0,
838
+ "step": 910
839
+ },
840
+ {
841
+ "epoch": 0.2507281677425948,
842
+ "grad_norm": 0.9090284109115601,
843
+ "learning_rate": 4.50976260710774e-05,
844
+ "loss": 0.9742,
845
+ "mean_token_accuracy": 0.7581370803527534,
846
+ "num_tokens": 160740690.0,
847
+ "step": 920
848
+ },
849
+ {
850
+ "epoch": 0.25345347391371,
851
+ "grad_norm": 0.907217800617218,
852
+ "learning_rate": 4.502739148756848e-05,
853
+ "loss": 0.9779,
854
+ "mean_token_accuracy": 0.7548290783539414,
855
+ "num_tokens": 162505210.0,
856
+ "step": 930
857
+ },
858
+ {
859
+ "epoch": 0.25617878008482514,
860
+ "grad_norm": 0.9016557335853577,
861
+ "learning_rate": 4.495715690405956e-05,
862
+ "loss": 0.9842,
863
+ "mean_token_accuracy": 0.7541791129857301,
864
+ "num_tokens": 164184119.0,
865
+ "step": 940
866
+ },
867
+ {
868
+ "epoch": 0.2589040862559403,
869
+ "grad_norm": 0.8286603093147278,
870
+ "learning_rate": 4.488692232055064e-05,
871
+ "loss": 0.9433,
872
+ "mean_token_accuracy": 0.7619124032557011,
873
+ "num_tokens": 165971146.0,
874
+ "step": 950
875
+ },
876
+ {
877
+ "epoch": 0.26162939242705546,
878
+ "grad_norm": 0.9046083092689514,
879
+ "learning_rate": 4.4816687737041726e-05,
880
+ "loss": 0.9781,
881
+ "mean_token_accuracy": 0.7572765513323247,
882
+ "num_tokens": 167701701.0,
883
+ "step": 960
884
+ },
885
+ {
886
+ "epoch": 0.2643546985981706,
887
+ "grad_norm": 0.936540961265564,
888
+ "learning_rate": 4.47464531535328e-05,
889
+ "loss": 0.969,
890
+ "mean_token_accuracy": 0.7577900117263198,
891
+ "num_tokens": 169443326.0,
892
+ "step": 970
893
+ },
894
+ {
895
+ "epoch": 0.2670800047692858,
896
+ "grad_norm": 0.8902041912078857,
897
+ "learning_rate": 4.467621857002388e-05,
898
+ "loss": 0.9706,
899
+ "mean_token_accuracy": 0.7567241456359625,
900
+ "num_tokens": 171199385.0,
901
+ "step": 980
902
+ },
903
+ {
904
+ "epoch": 0.26980531094040094,
905
+ "grad_norm": 0.9205445647239685,
906
+ "learning_rate": 4.460598398651496e-05,
907
+ "loss": 0.9383,
908
+ "mean_token_accuracy": 0.7634873200207949,
909
+ "num_tokens": 172929493.0,
910
+ "step": 990
911
+ },
912
+ {
913
+ "epoch": 0.2725306171115161,
914
+ "grad_norm": 0.9309903979301453,
915
+ "learning_rate": 4.453574940300605e-05,
916
+ "loss": 0.9442,
917
+ "mean_token_accuracy": 0.7614085223525763,
918
+ "num_tokens": 174672601.0,
919
+ "step": 1000
920
+ },
921
+ {
922
+ "epoch": 0.27525592328263127,
923
+ "grad_norm": 0.8428636193275452,
924
+ "learning_rate": 4.446551481949712e-05,
925
+ "loss": 0.9607,
926
+ "mean_token_accuracy": 0.7597405662760138,
927
+ "num_tokens": 176441962.0,
928
+ "step": 1010
929
+ },
930
+ {
931
+ "epoch": 0.2779812294537464,
932
+ "grad_norm": 0.9295870661735535,
933
+ "learning_rate": 4.43952802359882e-05,
934
+ "loss": 0.9468,
935
+ "mean_token_accuracy": 0.7614248601719737,
936
+ "num_tokens": 178197361.0,
937
+ "step": 1020
938
+ },
939
+ {
940
+ "epoch": 0.2807065356248616,
941
+ "grad_norm": 0.9550367593765259,
942
+ "learning_rate": 4.432504565247929e-05,
943
+ "loss": 0.9766,
944
+ "mean_token_accuracy": 0.7538504358381033,
945
+ "num_tokens": 179887580.0,
946
+ "step": 1030
947
+ },
948
+ {
949
+ "epoch": 0.28343184179597675,
950
+ "grad_norm": 0.9089730978012085,
951
+ "learning_rate": 4.425481106897036e-05,
952
+ "loss": 0.9382,
953
+ "mean_token_accuracy": 0.7631622815504671,
954
+ "num_tokens": 181655865.0,
955
+ "step": 1040
956
+ },
957
+ {
958
+ "epoch": 0.2861571479670919,
959
+ "grad_norm": 1.0070922374725342,
960
+ "learning_rate": 4.418457648546144e-05,
961
+ "loss": 0.962,
962
+ "mean_token_accuracy": 0.7598225042223931,
963
+ "num_tokens": 183445880.0,
964
+ "step": 1050
965
+ },
966
+ {
967
+ "epoch": 0.28888245413820707,
968
+ "grad_norm": 0.8544902205467224,
969
+ "learning_rate": 4.411434190195252e-05,
970
+ "loss": 0.9296,
971
+ "mean_token_accuracy": 0.7648898587562144,
972
+ "num_tokens": 185175706.0,
973
+ "step": 1060
974
+ },
975
+ {
976
+ "epoch": 0.29160776030932223,
977
+ "grad_norm": 0.9063568115234375,
978
+ "learning_rate": 4.404410731844361e-05,
979
+ "loss": 0.942,
980
+ "mean_token_accuracy": 0.7613558363169431,
981
+ "num_tokens": 186873396.0,
982
+ "step": 1070
983
+ },
984
+ {
985
+ "epoch": 0.2943330664804374,
986
+ "grad_norm": 0.8668667078018188,
987
+ "learning_rate": 4.397387273493468e-05,
988
+ "loss": 0.9765,
989
+ "mean_token_accuracy": 0.7555979456752538,
990
+ "num_tokens": 188591288.0,
991
+ "step": 1080
992
+ },
993
+ {
994
+ "epoch": 0.29705837265155255,
995
+ "grad_norm": 0.9103459119796753,
996
+ "learning_rate": 4.390363815142576e-05,
997
+ "loss": 0.9691,
998
+ "mean_token_accuracy": 0.7566804554313421,
999
+ "num_tokens": 190375182.0,
1000
+ "step": 1090
1001
+ },
1002
+ {
1003
+ "epoch": 0.2997836788226677,
1004
+ "grad_norm": 0.9022314548492432,
1005
+ "learning_rate": 4.383340356791684e-05,
1006
+ "loss": 0.955,
1007
+ "mean_token_accuracy": 0.7589525409042835,
1008
+ "num_tokens": 192104609.0,
1009
+ "step": 1100
1010
+ },
1011
+ {
1012
+ "epoch": 0.30250898499378287,
1013
+ "grad_norm": 0.8525939583778381,
1014
+ "learning_rate": 4.376316898440793e-05,
1015
+ "loss": 0.8952,
1016
+ "mean_token_accuracy": 0.7715249681845308,
1017
+ "num_tokens": 193819768.0,
1018
+ "step": 1110
1019
+ },
1020
+ {
1021
+ "epoch": 0.30523429116489803,
1022
+ "grad_norm": 0.8655187487602234,
1023
+ "learning_rate": 4.3692934400899e-05,
1024
+ "loss": 0.957,
1025
+ "mean_token_accuracy": 0.7608988376334309,
1026
+ "num_tokens": 195541705.0,
1027
+ "step": 1120
1028
+ },
1029
+ {
1030
+ "epoch": 0.3079595973360132,
1031
+ "grad_norm": 0.8646089434623718,
1032
+ "learning_rate": 4.362269981739008e-05,
1033
+ "loss": 0.944,
1034
+ "mean_token_accuracy": 0.7616777900606394,
1035
+ "num_tokens": 197250976.0,
1036
+ "step": 1130
1037
+ },
1038
+ {
1039
+ "epoch": 0.31068490350712835,
1040
+ "grad_norm": 0.8167153000831604,
1041
+ "learning_rate": 4.355246523388117e-05,
1042
+ "loss": 0.9149,
1043
+ "mean_token_accuracy": 0.7681540355086327,
1044
+ "num_tokens": 198964087.0,
1045
+ "step": 1140
1046
+ },
1047
+ {
1048
+ "epoch": 0.31341020967824357,
1049
+ "grad_norm": 0.9367976188659668,
1050
+ "learning_rate": 4.348223065037224e-05,
1051
+ "loss": 0.9599,
1052
+ "mean_token_accuracy": 0.7586290875449777,
1053
+ "num_tokens": 200654341.0,
1054
+ "step": 1150
1055
+ },
1056
+ {
1057
+ "epoch": 0.31613551584935873,
1058
+ "grad_norm": 0.8891621828079224,
1059
+ "learning_rate": 4.3411996066863323e-05,
1060
+ "loss": 0.9865,
1061
+ "mean_token_accuracy": 0.7538609990850091,
1062
+ "num_tokens": 202410389.0,
1063
+ "step": 1160
1064
+ },
1065
+ {
1066
+ "epoch": 0.3188608220204739,
1067
+ "grad_norm": 0.8583462834358215,
1068
+ "learning_rate": 4.3341761483354404e-05,
1069
+ "loss": 0.9421,
1070
+ "mean_token_accuracy": 0.7621322822757065,
1071
+ "num_tokens": 204176073.0,
1072
+ "step": 1170
1073
+ },
1074
+ {
1075
+ "epoch": 0.32158612819158905,
1076
+ "grad_norm": 0.9226254224777222,
1077
+ "learning_rate": 4.327152689984549e-05,
1078
+ "loss": 0.9369,
1079
+ "mean_token_accuracy": 0.7635338991880417,
1080
+ "num_tokens": 206017214.0,
1081
+ "step": 1180
1082
+ },
1083
+ {
1084
+ "epoch": 0.3243114343627042,
1085
+ "grad_norm": 0.958926796913147,
1086
+ "learning_rate": 4.3201292316336564e-05,
1087
+ "loss": 0.948,
1088
+ "mean_token_accuracy": 0.7608529534190893,
1089
+ "num_tokens": 207832984.0,
1090
+ "step": 1190
1091
+ },
1092
+ {
1093
+ "epoch": 0.32703674053381937,
1094
+ "grad_norm": 0.9389734268188477,
1095
+ "learning_rate": 4.3131057732827644e-05,
1096
+ "loss": 0.9579,
1097
+ "mean_token_accuracy": 0.7594182942993939,
1098
+ "num_tokens": 209533427.0,
1099
+ "step": 1200
1100
+ },
1101
+ {
1102
+ "epoch": 0.32976204670493453,
1103
+ "grad_norm": 0.8880856037139893,
1104
+ "learning_rate": 4.3060823149318724e-05,
1105
+ "loss": 0.9352,
1106
+ "mean_token_accuracy": 0.7635649677366019,
1107
+ "num_tokens": 211344332.0,
1108
+ "step": 1210
1109
+ },
1110
+ {
1111
+ "epoch": 0.3324873528760497,
1112
+ "grad_norm": 0.9591731429100037,
1113
+ "learning_rate": 4.299058856580981e-05,
1114
+ "loss": 0.9633,
1115
+ "mean_token_accuracy": 0.7583153389394284,
1116
+ "num_tokens": 213088746.0,
1117
+ "step": 1220
1118
+ },
1119
+ {
1120
+ "epoch": 0.33521265904716485,
1121
+ "grad_norm": 0.9024151563644409,
1122
+ "learning_rate": 4.2920353982300885e-05,
1123
+ "loss": 0.9647,
1124
+ "mean_token_accuracy": 0.7571177691221237,
1125
+ "num_tokens": 214889528.0,
1126
+ "step": 1230
1127
+ },
1128
+ {
1129
+ "epoch": 0.33793796521828,
1130
+ "grad_norm": 0.9884682893753052,
1131
+ "learning_rate": 4.2850119398791965e-05,
1132
+ "loss": 0.9719,
1133
+ "mean_token_accuracy": 0.7572076172567904,
1134
+ "num_tokens": 216603081.0,
1135
+ "step": 1240
1136
+ },
1137
+ {
1138
+ "epoch": 0.3406632713893952,
1139
+ "grad_norm": 0.8637387752532959,
1140
+ "learning_rate": 4.277988481528305e-05,
1141
+ "loss": 0.9211,
1142
+ "mean_token_accuracy": 0.7659906836226582,
1143
+ "num_tokens": 218292845.0,
1144
+ "step": 1250
1145
+ },
1146
+ {
1147
+ "epoch": 0.34338857756051033,
1148
+ "grad_norm": 0.9273796081542969,
1149
+ "learning_rate": 4.2709650231774125e-05,
1150
+ "loss": 0.9454,
1151
+ "mean_token_accuracy": 0.7626127757132053,
1152
+ "num_tokens": 220029696.0,
1153
+ "step": 1260
1154
+ },
1155
+ {
1156
+ "epoch": 0.3461138837316255,
1157
+ "grad_norm": 0.8521084189414978,
1158
+ "learning_rate": 4.2639415648265206e-05,
1159
+ "loss": 1.0002,
1160
+ "mean_token_accuracy": 0.7520326241850853,
1161
+ "num_tokens": 221810553.0,
1162
+ "step": 1270
1163
+ },
1164
+ {
1165
+ "epoch": 0.34883918990274065,
1166
+ "grad_norm": 0.9349818229675293,
1167
+ "learning_rate": 4.2569181064756286e-05,
1168
+ "loss": 0.9412,
1169
+ "mean_token_accuracy": 0.7652541293762625,
1170
+ "num_tokens": 223552801.0,
1171
+ "step": 1280
1172
+ },
1173
+ {
1174
+ "epoch": 0.3515644960738558,
1175
+ "grad_norm": 0.9496633410453796,
1176
+ "learning_rate": 4.249894648124737e-05,
1177
+ "loss": 0.9486,
1178
+ "mean_token_accuracy": 0.7618991539813578,
1179
+ "num_tokens": 225254929.0,
1180
+ "step": 1290
1181
+ },
1182
+ {
1183
+ "epoch": 0.354289802244971,
1184
+ "grad_norm": 0.8988118767738342,
1185
+ "learning_rate": 4.2428711897738446e-05,
1186
+ "loss": 0.9679,
1187
+ "mean_token_accuracy": 0.7575280690565706,
1188
+ "num_tokens": 227017304.0,
1189
+ "step": 1300
1190
+ },
1191
+ {
1192
+ "epoch": 0.35701510841608614,
1193
+ "grad_norm": 0.9880586862564087,
1194
+ "learning_rate": 4.2358477314229526e-05,
1195
+ "loss": 0.9199,
1196
+ "mean_token_accuracy": 0.7675149342045188,
1197
+ "num_tokens": 228785771.0,
1198
+ "step": 1310
1199
+ },
1200
+ {
1201
+ "epoch": 0.3597404145872013,
1202
+ "grad_norm": 0.9548985958099365,
1203
+ "learning_rate": 4.2288242730720607e-05,
1204
+ "loss": 0.9276,
1205
+ "mean_token_accuracy": 0.7658209035173058,
1206
+ "num_tokens": 230489667.0,
1207
+ "step": 1320
1208
+ },
1209
+ {
1210
+ "epoch": 0.36246572075831646,
1211
+ "grad_norm": 0.8222749829292297,
1212
+ "learning_rate": 4.2218008147211694e-05,
1213
+ "loss": 0.9464,
1214
+ "mean_token_accuracy": 0.7618509028106928,
1215
+ "num_tokens": 232269581.0,
1216
+ "step": 1330
1217
+ },
1218
+ {
1219
+ "epoch": 0.3651910269294316,
1220
+ "grad_norm": 0.8697149753570557,
1221
+ "learning_rate": 4.214777356370277e-05,
1222
+ "loss": 0.9507,
1223
+ "mean_token_accuracy": 0.7589831293560565,
1224
+ "num_tokens": 234028126.0,
1225
+ "step": 1340
1226
+ },
1227
+ {
1228
+ "epoch": 0.3679163331005468,
1229
+ "grad_norm": 1.0205236673355103,
1230
+ "learning_rate": 4.207753898019385e-05,
1231
+ "loss": 0.9703,
1232
+ "mean_token_accuracy": 0.7572768048383296,
1233
+ "num_tokens": 235817129.0,
1234
+ "step": 1350
1235
+ },
1236
+ {
1237
+ "epoch": 0.37064163927166194,
1238
+ "grad_norm": 0.8218963742256165,
1239
+ "learning_rate": 4.2007304396684934e-05,
1240
+ "loss": 0.9508,
1241
+ "mean_token_accuracy": 0.7612483019009233,
1242
+ "num_tokens": 237588984.0,
1243
+ "step": 1360
1244
+ },
1245
+ {
1246
+ "epoch": 0.3733669454427771,
1247
+ "grad_norm": 0.9388371109962463,
1248
+ "learning_rate": 4.193706981317601e-05,
1249
+ "loss": 0.9144,
1250
+ "mean_token_accuracy": 0.7678116019815207,
1251
+ "num_tokens": 239305200.0,
1252
+ "step": 1370
1253
+ },
1254
+ {
1255
+ "epoch": 0.37609225161389226,
1256
+ "grad_norm": 0.8573911190032959,
1257
+ "learning_rate": 4.186683522966709e-05,
1258
+ "loss": 0.9368,
1259
+ "mean_token_accuracy": 0.7639053143560887,
1260
+ "num_tokens": 241078759.0,
1261
+ "step": 1380
1262
+ },
1263
+ {
1264
+ "epoch": 0.3788175577850074,
1265
+ "grad_norm": 0.918056845664978,
1266
+ "learning_rate": 4.179660064615817e-05,
1267
+ "loss": 0.9596,
1268
+ "mean_token_accuracy": 0.7594995486550034,
1269
+ "num_tokens": 242876841.0,
1270
+ "step": 1390
1271
+ },
1272
+ {
1273
+ "epoch": 0.3815428639561226,
1274
+ "grad_norm": 0.8583497405052185,
1275
+ "learning_rate": 4.1726366062649255e-05,
1276
+ "loss": 0.9252,
1277
+ "mean_token_accuracy": 0.7642927357926965,
1278
+ "num_tokens": 244578724.0,
1279
+ "step": 1400
1280
+ },
1281
+ {
1282
+ "epoch": 0.38426817012723774,
1283
+ "grad_norm": 0.8453887104988098,
1284
+ "learning_rate": 4.165613147914033e-05,
1285
+ "loss": 0.9525,
1286
+ "mean_token_accuracy": 0.7590734004974365,
1287
+ "num_tokens": 246363009.0,
1288
+ "step": 1410
1289
+ },
1290
+ {
1291
+ "epoch": 0.3869934762983529,
1292
+ "grad_norm": 0.8691115975379944,
1293
+ "learning_rate": 4.158589689563141e-05,
1294
+ "loss": 0.9764,
1295
+ "mean_token_accuracy": 0.7560055317357183,
1296
+ "num_tokens": 248120613.0,
1297
+ "step": 1420
1298
+ },
1299
+ {
1300
+ "epoch": 0.38971878246946806,
1301
+ "grad_norm": 0.8741028904914856,
1302
+ "learning_rate": 4.151566231212249e-05,
1303
+ "loss": 0.9514,
1304
+ "mean_token_accuracy": 0.7612073054537177,
1305
+ "num_tokens": 249941867.0,
1306
+ "step": 1430
1307
+ },
1308
+ {
1309
+ "epoch": 0.3924440886405832,
1310
+ "grad_norm": 0.8339214324951172,
1311
+ "learning_rate": 4.1445427728613576e-05,
1312
+ "loss": 0.9189,
1313
+ "mean_token_accuracy": 0.7668958367779851,
1314
+ "num_tokens": 251695734.0,
1315
+ "step": 1440
1316
+ },
1317
+ {
1318
+ "epoch": 0.3951693948116984,
1319
+ "grad_norm": 0.8296242356300354,
1320
+ "learning_rate": 4.137519314510465e-05,
1321
+ "loss": 0.9627,
1322
+ "mean_token_accuracy": 0.7588108706288039,
1323
+ "num_tokens": 253503211.0,
1324
+ "step": 1450
1325
+ },
1326
+ {
1327
+ "epoch": 0.39789470098281354,
1328
+ "grad_norm": 1.0061241388320923,
1329
+ "learning_rate": 4.130495856159573e-05,
1330
+ "loss": 0.9399,
1331
+ "mean_token_accuracy": 0.7630201263353229,
1332
+ "num_tokens": 255313734.0,
1333
+ "step": 1460
1334
+ },
1335
+ {
1336
+ "epoch": 0.4006200071539287,
1337
+ "grad_norm": 0.795073390007019,
1338
+ "learning_rate": 4.1234723978086816e-05,
1339
+ "loss": 0.8906,
1340
+ "mean_token_accuracy": 0.7732970826327801,
1341
+ "num_tokens": 257016513.0,
1342
+ "step": 1470
1343
+ },
1344
+ {
1345
+ "epoch": 0.40334531332504386,
1346
+ "grad_norm": 0.9418059587478638,
1347
+ "learning_rate": 4.1164489394577896e-05,
1348
+ "loss": 0.9642,
1349
+ "mean_token_accuracy": 0.757817355170846,
1350
+ "num_tokens": 258760362.0,
1351
+ "step": 1480
1352
+ },
1353
+ {
1354
+ "epoch": 0.406070619496159,
1355
+ "grad_norm": 0.8528169989585876,
1356
+ "learning_rate": 4.109425481106897e-05,
1357
+ "loss": 0.924,
1358
+ "mean_token_accuracy": 0.7663666036911309,
1359
+ "num_tokens": 260546971.0,
1360
+ "step": 1490
1361
+ },
1362
+ {
1363
+ "epoch": 0.4087959256672742,
1364
+ "grad_norm": 0.8477678298950195,
1365
+ "learning_rate": 4.102402022756005e-05,
1366
+ "loss": 0.9325,
1367
+ "mean_token_accuracy": 0.7635815197601914,
1368
+ "num_tokens": 262307672.0,
1369
+ "step": 1500
1370
+ },
1371
+ {
1372
+ "epoch": 0.41152123183838935,
1373
+ "grad_norm": 0.9021455645561218,
1374
+ "learning_rate": 4.095378564405114e-05,
1375
+ "loss": 0.9451,
1376
+ "mean_token_accuracy": 0.7621158785186708,
1377
+ "num_tokens": 264023889.0,
1378
+ "step": 1510
1379
+ },
1380
+ {
1381
+ "epoch": 0.4142465380095045,
1382
+ "grad_norm": 0.8655399084091187,
1383
+ "learning_rate": 4.088355106054221e-05,
1384
+ "loss": 0.9435,
1385
+ "mean_token_accuracy": 0.76206961190328,
1386
+ "num_tokens": 265798519.0,
1387
+ "step": 1520
1388
+ },
1389
+ {
1390
+ "epoch": 0.41697184418061967,
1391
+ "grad_norm": 0.8364386558532715,
1392
+ "learning_rate": 4.081331647703329e-05,
1393
+ "loss": 0.9599,
1394
+ "mean_token_accuracy": 0.7590883946046233,
1395
+ "num_tokens": 267546100.0,
1396
+ "step": 1530
1397
+ },
1398
+ {
1399
+ "epoch": 0.4196971503517348,
1400
+ "grad_norm": 0.8579459190368652,
1401
+ "learning_rate": 4.074308189352437e-05,
1402
+ "loss": 0.9233,
1403
+ "mean_token_accuracy": 0.7652724193409085,
1404
+ "num_tokens": 269297641.0,
1405
+ "step": 1540
1406
+ },
1407
+ {
1408
+ "epoch": 0.42242245652285,
1409
+ "grad_norm": 0.7599934339523315,
1410
+ "learning_rate": 4.067284731001546e-05,
1411
+ "loss": 0.9606,
1412
+ "mean_token_accuracy": 0.7579821115359664,
1413
+ "num_tokens": 271053006.0,
1414
+ "step": 1550
1415
+ },
1416
+ {
1417
+ "epoch": 0.42514776269396515,
1418
+ "grad_norm": 0.934144139289856,
1419
+ "learning_rate": 4.060261272650653e-05,
1420
+ "loss": 0.9439,
1421
+ "mean_token_accuracy": 0.7628989959135651,
1422
+ "num_tokens": 272829487.0,
1423
+ "step": 1560
1424
+ },
1425
+ {
1426
+ "epoch": 0.4278730688650803,
1427
+ "grad_norm": 0.8444865345954895,
1428
+ "learning_rate": 4.053237814299761e-05,
1429
+ "loss": 0.937,
1430
+ "mean_token_accuracy": 0.7626888936385512,
1431
+ "num_tokens": 274621990.0,
1432
+ "step": 1570
1433
+ },
1434
+ {
1435
+ "epoch": 0.43059837503619547,
1436
+ "grad_norm": 0.854570209980011,
1437
+ "learning_rate": 4.04621435594887e-05,
1438
+ "loss": 0.9555,
1439
+ "mean_token_accuracy": 0.7589856326580048,
1440
+ "num_tokens": 276398989.0,
1441
+ "step": 1580
1442
+ },
1443
+ {
1444
+ "epoch": 0.43332368120731063,
1445
+ "grad_norm": 0.916355550289154,
1446
+ "learning_rate": 4.039190897597978e-05,
1447
+ "loss": 0.9217,
1448
+ "mean_token_accuracy": 0.7648066570982337,
1449
+ "num_tokens": 278192884.0,
1450
+ "step": 1590
1451
+ },
1452
+ {
1453
+ "epoch": 0.4360489873784258,
1454
+ "grad_norm": 0.9150972962379456,
1455
+ "learning_rate": 4.032167439247085e-05,
1456
+ "loss": 0.9223,
1457
+ "mean_token_accuracy": 0.7662323093041777,
1458
+ "num_tokens": 279937931.0,
1459
+ "step": 1600
1460
+ },
1461
+ {
1462
+ "epoch": 0.43877429354954095,
1463
+ "grad_norm": 0.7945618629455566,
1464
+ "learning_rate": 4.025143980896193e-05,
1465
+ "loss": 0.9226,
1466
+ "mean_token_accuracy": 0.7670198388397693,
1467
+ "num_tokens": 281698143.0,
1468
+ "step": 1610
1469
+ },
1470
+ {
1471
+ "epoch": 0.4414995997206561,
1472
+ "grad_norm": 0.826994776725769,
1473
+ "learning_rate": 4.018120522545302e-05,
1474
+ "loss": 0.9348,
1475
+ "mean_token_accuracy": 0.7642012946307659,
1476
+ "num_tokens": 283412565.0,
1477
+ "step": 1620
1478
+ },
1479
+ {
1480
+ "epoch": 0.44422490589177127,
1481
+ "grad_norm": 0.9012675881385803,
1482
+ "learning_rate": 4.011097064194409e-05,
1483
+ "loss": 0.9413,
1484
+ "mean_token_accuracy": 0.7626177223399282,
1485
+ "num_tokens": 285184244.0,
1486
+ "step": 1630
1487
+ },
1488
+ {
1489
+ "epoch": 0.44695021206288643,
1490
+ "grad_norm": 0.7998985648155212,
1491
+ "learning_rate": 4.004073605843517e-05,
1492
+ "loss": 0.8893,
1493
+ "mean_token_accuracy": 0.7731544019654393,
1494
+ "num_tokens": 286963226.0,
1495
+ "step": 1640
1496
+ },
1497
+ {
1498
+ "epoch": 0.4496755182340016,
1499
+ "grad_norm": 0.894074559211731,
1500
+ "learning_rate": 3.997050147492625e-05,
1501
+ "loss": 0.9101,
1502
+ "mean_token_accuracy": 0.7684361206367611,
1503
+ "num_tokens": 288742524.0,
1504
+ "step": 1650
1505
+ },
1506
+ {
1507
+ "epoch": 0.45240082440511675,
1508
+ "grad_norm": 0.9063016176223755,
1509
+ "learning_rate": 3.990026689141734e-05,
1510
+ "loss": 0.9408,
1511
+ "mean_token_accuracy": 0.7627444183453918,
1512
+ "num_tokens": 290486439.0,
1513
+ "step": 1660
1514
+ },
1515
+ {
1516
+ "epoch": 0.4551261305762319,
1517
+ "grad_norm": 0.9253380298614502,
1518
+ "learning_rate": 3.9830032307908413e-05,
1519
+ "loss": 0.9489,
1520
+ "mean_token_accuracy": 0.761582453828305,
1521
+ "num_tokens": 292244616.0,
1522
+ "step": 1670
1523
+ },
1524
+ {
1525
+ "epoch": 0.4578514367473471,
1526
+ "grad_norm": 0.908446192741394,
1527
+ "learning_rate": 3.9759797724399494e-05,
1528
+ "loss": 0.9435,
1529
+ "mean_token_accuracy": 0.7627298279665411,
1530
+ "num_tokens": 294028863.0,
1531
+ "step": 1680
1532
+ },
1533
+ {
1534
+ "epoch": 0.46057674291846223,
1535
+ "grad_norm": 0.8264852166175842,
1536
+ "learning_rate": 3.968956314089058e-05,
1537
+ "loss": 0.9158,
1538
+ "mean_token_accuracy": 0.765919309295714,
1539
+ "num_tokens": 295842660.0,
1540
+ "step": 1690
1541
+ },
1542
+ {
1543
+ "epoch": 0.4633020490895774,
1544
+ "grad_norm": 0.8423106074333191,
1545
+ "learning_rate": 3.961932855738166e-05,
1546
+ "loss": 0.9214,
1547
+ "mean_token_accuracy": 0.767298517934978,
1548
+ "num_tokens": 297606076.0,
1549
+ "step": 1700
1550
+ },
1551
+ {
1552
+ "epoch": 0.46602735526069256,
1553
+ "grad_norm": 0.8728176355361938,
1554
+ "learning_rate": 3.9549093973872734e-05,
1555
+ "loss": 0.9501,
1556
+ "mean_token_accuracy": 0.7602741612121463,
1557
+ "num_tokens": 299367632.0,
1558
+ "step": 1710
1559
+ },
1560
+ {
1561
+ "epoch": 0.4687526614318077,
1562
+ "grad_norm": 0.7749947309494019,
1563
+ "learning_rate": 3.9478859390363814e-05,
1564
+ "loss": 0.9076,
1565
+ "mean_token_accuracy": 0.770167650654912,
1566
+ "num_tokens": 301097990.0,
1567
+ "step": 1720
1568
+ },
1569
+ {
1570
+ "epoch": 0.4714779676029229,
1571
+ "grad_norm": 0.9254289865493774,
1572
+ "learning_rate": 3.94086248068549e-05,
1573
+ "loss": 0.9142,
1574
+ "mean_token_accuracy": 0.7686513620428741,
1575
+ "num_tokens": 302847308.0,
1576
+ "step": 1730
1577
+ },
1578
+ {
1579
+ "epoch": 0.47420327377403804,
1580
+ "grad_norm": 0.870252788066864,
1581
+ "learning_rate": 3.9338390223345975e-05,
1582
+ "loss": 0.9356,
1583
+ "mean_token_accuracy": 0.7628345469944179,
1584
+ "num_tokens": 304582625.0,
1585
+ "step": 1740
1586
+ },
1587
+ {
1588
+ "epoch": 0.4769285799451532,
1589
+ "grad_norm": 0.845609188079834,
1590
+ "learning_rate": 3.9268155639837055e-05,
1591
+ "loss": 0.9179,
1592
+ "mean_token_accuracy": 0.7681567031890154,
1593
+ "num_tokens": 306358206.0,
1594
+ "step": 1750
1595
+ },
1596
+ {
1597
+ "epoch": 0.47965388611626836,
1598
+ "grad_norm": 0.8974276185035706,
1599
+ "learning_rate": 3.9197921056328135e-05,
1600
+ "loss": 0.9172,
1601
+ "mean_token_accuracy": 0.7672481311485171,
1602
+ "num_tokens": 308026562.0,
1603
+ "step": 1760
1604
+ },
1605
+ {
1606
+ "epoch": 0.4823791922873835,
1607
+ "grad_norm": 0.9142021536827087,
1608
+ "learning_rate": 3.912768647281922e-05,
1609
+ "loss": 0.9492,
1610
+ "mean_token_accuracy": 0.7608077014796436,
1611
+ "num_tokens": 309804543.0,
1612
+ "step": 1770
1613
+ },
1614
+ {
1615
+ "epoch": 0.4851044984584987,
1616
+ "grad_norm": 0.8676534295082092,
1617
+ "learning_rate": 3.9057451889310296e-05,
1618
+ "loss": 0.9406,
1619
+ "mean_token_accuracy": 0.7620649378746748,
1620
+ "num_tokens": 311549771.0,
1621
+ "step": 1780
1622
+ },
1623
+ {
1624
+ "epoch": 0.48782980462961384,
1625
+ "grad_norm": 0.9465579986572266,
1626
+ "learning_rate": 3.8987217305801376e-05,
1627
+ "loss": 0.9559,
1628
+ "mean_token_accuracy": 0.7597391474992037,
1629
+ "num_tokens": 313326889.0,
1630
+ "step": 1790
1631
+ },
1632
+ {
1633
+ "epoch": 0.490555110800729,
1634
+ "grad_norm": 1.1392822265625,
1635
+ "learning_rate": 3.891698272229246e-05,
1636
+ "loss": 0.9123,
1637
+ "mean_token_accuracy": 0.7693880930542946,
1638
+ "num_tokens": 315064715.0,
1639
+ "step": 1800
1640
+ },
1641
+ {
1642
+ "epoch": 0.49328041697184416,
1643
+ "grad_norm": 0.8468901515007019,
1644
+ "learning_rate": 3.884674813878354e-05,
1645
+ "loss": 0.9293,
1646
+ "mean_token_accuracy": 0.7640517603605985,
1647
+ "num_tokens": 316809509.0,
1648
+ "step": 1810
1649
+ },
1650
+ {
1651
+ "epoch": 0.4960057231429593,
1652
+ "grad_norm": 0.831238865852356,
1653
+ "learning_rate": 3.8776513555274616e-05,
1654
+ "loss": 0.9097,
1655
+ "mean_token_accuracy": 0.7691411901265383,
1656
+ "num_tokens": 318597481.0,
1657
+ "step": 1820
1658
+ },
1659
+ {
1660
+ "epoch": 0.4987310293140745,
1661
+ "grad_norm": 0.8473038077354431,
1662
+ "learning_rate": 3.8706278971765697e-05,
1663
+ "loss": 0.9099,
1664
+ "mean_token_accuracy": 0.7685648757964373,
1665
+ "num_tokens": 320382863.0,
1666
+ "step": 1830
1667
+ },
1668
+ {
1669
+ "epoch": 0.5014563354851896,
1670
+ "grad_norm": 0.8610597848892212,
1671
+ "learning_rate": 3.8636044388256784e-05,
1672
+ "loss": 0.9255,
1673
+ "mean_token_accuracy": 0.7672168118879199,
1674
+ "num_tokens": 322136431.0,
1675
+ "step": 1840
1676
+ },
1677
+ {
1678
+ "epoch": 0.5041816416563049,
1679
+ "grad_norm": 0.8556321859359741,
1680
+ "learning_rate": 3.856580980474786e-05,
1681
+ "loss": 0.9108,
1682
+ "mean_token_accuracy": 0.7693611716851592,
1683
+ "num_tokens": 323892089.0,
1684
+ "step": 1850
1685
+ },
1686
+ {
1687
+ "epoch": 0.50690694782742,
1688
+ "grad_norm": 0.8000465631484985,
1689
+ "learning_rate": 3.849557522123894e-05,
1690
+ "loss": 0.9198,
1691
+ "mean_token_accuracy": 0.7665763789787888,
1692
+ "num_tokens": 325693457.0,
1693
+ "step": 1860
1694
+ },
1695
+ {
1696
+ "epoch": 0.5096322539985352,
1697
+ "grad_norm": 0.9535455107688904,
1698
+ "learning_rate": 3.842534063773002e-05,
1699
+ "loss": 0.8953,
1700
+ "mean_token_accuracy": 0.7704567154869437,
1701
+ "num_tokens": 327397884.0,
1702
+ "step": 1870
1703
+ },
1704
+ {
1705
+ "epoch": 0.5123575601696503,
1706
+ "grad_norm": 0.9103294610977173,
1707
+ "learning_rate": 3.8355106054221104e-05,
1708
+ "loss": 0.9415,
1709
+ "mean_token_accuracy": 0.7628723161295057,
1710
+ "num_tokens": 329077502.0,
1711
+ "step": 1880
1712
+ },
1713
+ {
1714
+ "epoch": 0.5150828663407655,
1715
+ "grad_norm": 0.8798259496688843,
1716
+ "learning_rate": 3.828487147071218e-05,
1717
+ "loss": 0.9294,
1718
+ "mean_token_accuracy": 0.7647034784778952,
1719
+ "num_tokens": 330847695.0,
1720
+ "step": 1890
1721
+ },
1722
+ {
1723
+ "epoch": 0.5178081725118806,
1724
+ "grad_norm": 0.8437985181808472,
1725
+ "learning_rate": 3.821463688720326e-05,
1726
+ "loss": 0.9127,
1727
+ "mean_token_accuracy": 0.7662377031520009,
1728
+ "num_tokens": 332578366.0,
1729
+ "step": 1900
1730
+ },
1731
+ {
1732
+ "epoch": 0.5205334786829958,
1733
+ "grad_norm": 0.877572774887085,
1734
+ "learning_rate": 3.8144402303694345e-05,
1735
+ "loss": 0.9395,
1736
+ "mean_token_accuracy": 0.7628075636923313,
1737
+ "num_tokens": 334293589.0,
1738
+ "step": 1910
1739
+ },
1740
+ {
1741
+ "epoch": 0.5232587848541109,
1742
+ "grad_norm": 0.7866714596748352,
1743
+ "learning_rate": 3.8074167720185425e-05,
1744
+ "loss": 0.9268,
1745
+ "mean_token_accuracy": 0.7632251624017954,
1746
+ "num_tokens": 336016972.0,
1747
+ "step": 1920
1748
+ },
1749
+ {
1750
+ "epoch": 0.5259840910252261,
1751
+ "grad_norm": 0.8031445741653442,
1752
+ "learning_rate": 3.80039331366765e-05,
1753
+ "loss": 0.9032,
1754
+ "mean_token_accuracy": 0.7699568081647158,
1755
+ "num_tokens": 337805977.0,
1756
+ "step": 1930
1757
+ },
1758
+ {
1759
+ "epoch": 0.5287093971963412,
1760
+ "grad_norm": 0.7914207577705383,
1761
+ "learning_rate": 3.793369855316758e-05,
1762
+ "loss": 0.8779,
1763
+ "mean_token_accuracy": 0.7737740287557244,
1764
+ "num_tokens": 339548221.0,
1765
+ "step": 1940
1766
+ },
1767
+ {
1768
+ "epoch": 0.5314347033674565,
1769
+ "grad_norm": 0.8687313795089722,
1770
+ "learning_rate": 3.7863463969658666e-05,
1771
+ "loss": 0.9549,
1772
+ "mean_token_accuracy": 0.7590207350440323,
1773
+ "num_tokens": 341268344.0,
1774
+ "step": 1950
1775
+ },
1776
+ {
1777
+ "epoch": 0.5341600095385716,
1778
+ "grad_norm": 0.9336804747581482,
1779
+ "learning_rate": 3.779322938614974e-05,
1780
+ "loss": 0.9221,
1781
+ "mean_token_accuracy": 0.765609790198505,
1782
+ "num_tokens": 343017522.0,
1783
+ "step": 1960
1784
+ },
1785
+ {
1786
+ "epoch": 0.5368853157096868,
1787
+ "grad_norm": 0.9002737998962402,
1788
+ "learning_rate": 3.772299480264082e-05,
1789
+ "loss": 0.9181,
1790
+ "mean_token_accuracy": 0.7670193919911981,
1791
+ "num_tokens": 344699192.0,
1792
+ "step": 1970
1793
+ },
1794
+ {
1795
+ "epoch": 0.5396106218808019,
1796
+ "grad_norm": 0.8401851654052734,
1797
+ "learning_rate": 3.7652760219131906e-05,
1798
+ "loss": 0.9043,
1799
+ "mean_token_accuracy": 0.7700289068743587,
1800
+ "num_tokens": 346458527.0,
1801
+ "step": 1980
1802
+ },
1803
+ {
1804
+ "epoch": 0.5423359280519171,
1805
+ "grad_norm": 0.8544691801071167,
1806
+ "learning_rate": 3.7582525635622986e-05,
1807
+ "loss": 0.9203,
1808
+ "mean_token_accuracy": 0.7662555683404207,
1809
+ "num_tokens": 348204585.0,
1810
+ "step": 1990
1811
+ },
1812
+ {
1813
+ "epoch": 0.5450612342230322,
1814
+ "grad_norm": 0.8688130378723145,
1815
+ "learning_rate": 3.751229105211406e-05,
1816
+ "loss": 0.9467,
1817
+ "mean_token_accuracy": 0.7613058963790535,
1818
+ "num_tokens": 349928259.0,
1819
+ "step": 2000
1820
+ },
1821
+ {
1822
+ "epoch": 0.5477865403941474,
1823
+ "grad_norm": 0.856677234172821,
1824
+ "learning_rate": 3.744205646860514e-05,
1825
+ "loss": 0.9124,
1826
+ "mean_token_accuracy": 0.767791903577745,
1827
+ "num_tokens": 351568018.0,
1828
+ "step": 2010
1829
+ },
1830
+ {
1831
+ "epoch": 0.5505118465652625,
1832
+ "grad_norm": 0.8962033987045288,
1833
+ "learning_rate": 3.737182188509623e-05,
1834
+ "loss": 0.8993,
1835
+ "mean_token_accuracy": 0.7711729943752289,
1836
+ "num_tokens": 353325716.0,
1837
+ "step": 2020
1838
+ },
1839
+ {
1840
+ "epoch": 0.5532371527363777,
1841
+ "grad_norm": 0.8988960385322571,
1842
+ "learning_rate": 3.730158730158731e-05,
1843
+ "loss": 0.9338,
1844
+ "mean_token_accuracy": 0.7638613487593829,
1845
+ "num_tokens": 355099655.0,
1846
+ "step": 2030
1847
+ },
1848
+ {
1849
+ "epoch": 0.5559624589074929,
1850
+ "grad_norm": 0.7775430083274841,
1851
+ "learning_rate": 3.723135271807838e-05,
1852
+ "loss": 0.9109,
1853
+ "mean_token_accuracy": 0.7683175005018711,
1854
+ "num_tokens": 356892448.0,
1855
+ "step": 2040
1856
+ },
1857
+ {
1858
+ "epoch": 0.5586877650786081,
1859
+ "grad_norm": 0.8810161352157593,
1860
+ "learning_rate": 3.716111813456946e-05,
1861
+ "loss": 0.9376,
1862
+ "mean_token_accuracy": 0.7628290761262179,
1863
+ "num_tokens": 358599855.0,
1864
+ "step": 2050
1865
+ },
1866
+ {
1867
+ "epoch": 0.5614130712497232,
1868
+ "grad_norm": 0.8723873496055603,
1869
+ "learning_rate": 3.709088355106055e-05,
1870
+ "loss": 0.9433,
1871
+ "mean_token_accuracy": 0.7625115603208542,
1872
+ "num_tokens": 360366528.0,
1873
+ "step": 2060
1874
+ },
1875
+ {
1876
+ "epoch": 0.5641383774208384,
1877
+ "grad_norm": 0.9130221605300903,
1878
+ "learning_rate": 3.702064896755162e-05,
1879
+ "loss": 0.915,
1880
+ "mean_token_accuracy": 0.7676266288384795,
1881
+ "num_tokens": 362123903.0,
1882
+ "step": 2070
1883
+ },
1884
+ {
1885
+ "epoch": 0.5668636835919535,
1886
+ "grad_norm": 0.7740192413330078,
1887
+ "learning_rate": 3.69504143840427e-05,
1888
+ "loss": 0.8998,
1889
+ "mean_token_accuracy": 0.7719444134272635,
1890
+ "num_tokens": 363907270.0,
1891
+ "step": 2080
1892
+ },
1893
+ {
1894
+ "epoch": 0.5695889897630687,
1895
+ "grad_norm": 0.931222677230835,
1896
+ "learning_rate": 3.688017980053379e-05,
1897
+ "loss": 0.8834,
1898
+ "mean_token_accuracy": 0.772805467247963,
1899
+ "num_tokens": 365602883.0,
1900
+ "step": 2090
1901
+ },
1902
+ {
1903
+ "epoch": 0.5723142959341838,
1904
+ "grad_norm": 0.9044320583343506,
1905
+ "learning_rate": 3.680994521702487e-05,
1906
+ "loss": 0.9081,
1907
+ "mean_token_accuracy": 0.7687753444537521,
1908
+ "num_tokens": 367308921.0,
1909
+ "step": 2100
1910
+ },
1911
+ {
1912
+ "epoch": 0.575039602105299,
1913
+ "grad_norm": 0.8750004768371582,
1914
+ "learning_rate": 3.673971063351594e-05,
1915
+ "loss": 0.909,
1916
+ "mean_token_accuracy": 0.7677365321666002,
1917
+ "num_tokens": 369068618.0,
1918
+ "step": 2110
1919
+ },
1920
+ {
1921
+ "epoch": 0.5777649082764141,
1922
+ "grad_norm": 0.836656391620636,
1923
+ "learning_rate": 3.666947605000702e-05,
1924
+ "loss": 0.9237,
1925
+ "mean_token_accuracy": 0.7663044361397624,
1926
+ "num_tokens": 370792272.0,
1927
+ "step": 2120
1928
+ },
1929
+ {
1930
+ "epoch": 0.5804902144475294,
1931
+ "grad_norm": 0.9042032361030579,
1932
+ "learning_rate": 3.659924146649811e-05,
1933
+ "loss": 0.9282,
1934
+ "mean_token_accuracy": 0.7646197730675339,
1935
+ "num_tokens": 372506541.0,
1936
+ "step": 2130
1937
+ },
1938
+ {
1939
+ "epoch": 0.5832155206186445,
1940
+ "grad_norm": 0.8443713784217834,
1941
+ "learning_rate": 3.652900688298919e-05,
1942
+ "loss": 0.8791,
1943
+ "mean_token_accuracy": 0.7750100614503026,
1944
+ "num_tokens": 374308542.0,
1945
+ "step": 2140
1946
+ },
1947
+ {
1948
+ "epoch": 0.5859408267897597,
1949
+ "grad_norm": 0.840281069278717,
1950
+ "learning_rate": 3.645877229948026e-05,
1951
+ "loss": 0.8876,
1952
+ "mean_token_accuracy": 0.7726960835978389,
1953
+ "num_tokens": 376016266.0,
1954
+ "step": 2150
1955
+ },
1956
+ {
1957
+ "epoch": 0.5886661329608748,
1958
+ "grad_norm": 0.80794757604599,
1959
+ "learning_rate": 3.638853771597134e-05,
1960
+ "loss": 0.9308,
1961
+ "mean_token_accuracy": 0.7654405352659523,
1962
+ "num_tokens": 377732713.0,
1963
+ "step": 2160
1964
+ },
1965
+ {
1966
+ "epoch": 0.59139143913199,
1967
+ "grad_norm": 0.9291797280311584,
1968
+ "learning_rate": 3.631830313246243e-05,
1969
+ "loss": 0.9172,
1970
+ "mean_token_accuracy": 0.76725367102772,
1971
+ "num_tokens": 379457927.0,
1972
+ "step": 2170
1973
+ },
1974
+ {
1975
+ "epoch": 0.5941167453031051,
1976
+ "grad_norm": 0.7284071445465088,
1977
+ "learning_rate": 3.6248068548953503e-05,
1978
+ "loss": 0.9533,
1979
+ "mean_token_accuracy": 0.7596265442669392,
1980
+ "num_tokens": 381311531.0,
1981
+ "step": 2180
1982
+ },
1983
+ {
1984
+ "epoch": 0.5968420514742203,
1985
+ "grad_norm": 0.8543251156806946,
1986
+ "learning_rate": 3.6177833965444584e-05,
1987
+ "loss": 0.9449,
1988
+ "mean_token_accuracy": 0.7630881817080081,
1989
+ "num_tokens": 383076036.0,
1990
+ "step": 2190
1991
+ },
1992
+ {
1993
+ "epoch": 0.5995673576453354,
1994
+ "grad_norm": 0.8819230198860168,
1995
+ "learning_rate": 3.610759938193567e-05,
1996
+ "loss": 0.9396,
1997
+ "mean_token_accuracy": 0.7628208780661225,
1998
+ "num_tokens": 384869981.0,
1999
+ "step": 2200
2000
+ },
2001
+ {
2002
+ "epoch": 0.6022926638164506,
2003
+ "grad_norm": 0.9766581058502197,
2004
+ "learning_rate": 3.603736479842675e-05,
2005
+ "loss": 0.9203,
2006
+ "mean_token_accuracy": 0.7657232660800218,
2007
+ "num_tokens": 386654361.0,
2008
+ "step": 2210
2009
+ },
2010
+ {
2011
+ "epoch": 0.6050179699875657,
2012
+ "grad_norm": 0.8702138662338257,
2013
+ "learning_rate": 3.5967130214917824e-05,
2014
+ "loss": 0.8969,
2015
+ "mean_token_accuracy": 0.7711658507585526,
2016
+ "num_tokens": 388405388.0,
2017
+ "step": 2220
2018
+ },
2019
+ {
2020
+ "epoch": 0.607743276158681,
2021
+ "grad_norm": 0.8681634068489075,
2022
+ "learning_rate": 3.5896895631408904e-05,
2023
+ "loss": 0.928,
2024
+ "mean_token_accuracy": 0.7651922093704343,
2025
+ "num_tokens": 390160411.0,
2026
+ "step": 2230
2027
+ },
2028
+ {
2029
+ "epoch": 0.6104685823297961,
2030
+ "grad_norm": 0.8256725668907166,
2031
+ "learning_rate": 3.582666104789999e-05,
2032
+ "loss": 0.9334,
2033
+ "mean_token_accuracy": 0.7633168527856469,
2034
+ "num_tokens": 391963393.0,
2035
+ "step": 2240
2036
+ },
2037
+ {
2038
+ "epoch": 0.6131938885009113,
2039
+ "grad_norm": 0.8524363040924072,
2040
+ "learning_rate": 3.575642646439107e-05,
2041
+ "loss": 0.9321,
2042
+ "mean_token_accuracy": 0.7641467567533254,
2043
+ "num_tokens": 393719494.0,
2044
+ "step": 2250
2045
+ },
2046
+ {
2047
+ "epoch": 0.6159191946720264,
2048
+ "grad_norm": 0.8173193335533142,
2049
+ "learning_rate": 3.5686191880882145e-05,
2050
+ "loss": 0.93,
2051
+ "mean_token_accuracy": 0.7653383906930685,
2052
+ "num_tokens": 395419867.0,
2053
+ "step": 2260
2054
+ },
2055
+ {
2056
+ "epoch": 0.6186445008431416,
2057
+ "grad_norm": 0.7731173634529114,
2058
+ "learning_rate": 3.5615957297373225e-05,
2059
+ "loss": 0.9061,
2060
+ "mean_token_accuracy": 0.7694511445239186,
2061
+ "num_tokens": 397164636.0,
2062
+ "step": 2270
2063
+ },
2064
+ {
2065
+ "epoch": 0.6213698070142567,
2066
+ "grad_norm": 0.9051828980445862,
2067
+ "learning_rate": 3.554572271386431e-05,
2068
+ "loss": 0.9409,
2069
+ "mean_token_accuracy": 0.761752724274993,
2070
+ "num_tokens": 398897880.0,
2071
+ "step": 2280
2072
+ },
2073
+ {
2074
+ "epoch": 0.6240951131853719,
2075
+ "grad_norm": 0.9036754965782166,
2076
+ "learning_rate": 3.5475488130355386e-05,
2077
+ "loss": 0.9232,
2078
+ "mean_token_accuracy": 0.7656918304041028,
2079
+ "num_tokens": 400668616.0,
2080
+ "step": 2290
2081
+ },
2082
+ {
2083
+ "epoch": 0.6268204193564871,
2084
+ "grad_norm": 0.8142061233520508,
2085
+ "learning_rate": 3.5405253546846466e-05,
2086
+ "loss": 0.915,
2087
+ "mean_token_accuracy": 0.7672401297837496,
2088
+ "num_tokens": 402379710.0,
2089
+ "step": 2300
2090
+ },
2091
+ {
2092
+ "epoch": 0.6295457255276022,
2093
+ "grad_norm": 0.8594102263450623,
2094
+ "learning_rate": 3.533501896333755e-05,
2095
+ "loss": 0.9119,
2096
+ "mean_token_accuracy": 0.7676404159516096,
2097
+ "num_tokens": 404084345.0,
2098
+ "step": 2310
2099
+ },
2100
+ {
2101
+ "epoch": 0.6322710316987175,
2102
+ "grad_norm": 0.8828473091125488,
2103
+ "learning_rate": 3.526478437982863e-05,
2104
+ "loss": 0.8836,
2105
+ "mean_token_accuracy": 0.7746170441620052,
2106
+ "num_tokens": 405770321.0,
2107
+ "step": 2320
2108
+ },
2109
+ {
2110
+ "epoch": 0.6349963378698326,
2111
+ "grad_norm": 0.8342154026031494,
2112
+ "learning_rate": 3.5194549796319706e-05,
2113
+ "loss": 0.9064,
2114
+ "mean_token_accuracy": 0.7688289485871792,
2115
+ "num_tokens": 407492020.0,
2116
+ "step": 2330
2117
+ },
2118
+ {
2119
+ "epoch": 0.6377216440409478,
2120
+ "grad_norm": 0.8321424722671509,
2121
+ "learning_rate": 3.5124315212810787e-05,
2122
+ "loss": 0.9023,
2123
+ "mean_token_accuracy": 0.7712408562190831,
2124
+ "num_tokens": 409257115.0,
2125
+ "step": 2340
2126
+ },
2127
+ {
2128
+ "epoch": 0.6404469502120629,
2129
+ "grad_norm": 0.8814114332199097,
2130
+ "learning_rate": 3.5054080629301874e-05,
2131
+ "loss": 0.9279,
2132
+ "mean_token_accuracy": 0.7646240370348096,
2133
+ "num_tokens": 410986071.0,
2134
+ "step": 2350
2135
+ },
2136
+ {
2137
+ "epoch": 0.6431722563831781,
2138
+ "grad_norm": 0.8669275045394897,
2139
+ "learning_rate": 3.4983846045792954e-05,
2140
+ "loss": 0.9115,
2141
+ "mean_token_accuracy": 0.7685579599812626,
2142
+ "num_tokens": 412763897.0,
2143
+ "step": 2360
2144
+ },
2145
+ {
2146
+ "epoch": 0.6458975625542932,
2147
+ "grad_norm": 0.8246127367019653,
2148
+ "learning_rate": 3.491361146228403e-05,
2149
+ "loss": 0.8796,
2150
+ "mean_token_accuracy": 0.7756225952878595,
2151
+ "num_tokens": 414503168.0,
2152
+ "step": 2370
2153
+ },
2154
+ {
2155
+ "epoch": 0.6486228687254084,
2156
+ "grad_norm": 0.811538577079773,
2157
+ "learning_rate": 3.484337687877511e-05,
2158
+ "loss": 0.9046,
2159
+ "mean_token_accuracy": 0.7706884844228625,
2160
+ "num_tokens": 416227145.0,
2161
+ "step": 2380
2162
+ },
2163
+ {
2164
+ "epoch": 0.6513481748965235,
2165
+ "grad_norm": 0.8389571309089661,
2166
+ "learning_rate": 3.4773142295266194e-05,
2167
+ "loss": 0.9023,
2168
+ "mean_token_accuracy": 0.7694288929924369,
2169
+ "num_tokens": 417973040.0,
2170
+ "step": 2390
2171
+ },
2172
+ {
2173
+ "epoch": 0.6540734810676387,
2174
+ "grad_norm": 0.9339456558227539,
2175
+ "learning_rate": 3.470290771175727e-05,
2176
+ "loss": 0.9104,
2177
+ "mean_token_accuracy": 0.7684263320639729,
2178
+ "num_tokens": 419736780.0,
2179
+ "step": 2400
2180
+ },
2181
+ {
2182
+ "epoch": 0.6567987872387538,
2183
+ "grad_norm": 0.7914407253265381,
2184
+ "learning_rate": 3.463267312824835e-05,
2185
+ "loss": 0.9007,
2186
+ "mean_token_accuracy": 0.7712036456912756,
2187
+ "num_tokens": 421516023.0,
2188
+ "step": 2410
2189
+ },
2190
+ {
2191
+ "epoch": 0.6595240934098691,
2192
+ "grad_norm": 0.8769999146461487,
2193
+ "learning_rate": 3.4562438544739435e-05,
2194
+ "loss": 0.9433,
2195
+ "mean_token_accuracy": 0.7634194398298859,
2196
+ "num_tokens": 423324296.0,
2197
+ "step": 2420
2198
+ },
2199
+ {
2200
+ "epoch": 0.6622493995809842,
2201
+ "grad_norm": 0.9201934337615967,
2202
+ "learning_rate": 3.4492203961230515e-05,
2203
+ "loss": 0.9272,
2204
+ "mean_token_accuracy": 0.7628090158104897,
2205
+ "num_tokens": 425114987.0,
2206
+ "step": 2430
2207
+ },
2208
+ {
2209
+ "epoch": 0.6649747057520994,
2210
+ "grad_norm": 0.833123505115509,
2211
+ "learning_rate": 3.442196937772159e-05,
2212
+ "loss": 0.8962,
2213
+ "mean_token_accuracy": 0.7704461555927992,
2214
+ "num_tokens": 426875751.0,
2215
+ "step": 2440
2216
+ },
2217
+ {
2218
+ "epoch": 0.6677000119232145,
2219
+ "grad_norm": 0.84730464220047,
2220
+ "learning_rate": 3.435173479421267e-05,
2221
+ "loss": 0.9062,
2222
+ "mean_token_accuracy": 0.7695771548897028,
2223
+ "num_tokens": 428715433.0,
2224
+ "step": 2450
2225
+ },
2226
+ {
2227
+ "epoch": 0.6704253180943297,
2228
+ "grad_norm": 0.9411203265190125,
2229
+ "learning_rate": 3.4281500210703756e-05,
2230
+ "loss": 0.9435,
2231
+ "mean_token_accuracy": 0.761421543173492,
2232
+ "num_tokens": 430443301.0,
2233
+ "step": 2460
2234
+ },
2235
+ {
2236
+ "epoch": 0.6731506242654448,
2237
+ "grad_norm": 0.8246670365333557,
2238
+ "learning_rate": 3.4211265627194836e-05,
2239
+ "loss": 0.8701,
2240
+ "mean_token_accuracy": 0.7766519106924534,
2241
+ "num_tokens": 432149898.0,
2242
+ "step": 2470
2243
+ },
2244
+ {
2245
+ "epoch": 0.67587593043656,
2246
+ "grad_norm": 0.8873474597930908,
2247
+ "learning_rate": 3.414103104368591e-05,
2248
+ "loss": 0.8906,
2249
+ "mean_token_accuracy": 0.7729644425213337,
2250
+ "num_tokens": 433904775.0,
2251
+ "step": 2480
2252
+ },
2253
+ {
2254
+ "epoch": 0.6786012366076751,
2255
+ "grad_norm": 0.8719823956489563,
2256
+ "learning_rate": 3.407079646017699e-05,
2257
+ "loss": 0.8915,
2258
+ "mean_token_accuracy": 0.7718378273770213,
2259
+ "num_tokens": 435648944.0,
2260
+ "step": 2490
2261
+ },
2262
+ {
2263
+ "epoch": 0.6813265427787903,
2264
+ "grad_norm": 0.8626745343208313,
2265
+ "learning_rate": 3.4000561876668076e-05,
2266
+ "loss": 0.9286,
2267
+ "mean_token_accuracy": 0.7654551040381193,
2268
+ "num_tokens": 437390122.0,
2269
+ "step": 2500
2270
+ },
2271
+ {
2272
+ "epoch": 0.6840518489499054,
2273
+ "grad_norm": 0.8651098608970642,
2274
+ "learning_rate": 3.393032729315915e-05,
2275
+ "loss": 0.9176,
2276
+ "mean_token_accuracy": 0.767458095960319,
2277
+ "num_tokens": 439126311.0,
2278
+ "step": 2510
2279
+ },
2280
+ {
2281
+ "epoch": 0.6867771551210207,
2282
+ "grad_norm": 0.82646644115448,
2283
+ "learning_rate": 3.386009270965023e-05,
2284
+ "loss": 0.9056,
2285
+ "mean_token_accuracy": 0.7701940412633121,
2286
+ "num_tokens": 440890933.0,
2287
+ "step": 2520
2288
+ },
2289
+ {
2290
+ "epoch": 0.6895024612921358,
2291
+ "grad_norm": 0.9855936169624329,
2292
+ "learning_rate": 3.378985812614132e-05,
2293
+ "loss": 0.8948,
2294
+ "mean_token_accuracy": 0.7717040576040745,
2295
+ "num_tokens": 442611770.0,
2296
+ "step": 2530
2297
+ },
2298
+ {
2299
+ "epoch": 0.692227767463251,
2300
+ "grad_norm": 0.8282904624938965,
2301
+ "learning_rate": 3.37196235426324e-05,
2302
+ "loss": 0.8981,
2303
+ "mean_token_accuracy": 0.7711870735511184,
2304
+ "num_tokens": 444358166.0,
2305
+ "step": 2540
2306
+ },
2307
+ {
2308
+ "epoch": 0.6949530736343661,
2309
+ "grad_norm": 0.8497506380081177,
2310
+ "learning_rate": 3.364938895912347e-05,
2311
+ "loss": 0.9359,
2312
+ "mean_token_accuracy": 0.7633687354624271,
2313
+ "num_tokens": 446078790.0,
2314
+ "step": 2550
2315
+ },
2316
+ {
2317
+ "epoch": 0.6976783798054813,
2318
+ "grad_norm": 0.9366584420204163,
2319
+ "learning_rate": 3.357915437561455e-05,
2320
+ "loss": 0.9024,
2321
+ "mean_token_accuracy": 0.769352519325912,
2322
+ "num_tokens": 447811186.0,
2323
+ "step": 2560
2324
+ },
2325
+ {
2326
+ "epoch": 0.7004036859765964,
2327
+ "grad_norm": 0.8402838110923767,
2328
+ "learning_rate": 3.350891979210564e-05,
2329
+ "loss": 0.9157,
2330
+ "mean_token_accuracy": 0.7674468743614853,
2331
+ "num_tokens": 449537962.0,
2332
+ "step": 2570
2333
+ },
2334
+ {
2335
+ "epoch": 0.7031289921477116,
2336
+ "grad_norm": 0.8306329250335693,
2337
+ "learning_rate": 3.343868520859672e-05,
2338
+ "loss": 0.8868,
2339
+ "mean_token_accuracy": 0.7735649706795812,
2340
+ "num_tokens": 451259469.0,
2341
+ "step": 2580
2342
+ },
2343
+ {
2344
+ "epoch": 0.7058542983188267,
2345
+ "grad_norm": 0.8826439380645752,
2346
+ "learning_rate": 3.336845062508779e-05,
2347
+ "loss": 0.8963,
2348
+ "mean_token_accuracy": 0.7722868470475077,
2349
+ "num_tokens": 453041146.0,
2350
+ "step": 2590
2351
+ },
2352
+ {
2353
+ "epoch": 0.708579604489942,
2354
+ "grad_norm": 0.8931313157081604,
2355
+ "learning_rate": 3.329821604157887e-05,
2356
+ "loss": 0.9309,
2357
+ "mean_token_accuracy": 0.7645103015005589,
2358
+ "num_tokens": 454753651.0,
2359
+ "step": 2600
2360
+ },
2361
+ {
2362
+ "epoch": 0.711304910661057,
2363
+ "grad_norm": 0.8628774285316467,
2364
+ "learning_rate": 3.322798145806996e-05,
2365
+ "loss": 0.8983,
2366
+ "mean_token_accuracy": 0.7708307450637222,
2367
+ "num_tokens": 456537057.0,
2368
+ "step": 2610
2369
+ },
2370
+ {
2371
+ "epoch": 0.7140302168321723,
2372
+ "grad_norm": 0.8462346196174622,
2373
+ "learning_rate": 3.315774687456103e-05,
2374
+ "loss": 0.9424,
2375
+ "mean_token_accuracy": 0.7633946686983109,
2376
+ "num_tokens": 458271296.0,
2377
+ "step": 2620
2378
+ },
2379
+ {
2380
+ "epoch": 0.7167555230032874,
2381
+ "grad_norm": 0.7972195744514465,
2382
+ "learning_rate": 3.308751229105211e-05,
2383
+ "loss": 0.9151,
2384
+ "mean_token_accuracy": 0.7673410098999739,
2385
+ "num_tokens": 460014932.0,
2386
+ "step": 2630
2387
+ },
2388
+ {
2389
+ "epoch": 0.7194808291744026,
2390
+ "grad_norm": 0.912936270236969,
2391
+ "learning_rate": 3.30172777075432e-05,
2392
+ "loss": 0.9266,
2393
+ "mean_token_accuracy": 0.7656251262873411,
2394
+ "num_tokens": 461818480.0,
2395
+ "step": 2640
2396
+ },
2397
+ {
2398
+ "epoch": 0.7222061353455177,
2399
+ "grad_norm": 0.9048453569412231,
2400
+ "learning_rate": 3.294704312403428e-05,
2401
+ "loss": 0.9464,
2402
+ "mean_token_accuracy": 0.7605526108294726,
2403
+ "num_tokens": 463519993.0,
2404
+ "step": 2650
2405
+ },
2406
+ {
2407
+ "epoch": 0.7249314415166329,
2408
+ "grad_norm": 0.947562038898468,
2409
+ "learning_rate": 3.287680854052535e-05,
2410
+ "loss": 0.9135,
2411
+ "mean_token_accuracy": 0.7671187071129679,
2412
+ "num_tokens": 465333101.0,
2413
+ "step": 2660
2414
+ },
2415
+ {
2416
+ "epoch": 0.727656747687748,
2417
+ "grad_norm": 0.8569560050964355,
2418
+ "learning_rate": 3.280657395701643e-05,
2419
+ "loss": 0.9084,
2420
+ "mean_token_accuracy": 0.76853600833565,
2421
+ "num_tokens": 467047198.0,
2422
+ "step": 2670
2423
+ },
2424
+ {
2425
+ "epoch": 0.7303820538588632,
2426
+ "grad_norm": 0.8648985624313354,
2427
+ "learning_rate": 3.273633937350752e-05,
2428
+ "loss": 0.9213,
2429
+ "mean_token_accuracy": 0.766769905667752,
2430
+ "num_tokens": 468806711.0,
2431
+ "step": 2680
2432
+ },
2433
+ {
2434
+ "epoch": 0.7331073600299783,
2435
+ "grad_norm": 0.8183128237724304,
2436
+ "learning_rate": 3.26661047899986e-05,
2437
+ "loss": 0.8789,
2438
+ "mean_token_accuracy": 0.7748562408611178,
2439
+ "num_tokens": 470600782.0,
2440
+ "step": 2690
2441
+ },
2442
+ {
2443
+ "epoch": 0.7358326662010936,
2444
+ "grad_norm": 0.8345265984535217,
2445
+ "learning_rate": 3.2595870206489674e-05,
2446
+ "loss": 0.9198,
2447
+ "mean_token_accuracy": 0.767015983723104,
2448
+ "num_tokens": 472349330.0,
2449
+ "step": 2700
2450
+ },
2451
+ {
2452
+ "epoch": 0.7385579723722087,
2453
+ "grad_norm": 0.836322009563446,
2454
+ "learning_rate": 3.2525635622980754e-05,
2455
+ "loss": 0.907,
2456
+ "mean_token_accuracy": 0.7705251289531588,
2457
+ "num_tokens": 474106983.0,
2458
+ "step": 2710
2459
+ },
2460
+ {
2461
+ "epoch": 0.7412832785433239,
2462
+ "grad_norm": 0.8931109309196472,
2463
+ "learning_rate": 3.245540103947184e-05,
2464
+ "loss": 0.939,
2465
+ "mean_token_accuracy": 0.7626953650265932,
2466
+ "num_tokens": 475902407.0,
2467
+ "step": 2720
2468
+ },
2469
+ {
2470
+ "epoch": 0.744008584714439,
2471
+ "grad_norm": 0.8067692518234253,
2472
+ "learning_rate": 3.2385166455962914e-05,
2473
+ "loss": 0.8629,
2474
+ "mean_token_accuracy": 0.7779960019513965,
2475
+ "num_tokens": 477654644.0,
2476
+ "step": 2730
2477
+ },
2478
+ {
2479
+ "epoch": 0.7467338908855542,
2480
+ "grad_norm": 0.8653134107589722,
2481
+ "learning_rate": 3.2314931872453994e-05,
2482
+ "loss": 0.9292,
2483
+ "mean_token_accuracy": 0.764538905583322,
2484
+ "num_tokens": 479409078.0,
2485
+ "step": 2740
2486
+ },
2487
+ {
2488
+ "epoch": 0.7494591970566693,
2489
+ "grad_norm": 0.9657193422317505,
2490
+ "learning_rate": 3.224469728894508e-05,
2491
+ "loss": 0.9136,
2492
+ "mean_token_accuracy": 0.767500625923276,
2493
+ "num_tokens": 481154972.0,
2494
+ "step": 2750
2495
+ },
2496
+ {
2497
+ "epoch": 0.7521845032277845,
2498
+ "grad_norm": 0.9113588929176331,
2499
+ "learning_rate": 3.217446270543616e-05,
2500
+ "loss": 0.9395,
2501
+ "mean_token_accuracy": 0.7628454895690083,
2502
+ "num_tokens": 482957554.0,
2503
+ "step": 2760
2504
+ },
2505
+ {
2506
+ "epoch": 0.7549098093988996,
2507
+ "grad_norm": 0.8665691614151001,
2508
+ "learning_rate": 3.2104228121927235e-05,
2509
+ "loss": 0.9139,
2510
+ "mean_token_accuracy": 0.7677292965352536,
2511
+ "num_tokens": 484730289.0,
2512
+ "step": 2770
2513
+ },
2514
+ {
2515
+ "epoch": 0.7576351155700148,
2516
+ "grad_norm": 0.9618145227432251,
2517
+ "learning_rate": 3.2033993538418315e-05,
2518
+ "loss": 0.9206,
2519
+ "mean_token_accuracy": 0.7647297469899058,
2520
+ "num_tokens": 486390658.0,
2521
+ "step": 2780
2522
+ },
2523
+ {
2524
+ "epoch": 0.7603604217411299,
2525
+ "grad_norm": 0.9165218472480774,
2526
+ "learning_rate": 3.19637589549094e-05,
2527
+ "loss": 0.8822,
2528
+ "mean_token_accuracy": 0.7746361203491687,
2529
+ "num_tokens": 488084357.0,
2530
+ "step": 2790
2531
+ },
2532
+ {
2533
+ "epoch": 0.7630857279122452,
2534
+ "grad_norm": 0.9210852980613708,
2535
+ "learning_rate": 3.189352437140048e-05,
2536
+ "loss": 0.9589,
2537
+ "mean_token_accuracy": 0.7591536954045296,
2538
+ "num_tokens": 489867504.0,
2539
+ "step": 2800
2540
+ },
2541
+ {
2542
+ "epoch": 0.7658110340833603,
2543
+ "grad_norm": 0.8209601044654846,
2544
+ "learning_rate": 3.1823289787891556e-05,
2545
+ "loss": 0.926,
2546
+ "mean_token_accuracy": 0.7646390557289123,
2547
+ "num_tokens": 491614120.0,
2548
+ "step": 2810
2549
+ },
2550
+ {
2551
+ "epoch": 0.7685363402544755,
2552
+ "grad_norm": 0.8096806406974792,
2553
+ "learning_rate": 3.1753055204382636e-05,
2554
+ "loss": 0.9237,
2555
+ "mean_token_accuracy": 0.7666580315679312,
2556
+ "num_tokens": 493360108.0,
2557
+ "step": 2820
2558
+ },
2559
+ {
2560
+ "epoch": 0.7712616464255906,
2561
+ "grad_norm": 0.8601098656654358,
2562
+ "learning_rate": 3.168282062087372e-05,
2563
+ "loss": 0.9003,
2564
+ "mean_token_accuracy": 0.7720298429951071,
2565
+ "num_tokens": 495060867.0,
2566
+ "step": 2830
2567
+ },
2568
+ {
2569
+ "epoch": 0.7739869525967058,
2570
+ "grad_norm": 0.896539568901062,
2571
+ "learning_rate": 3.16125860373648e-05,
2572
+ "loss": 0.9224,
2573
+ "mean_token_accuracy": 0.7646079597994685,
2574
+ "num_tokens": 496731129.0,
2575
+ "step": 2840
2576
+ },
2577
+ {
2578
+ "epoch": 0.7767122587678209,
2579
+ "grad_norm": 0.7847779393196106,
2580
+ "learning_rate": 3.1542351453855877e-05,
2581
+ "loss": 0.8955,
2582
+ "mean_token_accuracy": 0.7723177457228303,
2583
+ "num_tokens": 498416929.0,
2584
+ "step": 2850
2585
+ },
2586
+ {
2587
+ "epoch": 0.7794375649389361,
2588
+ "grad_norm": 0.9052691459655762,
2589
+ "learning_rate": 3.1472116870346964e-05,
2590
+ "loss": 0.8798,
2591
+ "mean_token_accuracy": 0.7750480374321341,
2592
+ "num_tokens": 500131030.0,
2593
+ "step": 2860
2594
+ },
2595
+ {
2596
+ "epoch": 0.7821628711100512,
2597
+ "grad_norm": 0.7930359244346619,
2598
+ "learning_rate": 3.1401882286838044e-05,
2599
+ "loss": 0.8905,
2600
+ "mean_token_accuracy": 0.7722199564799667,
2601
+ "num_tokens": 501860254.0,
2602
+ "step": 2870
2603
+ },
2604
+ {
2605
+ "epoch": 0.7848881772811664,
2606
+ "grad_norm": 0.89003986120224,
2607
+ "learning_rate": 3.133164770332912e-05,
2608
+ "loss": 0.9411,
2609
+ "mean_token_accuracy": 0.7619166776537896,
2610
+ "num_tokens": 503599791.0,
2611
+ "step": 2880
2612
+ },
2613
+ {
2614
+ "epoch": 0.7876134834522815,
2615
+ "grad_norm": 0.8397514224052429,
2616
+ "learning_rate": 3.12614131198202e-05,
2617
+ "loss": 0.96,
2618
+ "mean_token_accuracy": 0.7589967941865325,
2619
+ "num_tokens": 505277304.0,
2620
+ "step": 2890
2621
+ },
2622
+ {
2623
+ "epoch": 0.7903387896233968,
2624
+ "grad_norm": 0.9073812365531921,
2625
+ "learning_rate": 3.1191178536311284e-05,
2626
+ "loss": 0.9119,
2627
+ "mean_token_accuracy": 0.767801726795733,
2628
+ "num_tokens": 507010506.0,
2629
+ "step": 2900
2630
+ },
2631
+ {
2632
+ "epoch": 0.7930640957945119,
2633
+ "grad_norm": 0.8646041750907898,
2634
+ "learning_rate": 3.1120943952802364e-05,
2635
+ "loss": 0.8999,
2636
+ "mean_token_accuracy": 0.7706972565501928,
2637
+ "num_tokens": 508717678.0,
2638
+ "step": 2910
2639
+ },
2640
+ {
2641
+ "epoch": 0.7957894019656271,
2642
+ "grad_norm": 0.8266422748565674,
2643
+ "learning_rate": 3.105070936929344e-05,
2644
+ "loss": 0.878,
2645
+ "mean_token_accuracy": 0.7762510352768004,
2646
+ "num_tokens": 510478982.0,
2647
+ "step": 2920
2648
+ },
2649
+ {
2650
+ "epoch": 0.7985147081367422,
2651
+ "grad_norm": 0.8507207036018372,
2652
+ "learning_rate": 3.0980474785784525e-05,
2653
+ "loss": 0.8742,
2654
+ "mean_token_accuracy": 0.774853196553886,
2655
+ "num_tokens": 512223492.0,
2656
+ "step": 2930
2657
+ },
2658
+ {
2659
+ "epoch": 0.8012400143078574,
2660
+ "grad_norm": 0.8072352409362793,
2661
+ "learning_rate": 3.0910240202275605e-05,
2662
+ "loss": 0.8996,
2663
+ "mean_token_accuracy": 0.770034134015441,
2664
+ "num_tokens": 513950826.0,
2665
+ "step": 2940
2666
+ },
2667
+ {
2668
+ "epoch": 0.8039653204789725,
2669
+ "grad_norm": 0.88204026222229,
2670
+ "learning_rate": 3.0840005618766685e-05,
2671
+ "loss": 0.9293,
2672
+ "mean_token_accuracy": 0.7653471390716732,
2673
+ "num_tokens": 515747512.0,
2674
+ "step": 2950
2675
+ },
2676
+ {
2677
+ "epoch": 0.8066906266500877,
2678
+ "grad_norm": 0.8134139180183411,
2679
+ "learning_rate": 3.076977103525776e-05,
2680
+ "loss": 0.8995,
2681
+ "mean_token_accuracy": 0.7705760426819325,
2682
+ "num_tokens": 517564408.0,
2683
+ "step": 2960
2684
+ },
2685
+ {
2686
+ "epoch": 0.8094159328212028,
2687
+ "grad_norm": 0.810004711151123,
2688
+ "learning_rate": 3.0699536451748846e-05,
2689
+ "loss": 0.8433,
2690
+ "mean_token_accuracy": 0.7816835820674897,
2691
+ "num_tokens": 519335113.0,
2692
+ "step": 2970
2693
+ },
2694
+ {
2695
+ "epoch": 0.812141238992318,
2696
+ "grad_norm": 0.8259313106536865,
2697
+ "learning_rate": 3.0629301868239926e-05,
2698
+ "loss": 0.8837,
2699
+ "mean_token_accuracy": 0.7741564935073256,
2700
+ "num_tokens": 521052694.0,
2701
+ "step": 2980
2702
+ },
2703
+ {
2704
+ "epoch": 0.8148665451634332,
2705
+ "grad_norm": 0.8008886575698853,
2706
+ "learning_rate": 3.0559067284731e-05,
2707
+ "loss": 0.9214,
2708
+ "mean_token_accuracy": 0.7659341244027018,
2709
+ "num_tokens": 522838886.0,
2710
+ "step": 2990
2711
+ },
2712
+ {
2713
+ "epoch": 0.8175918513345484,
2714
+ "grad_norm": 0.7934632897377014,
2715
+ "learning_rate": 3.048883270122208e-05,
2716
+ "loss": 0.9041,
2717
+ "mean_token_accuracy": 0.7696094507351517,
2718
+ "num_tokens": 524659688.0,
2719
+ "step": 3000
2720
+ },
2721
+ {
2722
+ "epoch": 0.8203171575056635,
2723
+ "grad_norm": 0.839688241481781,
2724
+ "learning_rate": 3.0418598117713166e-05,
2725
+ "loss": 0.8899,
2726
+ "mean_token_accuracy": 0.7726705798879265,
2727
+ "num_tokens": 526428828.0,
2728
+ "step": 3010
2729
+ },
2730
+ {
2731
+ "epoch": 0.8230424636767787,
2732
+ "grad_norm": 0.8166800737380981,
2733
+ "learning_rate": 3.0348363534204243e-05,
2734
+ "loss": 0.9119,
2735
+ "mean_token_accuracy": 0.7672878581099212,
2736
+ "num_tokens": 528178357.0,
2737
+ "step": 3020
2738
+ },
2739
+ {
2740
+ "epoch": 0.8257677698478938,
2741
+ "grad_norm": 0.8090632557868958,
2742
+ "learning_rate": 3.0278128950695323e-05,
2743
+ "loss": 0.8927,
2744
+ "mean_token_accuracy": 0.7714125035330653,
2745
+ "num_tokens": 529968661.0,
2746
+ "step": 3030
2747
+ },
2748
+ {
2749
+ "epoch": 0.828493076019009,
2750
+ "grad_norm": 0.9141647219657898,
2751
+ "learning_rate": 3.0207894367186407e-05,
2752
+ "loss": 0.9022,
2753
+ "mean_token_accuracy": 0.7716053297743202,
2754
+ "num_tokens": 531735846.0,
2755
+ "step": 3040
2756
+ },
2757
+ {
2758
+ "epoch": 0.8312183821901241,
2759
+ "grad_norm": 0.8853212594985962,
2760
+ "learning_rate": 3.0137659783677484e-05,
2761
+ "loss": 0.923,
2762
+ "mean_token_accuracy": 0.7664798295125366,
2763
+ "num_tokens": 533482605.0,
2764
+ "step": 3050
2765
+ },
2766
+ {
2767
+ "epoch": 0.8339436883612393,
2768
+ "grad_norm": 0.8621941804885864,
2769
+ "learning_rate": 3.0067425200168564e-05,
2770
+ "loss": 0.9064,
2771
+ "mean_token_accuracy": 0.7699273196980357,
2772
+ "num_tokens": 535172117.0,
2773
+ "step": 3060
2774
+ },
2775
+ {
2776
+ "epoch": 0.8366689945323545,
2777
+ "grad_norm": 0.840312123298645,
2778
+ "learning_rate": 2.9997190616659644e-05,
2779
+ "loss": 0.8953,
2780
+ "mean_token_accuracy": 0.7715860839933157,
2781
+ "num_tokens": 536922982.0,
2782
+ "step": 3070
2783
+ },
2784
+ {
2785
+ "epoch": 0.8393943007034697,
2786
+ "grad_norm": 0.863222062587738,
2787
+ "learning_rate": 2.9926956033150728e-05,
2788
+ "loss": 0.906,
2789
+ "mean_token_accuracy": 0.7695185262709856,
2790
+ "num_tokens": 538697561.0,
2791
+ "step": 3080
2792
+ },
2793
+ {
2794
+ "epoch": 0.8421196068745849,
2795
+ "grad_norm": 0.925609290599823,
2796
+ "learning_rate": 2.9856721449641805e-05,
2797
+ "loss": 0.9123,
2798
+ "mean_token_accuracy": 0.7679164627566933,
2799
+ "num_tokens": 540498809.0,
2800
+ "step": 3090
2801
+ },
2802
+ {
2803
+ "epoch": 0.8448449130457,
2804
+ "grad_norm": 0.8675920367240906,
2805
+ "learning_rate": 2.9786486866132885e-05,
2806
+ "loss": 0.882,
2807
+ "mean_token_accuracy": 0.7736795306205749,
2808
+ "num_tokens": 542220573.0,
2809
+ "step": 3100
2810
+ },
2811
+ {
2812
+ "epoch": 0.8475702192168152,
2813
+ "grad_norm": 0.8793986439704895,
2814
+ "learning_rate": 2.971625228262396e-05,
2815
+ "loss": 0.9003,
2816
+ "mean_token_accuracy": 0.770465992949903,
2817
+ "num_tokens": 543995979.0,
2818
+ "step": 3110
2819
+ },
2820
+ {
2821
+ "epoch": 0.8502955253879303,
2822
+ "grad_norm": 0.8559759259223938,
2823
+ "learning_rate": 2.964601769911505e-05,
2824
+ "loss": 0.9159,
2825
+ "mean_token_accuracy": 0.7657374052330852,
2826
+ "num_tokens": 545762046.0,
2827
+ "step": 3120
2828
+ },
2829
+ {
2830
+ "epoch": 0.8530208315590455,
2831
+ "grad_norm": 0.7674428224563599,
2832
+ "learning_rate": 2.9575783115606125e-05,
2833
+ "loss": 0.8943,
2834
+ "mean_token_accuracy": 0.7716390110552311,
2835
+ "num_tokens": 547563120.0,
2836
+ "step": 3130
2837
+ },
2838
+ {
2839
+ "epoch": 0.8557461377301606,
2840
+ "grad_norm": 0.8808310627937317,
2841
+ "learning_rate": 2.9505548532097206e-05,
2842
+ "loss": 0.9121,
2843
+ "mean_token_accuracy": 0.7677836719900369,
2844
+ "num_tokens": 549276605.0,
2845
+ "step": 3140
2846
+ },
2847
+ {
2848
+ "epoch": 0.8584714439012758,
2849
+ "grad_norm": 0.8529791235923767,
2850
+ "learning_rate": 2.943531394858829e-05,
2851
+ "loss": 0.901,
2852
+ "mean_token_accuracy": 0.7700381241738796,
2853
+ "num_tokens": 550991479.0,
2854
+ "step": 3150
2855
+ },
2856
+ {
2857
+ "epoch": 0.8611967500723909,
2858
+ "grad_norm": 0.8383409976959229,
2859
+ "learning_rate": 2.9365079365079366e-05,
2860
+ "loss": 0.8921,
2861
+ "mean_token_accuracy": 0.7733515216037631,
2862
+ "num_tokens": 552777662.0,
2863
+ "step": 3160
2864
+ },
2865
+ {
2866
+ "epoch": 0.8639220562435062,
2867
+ "grad_norm": 0.9514780640602112,
2868
+ "learning_rate": 2.9294844781570446e-05,
2869
+ "loss": 0.8801,
2870
+ "mean_token_accuracy": 0.7743920419365168,
2871
+ "num_tokens": 554539356.0,
2872
+ "step": 3170
2873
+ },
2874
+ {
2875
+ "epoch": 0.8666473624146213,
2876
+ "grad_norm": 0.8511672019958496,
2877
+ "learning_rate": 2.9224610198061526e-05,
2878
+ "loss": 0.9443,
2879
+ "mean_token_accuracy": 0.7616958959959448,
2880
+ "num_tokens": 556212548.0,
2881
+ "step": 3180
2882
+ },
2883
+ {
2884
+ "epoch": 0.8693726685857365,
2885
+ "grad_norm": 0.8855804800987244,
2886
+ "learning_rate": 2.915437561455261e-05,
2887
+ "loss": 0.9129,
2888
+ "mean_token_accuracy": 0.7673460508696734,
2889
+ "num_tokens": 557950461.0,
2890
+ "step": 3190
2891
+ },
2892
+ {
2893
+ "epoch": 0.8720979747568516,
2894
+ "grad_norm": 0.8707140684127808,
2895
+ "learning_rate": 2.9084141031043687e-05,
2896
+ "loss": 0.8903,
2897
+ "mean_token_accuracy": 0.771921418607235,
2898
+ "num_tokens": 559715926.0,
2899
+ "step": 3200
2900
+ },
2901
+ {
2902
+ "epoch": 0.8748232809279668,
2903
+ "grad_norm": 0.9066351652145386,
2904
+ "learning_rate": 2.9013906447534767e-05,
2905
+ "loss": 0.9122,
2906
+ "mean_token_accuracy": 0.7690841609612107,
2907
+ "num_tokens": 561479394.0,
2908
+ "step": 3210
2909
+ },
2910
+ {
2911
+ "epoch": 0.8775485870990819,
2912
+ "grad_norm": 0.8212366104125977,
2913
+ "learning_rate": 2.8943671864025844e-05,
2914
+ "loss": 0.9047,
2915
+ "mean_token_accuracy": 0.7691477432847023,
2916
+ "num_tokens": 563271872.0,
2917
+ "step": 3220
2918
+ },
2919
+ {
2920
+ "epoch": 0.8802738932701971,
2921
+ "grad_norm": 0.8219311833381653,
2922
+ "learning_rate": 2.887343728051693e-05,
2923
+ "loss": 0.8994,
2924
+ "mean_token_accuracy": 0.7715168165042996,
2925
+ "num_tokens": 565002210.0,
2926
+ "step": 3230
2927
+ },
2928
+ {
2929
+ "epoch": 0.8829991994413122,
2930
+ "grad_norm": 0.898818850517273,
2931
+ "learning_rate": 2.8803202697008008e-05,
2932
+ "loss": 0.8221,
2933
+ "mean_token_accuracy": 0.786996117606759,
2934
+ "num_tokens": 566740287.0,
2935
+ "step": 3240
2936
+ },
2937
+ {
2938
+ "epoch": 0.8857245056124274,
2939
+ "grad_norm": 0.7879354953765869,
2940
+ "learning_rate": 2.8732968113499088e-05,
2941
+ "loss": 0.9305,
2942
+ "mean_token_accuracy": 0.7649352782405913,
2943
+ "num_tokens": 568560935.0,
2944
+ "step": 3250
2945
+ },
2946
+ {
2947
+ "epoch": 0.8884498117835425,
2948
+ "grad_norm": 0.9506646394729614,
2949
+ "learning_rate": 2.866273352999017e-05,
2950
+ "loss": 0.9164,
2951
+ "mean_token_accuracy": 0.7674483682960271,
2952
+ "num_tokens": 570290159.0,
2953
+ "step": 3260
2954
+ },
2955
+ {
2956
+ "epoch": 0.8911751179546578,
2957
+ "grad_norm": 0.8495944142341614,
2958
+ "learning_rate": 2.8592498946481248e-05,
2959
+ "loss": 0.9218,
2960
+ "mean_token_accuracy": 0.76632971893996,
2961
+ "num_tokens": 572076704.0,
2962
+ "step": 3270
2963
+ },
2964
+ {
2965
+ "epoch": 0.8939004241257729,
2966
+ "grad_norm": 0.8082846999168396,
2967
+ "learning_rate": 2.852226436297233e-05,
2968
+ "loss": 0.9318,
2969
+ "mean_token_accuracy": 0.76468366086483,
2970
+ "num_tokens": 573841975.0,
2971
+ "step": 3280
2972
+ },
2973
+ {
2974
+ "epoch": 0.8966257302968881,
2975
+ "grad_norm": 0.809038519859314,
2976
+ "learning_rate": 2.845202977946341e-05,
2977
+ "loss": 0.9057,
2978
+ "mean_token_accuracy": 0.7704976342618466,
2979
+ "num_tokens": 575587035.0,
2980
+ "step": 3290
2981
+ },
2982
+ {
2983
+ "epoch": 0.8993510364680032,
2984
+ "grad_norm": 0.8163307309150696,
2985
+ "learning_rate": 2.8381795195954492e-05,
2986
+ "loss": 0.9047,
2987
+ "mean_token_accuracy": 0.769212381914258,
2988
+ "num_tokens": 577285852.0,
2989
+ "step": 3300
2990
+ },
2991
+ {
2992
+ "epoch": 0.9020763426391184,
2993
+ "grad_norm": 0.8320499658584595,
2994
+ "learning_rate": 2.831156061244557e-05,
2995
+ "loss": 0.9208,
2996
+ "mean_token_accuracy": 0.7671002107672393,
2997
+ "num_tokens": 579085053.0,
2998
+ "step": 3310
2999
+ },
3000
+ {
3001
+ "epoch": 0.9048016488102335,
3002
+ "grad_norm": 0.7883128523826599,
3003
+ "learning_rate": 2.824132602893665e-05,
3004
+ "loss": 0.8903,
3005
+ "mean_token_accuracy": 0.7726314144209028,
3006
+ "num_tokens": 580874041.0,
3007
+ "step": 3320
3008
+ },
3009
+ {
3010
+ "epoch": 0.9075269549813487,
3011
+ "grad_norm": 0.8674151301383972,
3012
+ "learning_rate": 2.8171091445427726e-05,
3013
+ "loss": 0.8888,
3014
+ "mean_token_accuracy": 0.7714107790961862,
3015
+ "num_tokens": 582616757.0,
3016
+ "step": 3330
3017
+ },
3018
+ {
3019
+ "epoch": 0.9102522611524638,
3020
+ "grad_norm": 0.8607017397880554,
3021
+ "learning_rate": 2.8100856861918813e-05,
3022
+ "loss": 0.8852,
3023
+ "mean_token_accuracy": 0.7738766206428409,
3024
+ "num_tokens": 584370810.0,
3025
+ "step": 3340
3026
+ },
3027
+ {
3028
+ "epoch": 0.912977567323579,
3029
+ "grad_norm": 0.850727915763855,
3030
+ "learning_rate": 2.803062227840989e-05,
3031
+ "loss": 0.9006,
3032
+ "mean_token_accuracy": 0.7707684559747576,
3033
+ "num_tokens": 586143330.0,
3034
+ "step": 3350
3035
+ },
3036
+ {
3037
+ "epoch": 0.9157028734946941,
3038
+ "grad_norm": 0.8423539400100708,
3039
+ "learning_rate": 2.796038769490097e-05,
3040
+ "loss": 0.8657,
3041
+ "mean_token_accuracy": 0.777682445757091,
3042
+ "num_tokens": 587851884.0,
3043
+ "step": 3360
3044
+ },
3045
+ {
3046
+ "epoch": 0.9184281796658094,
3047
+ "grad_norm": 0.8592097759246826,
3048
+ "learning_rate": 2.7890153111392054e-05,
3049
+ "loss": 0.8937,
3050
+ "mean_token_accuracy": 0.7706205818802119,
3051
+ "num_tokens": 589601267.0,
3052
+ "step": 3370
3053
+ },
3054
+ {
3055
+ "epoch": 0.9211534858369245,
3056
+ "grad_norm": 0.943073034286499,
3057
+ "learning_rate": 2.7819918527883134e-05,
3058
+ "loss": 0.9254,
3059
+ "mean_token_accuracy": 0.7644982000812888,
3060
+ "num_tokens": 591321205.0,
3061
+ "step": 3380
3062
+ },
3063
+ {
3064
+ "epoch": 0.9238787920080397,
3065
+ "grad_norm": 0.7838475704193115,
3066
+ "learning_rate": 2.774968394437421e-05,
3067
+ "loss": 0.8704,
3068
+ "mean_token_accuracy": 0.7773344025947153,
3069
+ "num_tokens": 593110103.0,
3070
+ "step": 3390
3071
+ },
3072
+ {
3073
+ "epoch": 0.9266040981791548,
3074
+ "grad_norm": 0.8899914622306824,
3075
+ "learning_rate": 2.767944936086529e-05,
3076
+ "loss": 0.9234,
3077
+ "mean_token_accuracy": 0.7661155204288661,
3078
+ "num_tokens": 594823039.0,
3079
+ "step": 3400
3080
+ },
3081
+ {
3082
+ "epoch": 0.92932940435027,
3083
+ "grad_norm": 0.8291741609573364,
3084
+ "learning_rate": 2.7609214777356374e-05,
3085
+ "loss": 0.9049,
3086
+ "mean_token_accuracy": 0.7698826493695379,
3087
+ "num_tokens": 596625643.0,
3088
+ "step": 3410
3089
+ },
3090
+ {
3091
+ "epoch": 0.9320547105213851,
3092
+ "grad_norm": 0.8303592205047607,
3093
+ "learning_rate": 2.753898019384745e-05,
3094
+ "loss": 0.9194,
3095
+ "mean_token_accuracy": 0.7646890787407756,
3096
+ "num_tokens": 598409404.0,
3097
+ "step": 3420
3098
+ },
3099
+ {
3100
+ "epoch": 0.9347800166925003,
3101
+ "grad_norm": 1.093499779701233,
3102
+ "learning_rate": 2.746874561033853e-05,
3103
+ "loss": 0.9067,
3104
+ "mean_token_accuracy": 0.7687176914885641,
3105
+ "num_tokens": 600131817.0,
3106
+ "step": 3430
3107
+ },
3108
+ {
3109
+ "epoch": 0.9375053228636154,
3110
+ "grad_norm": 0.7767653465270996,
3111
+ "learning_rate": 2.7398511026829608e-05,
3112
+ "loss": 0.9087,
3113
+ "mean_token_accuracy": 0.7694620480760932,
3114
+ "num_tokens": 601867823.0,
3115
+ "step": 3440
3116
+ },
3117
+ {
3118
+ "epoch": 0.9402306290347306,
3119
+ "grad_norm": 0.8396127223968506,
3120
+ "learning_rate": 2.7328276443320695e-05,
3121
+ "loss": 0.8845,
3122
+ "mean_token_accuracy": 0.7755044038407505,
3123
+ "num_tokens": 603591263.0,
3124
+ "step": 3450
3125
+ },
3126
+ {
3127
+ "epoch": 0.9429559352058458,
3128
+ "grad_norm": 0.8479565382003784,
3129
+ "learning_rate": 2.7258041859811772e-05,
3130
+ "loss": 0.9151,
3131
+ "mean_token_accuracy": 0.7681015016511082,
3132
+ "num_tokens": 605400422.0,
3133
+ "step": 3460
3134
+ },
3135
+ {
3136
+ "epoch": 0.945681241376961,
3137
+ "grad_norm": 0.8595352172851562,
3138
+ "learning_rate": 2.7187807276302852e-05,
3139
+ "loss": 0.9071,
3140
+ "mean_token_accuracy": 0.7697312761098146,
3141
+ "num_tokens": 607180330.0,
3142
+ "step": 3470
3143
+ },
3144
+ {
3145
+ "epoch": 0.9484065475480761,
3146
+ "grad_norm": 0.7876641154289246,
3147
+ "learning_rate": 2.7117572692793936e-05,
3148
+ "loss": 0.8734,
3149
+ "mean_token_accuracy": 0.7765149438753725,
3150
+ "num_tokens": 608937194.0,
3151
+ "step": 3480
3152
+ },
3153
+ {
3154
+ "epoch": 0.9511318537191913,
3155
+ "grad_norm": 0.8947613835334778,
3156
+ "learning_rate": 2.7047338109285016e-05,
3157
+ "loss": 0.9133,
3158
+ "mean_token_accuracy": 0.7668988397344947,
3159
+ "num_tokens": 610676168.0,
3160
+ "step": 3490
3161
+ },
3162
+ {
3163
+ "epoch": 0.9538571598903064,
3164
+ "grad_norm": 0.8162944912910461,
3165
+ "learning_rate": 2.6977103525776093e-05,
3166
+ "loss": 0.8681,
3167
+ "mean_token_accuracy": 0.7774568162858486,
3168
+ "num_tokens": 612491555.0,
3169
+ "step": 3500
3170
+ },
3171
+ {
3172
+ "epoch": 0.9565824660614216,
3173
+ "grad_norm": 0.900248110294342,
3174
+ "learning_rate": 2.6906868942267173e-05,
3175
+ "loss": 0.9189,
3176
+ "mean_token_accuracy": 0.7660327769815922,
3177
+ "num_tokens": 614310820.0,
3178
+ "step": 3510
3179
+ },
3180
+ {
3181
+ "epoch": 0.9593077722325367,
3182
+ "grad_norm": 0.8359858989715576,
3183
+ "learning_rate": 2.6836634358758256e-05,
3184
+ "loss": 0.9224,
3185
+ "mean_token_accuracy": 0.7663468400016427,
3186
+ "num_tokens": 616071087.0,
3187
+ "step": 3520
3188
+ },
3189
+ {
3190
+ "epoch": 0.9620330784036519,
3191
+ "grad_norm": 0.872882604598999,
3192
+ "learning_rate": 2.6766399775249333e-05,
3193
+ "loss": 0.8712,
3194
+ "mean_token_accuracy": 0.7756553223356605,
3195
+ "num_tokens": 617769166.0,
3196
+ "step": 3530
3197
+ },
3198
+ {
3199
+ "epoch": 0.964758384574767,
3200
+ "grad_norm": 0.8891421556472778,
3201
+ "learning_rate": 2.6696165191740413e-05,
3202
+ "loss": 0.9005,
3203
+ "mean_token_accuracy": 0.7701712142676115,
3204
+ "num_tokens": 619538232.0,
3205
+ "step": 3540
3206
+ },
3207
+ {
3208
+ "epoch": 0.9674836907458823,
3209
+ "grad_norm": 0.7362189888954163,
3210
+ "learning_rate": 2.662593060823149e-05,
3211
+ "loss": 0.8938,
3212
+ "mean_token_accuracy": 0.7728794073686004,
3213
+ "num_tokens": 621303314.0,
3214
+ "step": 3550
3215
+ },
3216
+ {
3217
+ "epoch": 0.9702089969169974,
3218
+ "grad_norm": 0.7591912150382996,
3219
+ "learning_rate": 2.6555696024722577e-05,
3220
+ "loss": 0.8785,
3221
+ "mean_token_accuracy": 0.7754427226260304,
3222
+ "num_tokens": 623100776.0,
3223
+ "step": 3560
3224
+ },
3225
+ {
3226
+ "epoch": 0.9729343030881126,
3227
+ "grad_norm": 0.8404043912887573,
3228
+ "learning_rate": 2.6485461441213654e-05,
3229
+ "loss": 0.9387,
3230
+ "mean_token_accuracy": 0.7616929830983281,
3231
+ "num_tokens": 624821413.0,
3232
+ "step": 3570
3233
+ },
3234
+ {
3235
+ "epoch": 0.9756596092592277,
3236
+ "grad_norm": 0.8890523314476013,
3237
+ "learning_rate": 2.6415226857704734e-05,
3238
+ "loss": 0.9095,
3239
+ "mean_token_accuracy": 0.7678571032360196,
3240
+ "num_tokens": 626628545.0,
3241
+ "step": 3580
3242
+ },
3243
+ {
3244
+ "epoch": 0.9783849154303429,
3245
+ "grad_norm": 0.7919222712516785,
3246
+ "learning_rate": 2.6344992274195818e-05,
3247
+ "loss": 0.8906,
3248
+ "mean_token_accuracy": 0.7729906640946865,
3249
+ "num_tokens": 628425036.0,
3250
+ "step": 3590
3251
+ },
3252
+ {
3253
+ "epoch": 0.981110221601458,
3254
+ "grad_norm": 0.7847779393196106,
3255
+ "learning_rate": 2.6274757690686898e-05,
3256
+ "loss": 0.8636,
3257
+ "mean_token_accuracy": 0.7760783301666379,
3258
+ "num_tokens": 630092739.0,
3259
+ "step": 3600
3260
+ },
3261
+ {
3262
+ "epoch": 0.9838355277725732,
3263
+ "grad_norm": 0.964148998260498,
3264
+ "learning_rate": 2.6204523107177975e-05,
3265
+ "loss": 0.915,
3266
+ "mean_token_accuracy": 0.7690061263740062,
3267
+ "num_tokens": 631791605.0,
3268
+ "step": 3610
3269
+ },
3270
+ {
3271
+ "epoch": 0.9865608339436883,
3272
+ "grad_norm": 0.9110335111618042,
3273
+ "learning_rate": 2.6134288523669055e-05,
3274
+ "loss": 0.9026,
3275
+ "mean_token_accuracy": 0.7687728941440582,
3276
+ "num_tokens": 633554055.0,
3277
+ "step": 3620
3278
+ },
3279
+ {
3280
+ "epoch": 0.9892861401148035,
3281
+ "grad_norm": 0.8210707902908325,
3282
+ "learning_rate": 2.606405394016014e-05,
3283
+ "loss": 0.8729,
3284
+ "mean_token_accuracy": 0.7756300155073405,
3285
+ "num_tokens": 635269789.0,
3286
+ "step": 3630
3287
+ },
3288
+ {
3289
+ "epoch": 0.9920114462859186,
3290
+ "grad_norm": 0.8379694819450378,
3291
+ "learning_rate": 2.5993819356651215e-05,
3292
+ "loss": 0.8912,
3293
+ "mean_token_accuracy": 0.7726977992802858,
3294
+ "num_tokens": 637078092.0,
3295
+ "step": 3640
3296
+ },
3297
+ {
3298
+ "epoch": 0.9947367524570339,
3299
+ "grad_norm": 0.867736279964447,
3300
+ "learning_rate": 2.5923584773142296e-05,
3301
+ "loss": 0.8709,
3302
+ "mean_token_accuracy": 0.7778057091869414,
3303
+ "num_tokens": 638813927.0,
3304
+ "step": 3650
3305
+ },
3306
+ {
3307
+ "epoch": 0.997462058628149,
3308
+ "grad_norm": 0.9498102068901062,
3309
+ "learning_rate": 2.5853350189633372e-05,
3310
+ "loss": 0.8953,
3311
+ "mean_token_accuracy": 0.7722262794151902,
3312
+ "num_tokens": 640594007.0,
3313
+ "step": 3660
3314
+ },
3315
+ {
3316
+ "epoch": 1.0,
3317
+ "grad_norm": 0.4446529746055603,
3318
+ "learning_rate": 2.578311560612446e-05,
3319
+ "loss": 0.8011,
3320
+ "mean_token_accuracy": 0.7779095188883327,
3321
+ "num_tokens": 642209396.0,
3322
+ "step": 3670
3323
+ }
3324
+ ],
3325
+ "logging_steps": 10,
3326
+ "max_steps": 7340,
3327
+ "num_input_tokens_seen": 0,
3328
+ "num_train_epochs": 2,
3329
+ "save_steps": 500.0,
3330
+ "stateful_callbacks": {
3331
+ "TrainerControl": {
3332
+ "args": {
3333
+ "should_epoch_stop": false,
3334
+ "should_evaluate": false,
3335
+ "should_log": false,
3336
+ "should_save": true,
3337
+ "should_training_stop": false
3338
+ },
3339
+ "attributes": {}
3340
+ }
3341
+ },
3342
+ "total_flos": 1.1108118326981394e+19,
3343
+ "train_batch_size": 8,
3344
+ "trial_name": null,
3345
+ "trial_params": null
3346
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38a7388080638a4a7e711db054c9c235b2f3d0b9c95f7647c7c48ddfb1b33262
3
+ size 6072