diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..d0af36d1a1873fba38b127175cb22bb7426deb87 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-684/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-456/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-570/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-342/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ca4838942a3e3fcabcad64936fc2f013dc5e2f7a --- /dev/null +++ b/README.md @@ -0,0 +1,202 @@ +--- +base_model: anthracite-core/Mistral-Small-3.2-24B-Instruct-2506-ChatML +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..411d9073ee2dcf24cc3e3e249c2e1309628e22cb --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Gryphe/Codex-24B-Small-3.2", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": false, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.5, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "o_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/adapter_model.safetensors b/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..806e3619c8030f53e08abcd6250432c8f04235f9 --- /dev/null +++ b/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866acc0f7d32e7ea89fe355667d349175c12476b0e2777f78da44966deec3355 +size 63984 diff --git a/checkpoint-342/README.md b/checkpoint-342/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb5cf17f38ede36f3fb46a89995daa3f44167898 --- /dev/null +++ b/checkpoint-342/README.md @@ -0,0 +1,202 @@ +--- +base_model: Gryphe/Codex-24B-Small-3.2 +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/checkpoint-342/adapter_config.json b/checkpoint-342/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa0535ca25a4eef78d9c51576f13234c64c95fb --- /dev/null +++ b/checkpoint-342/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Gryphe/Codex-24B-Small-3.2", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.5, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "o_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/checkpoint-342/adapter_model.safetensors b/checkpoint-342/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..246c1fb7bc37f9e38e998d50cc9c90cc15e1aab7 --- /dev/null +++ b/checkpoint-342/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b8b5b06469223b9ce309464eda3e6f334c8786eab0f073840e6923f088035e5 +size 1478569432 diff --git a/checkpoint-342/global_step342/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-342/global_step342/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01b7ecc3757b024ca6ab3fa1866609114ece4d76 --- /dev/null +++ b/checkpoint-342/global_step342/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbeca2d9f3ca2790f204aac7d55053d123ae322a58723a081066910bcc15df6f +size 4459517761 diff --git a/checkpoint-342/global_step342/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-342/global_step342/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdb4c7ea459f652f81b429b5f10b191258d33d3e --- /dev/null +++ b/checkpoint-342/global_step342/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acec317f64766e754bb4b902816242338b3a77dbac7303be4d52929ee54aa8fa +size 557481 diff --git a/checkpoint-342/latest b/checkpoint-342/latest new file mode 100644 index 0000000000000000000000000000000000000000..c865948ad34ed67e3b6a2d0505df96492e4bcc82 --- /dev/null +++ b/checkpoint-342/latest @@ -0,0 +1 @@ +global_step342 \ No newline at end of file diff --git a/checkpoint-342/rng_state.pth b/checkpoint-342/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..ed69a3eee2bf9b739f6c97814079050ca031b570 --- /dev/null +++ b/checkpoint-342/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d53eaec910b7aa0fc2c4ce89ce1a23cae0d3aae8a6d137840114446995192f +size 14709 diff --git a/checkpoint-342/scheduler.pt b/checkpoint-342/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5ecfc66e8eec097b15c66f0bd18765b5939afb0 --- /dev/null +++ b/checkpoint-342/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f24f9ce72f8db3b91aa0e7cf6e4dbee35ae7716b75f8003ebc0864c49fee3597 +size 1401 diff --git a/checkpoint-342/special_tokens_map.json b/checkpoint-342/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..4332a4c6d2a485806f55e02260db8a73698e11fb --- /dev/null +++ b/checkpoint-342/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-342/tokenizer.json b/checkpoint-342/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8fab5451b04e019d1612cc5088df62aee9a8208c --- /dev/null +++ b/checkpoint-342/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e5daf1e619597fd56ff036925ea189916417c665d5bdd61b80199cf8d4e6d2 +size 17078029 diff --git a/checkpoint-342/tokenizer_config.json b/checkpoint-342/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..814201fe59040b12e956fb649c749a42bdce1e2f --- /dev/null +++ b/checkpoint-342/tokenizer_config.json @@ -0,0 +1,8021 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "[IMG]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "[IMG_BREAK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "[IMG_END]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "[PREFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "[MIDDLE]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "[SUFFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "[SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "[/SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "[TOOL_CONTENT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "109": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "110": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "111": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "112": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "113": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "114": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "115": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "116": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "117": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "118": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "119": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "120": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "121": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "122": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "123": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "124": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "125": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "126": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "127": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "129": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "130": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "132": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "133": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "134": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "135": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "136": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "137": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "138": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "139": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "140": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "141": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "142": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "143": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "144": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "145": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "146": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "147": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "148": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "149": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "150": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "152": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "153": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "154": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "155": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "156": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "157": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "158": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "159": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "160": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "161": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "162": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "164": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "165": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "166": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "167": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "185": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "186": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "187": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "188": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "189": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "190": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "191": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "222": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "223": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "224": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "225": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "226": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "227": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "228": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "229": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "230": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "231": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "232": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "233": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "234": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "235": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "236": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "237": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "238": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "239": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "240": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "241": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "242": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "243": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "244": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "245": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "246": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "247": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "249": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "251": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "252": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "253": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "254": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "255": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "257": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "258": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "259": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "260": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "262": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "263": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "264": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "265": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "266": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "267": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "268": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "285": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "286": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "287": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "288": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "289": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "290": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "291": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "292": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "293": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "294": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "295": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "296": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "297": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "298": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "299": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "300": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "301": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "302": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "303": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "304": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "305": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "306": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "307": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "308": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "309": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "310": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "311": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "312": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "313": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "314": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "315": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "316": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "317": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "318": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "319": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "320": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "321": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "322": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "323": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "324": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "325": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "326": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "327": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "328": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "329": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "330": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "331": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "332": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "335": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "336": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "337": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "338": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "339": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "340": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "341": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "342": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "343": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "344": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "345": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "346": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "347": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "348": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "349": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "360": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "361": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "362": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "363": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "364": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "365": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "366": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "367": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "368": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "369": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "370": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "371": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "372": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "373": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "374": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "375": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "376": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "377": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "378": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "379": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "380": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "381": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "382": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "383": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "384": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "385": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "386": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "387": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "388": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "389": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "390": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "391": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "392": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "393": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "394": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "395": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "396": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "397": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "398": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "399": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "400": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "401": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "402": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "403": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "404": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "405": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "406": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "407": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "408": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "409": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "410": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "411": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "412": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "413": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "414": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "415": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "416": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "417": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "418": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "419": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "420": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "421": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "422": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "423": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "424": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "425": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "426": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "427": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "428": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "429": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "430": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "431": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "432": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "433": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "434": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "435": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "436": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "437": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "438": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "439": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "440": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "441": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "442": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "443": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "444": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "445": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "446": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "447": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "448": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "449": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "450": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "451": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "452": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "453": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "454": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "455": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "456": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "457": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "458": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "459": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "460": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "461": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "462": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "463": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "464": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "465": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "466": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "467": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "468": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "469": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "470": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "471": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "472": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "473": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "474": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "475": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "476": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "477": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "478": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "479": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "480": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "481": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "482": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "483": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "484": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "485": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "486": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "487": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "488": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "489": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "490": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "491": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "492": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "493": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "494": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "495": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "496": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "497": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "498": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "499": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "500": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "501": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "502": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "503": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "504": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "505": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "506": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "507": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "508": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "509": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "510": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "511": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "512": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "513": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "514": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "515": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "516": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "517": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "518": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "519": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "520": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "521": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "522": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "523": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "524": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "525": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "526": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "527": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "528": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "529": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "530": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "531": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "532": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "533": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "534": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "535": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "536": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "537": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "538": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "539": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "540": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "541": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "542": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "543": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "544": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "545": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "546": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "547": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "548": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "549": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "550": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "551": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "552": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "553": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "554": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "555": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "556": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "557": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "558": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "559": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "560": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "561": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "562": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "563": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "564": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "565": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "566": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "567": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "568": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "569": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "570": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "571": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "572": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "573": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "574": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "575": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "576": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "577": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "578": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "579": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "580": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "581": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "582": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "583": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "584": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "585": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "586": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "587": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "588": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "589": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "590": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "591": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "592": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "593": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "594": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "595": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "596": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "597": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "598": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "599": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "600": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "601": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "602": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "603": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "604": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "605": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "606": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "607": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "608": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "609": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "610": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "611": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "612": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "613": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "614": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "615": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "616": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "617": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "618": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "619": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "620": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "621": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "622": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "623": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "624": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "625": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "626": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "627": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "628": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "629": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "630": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "631": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "632": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "633": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "634": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "635": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "636": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "637": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "638": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "639": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "640": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "641": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "642": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "643": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "644": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "645": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "646": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "647": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "650": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "651": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "652": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "653": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "654": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "655": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "656": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "659": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "660": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "661": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "662": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "663": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "664": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "675": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "676": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "677": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "678": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "679": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "680": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "681": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "682": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "683": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "684": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "685": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "686": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "687": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "688": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "689": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "690": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "691": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "692": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "693": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "694": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "695": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "696": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "697": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "698": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "699": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "700": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "701": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "702": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "703": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "704": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "705": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "706": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "707": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "708": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "709": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "710": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "711": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "712": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "713": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "714": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "715": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "716": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "717": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "718": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "719": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "720": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "721": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "722": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "723": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "724": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "725": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "726": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "727": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "728": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "729": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "730": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "731": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "732": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "733": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "734": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "735": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "736": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "737": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "738": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "739": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "740": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "741": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "742": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "743": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "744": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "745": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "746": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "747": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "748": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "749": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "750": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "751": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "752": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "753": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "754": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "755": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "756": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "757": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "758": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "759": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "760": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "761": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "762": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "763": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "764": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "765": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "766": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "767": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "768": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "769": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "770": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "771": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "772": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "773": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "774": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "775": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "776": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "777": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "778": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "779": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "780": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "781": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "782": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "783": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "784": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "785": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "786": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "787": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "788": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "789": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "790": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "791": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "792": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "793": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "794": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "795": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "796": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "797": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "798": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "799": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "800": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "801": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "802": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "803": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "804": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "805": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "806": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "807": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "808": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "809": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "810": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "811": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "812": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "813": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "814": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "815": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "816": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "817": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "818": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "819": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "820": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "821": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "822": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "823": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "824": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "825": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "826": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "827": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "828": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "829": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "830": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "831": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "832": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "833": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "834": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "835": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "836": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "837": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "838": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "839": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "840": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "841": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "842": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "843": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "844": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "845": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "846": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "847": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "848": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "849": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "850": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "851": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "852": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "853": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "854": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "855": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "856": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "857": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "863": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "864": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "865": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "866": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "867": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "868": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "869": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "870": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "871": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "880": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "881": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "882": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "883": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "884": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "885": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "886": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "887": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "888": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "889": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "890": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "891": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "892": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "893": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "894": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "895": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "896": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "897": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "898": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "899": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "900": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "901": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "902": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "903": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "904": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "905": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "906": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "907": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "908": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "909": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "910": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "911": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "912": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "913": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "914": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "915": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "916": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "917": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "918": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "919": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "920": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "921": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "922": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "923": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "924": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "925": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "926": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "927": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "928": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "929": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "930": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "931": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "932": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "933": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "934": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "935": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "936": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "937": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "938": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "939": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "940": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "941": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "942": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "943": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "944": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "945": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "946": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "947": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "948": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "949": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "950": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "951": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "952": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "953": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "954": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "955": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "956": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "957": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "958": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "959": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "960": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "961": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "962": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "963": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "964": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "965": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "966": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "967": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "968": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "969": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "970": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "971": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "972": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "973": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "974": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "975": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "976": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "977": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "978": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "979": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "980": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "981": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "982": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "983": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "984": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "985": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "986": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "987": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "988": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "989": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "990": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "991": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "992": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "993": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "994": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "995": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "996": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "997": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "998": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "999": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 131072, + "pad_token": "", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-342/trainer_state.json b/checkpoint-342/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..709f5676887dd9f3d56edfa84b0a708376194756 --- /dev/null +++ b/checkpoint-342/trainer_state.json @@ -0,0 +1,2428 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.505124450951684, + "eval_steps": 500, + "global_step": 342, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004392386530014641, + "grad_norm": 2.023870796411983, + "learning_rate": 0.0, + "loss": 1.8021, + "step": 1 + }, + { + "epoch": 0.008784773060029283, + "grad_norm": 2.0723948799697984, + "learning_rate": 2.0000000000000002e-07, + "loss": 1.8796, + "step": 2 + }, + { + "epoch": 0.013177159590043924, + "grad_norm": 2.0322569672979474, + "learning_rate": 4.0000000000000003e-07, + "loss": 1.855, + "step": 3 + }, + { + "epoch": 0.017569546120058566, + "grad_norm": 2.111472782868084, + "learning_rate": 6.000000000000001e-07, + "loss": 1.8545, + "step": 4 + }, + { + "epoch": 0.021961932650073207, + "grad_norm": 2.096607607233168, + "learning_rate": 8.000000000000001e-07, + "loss": 1.807, + "step": 5 + }, + { + "epoch": 0.02635431918008785, + "grad_norm": 2.0771855831079518, + "learning_rate": 1.0000000000000002e-06, + "loss": 1.8208, + "step": 6 + }, + { + "epoch": 0.03074670571010249, + "grad_norm": 2.0341840619969567, + "learning_rate": 1.2000000000000002e-06, + "loss": 1.8754, + "step": 7 + }, + { + "epoch": 0.03513909224011713, + "grad_norm": 2.0564329237094356, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.8882, + "step": 8 + }, + { + "epoch": 0.03953147877013177, + "grad_norm": 1.9900074702634307, + "learning_rate": 1.6000000000000001e-06, + "loss": 1.8733, + "step": 9 + }, + { + "epoch": 0.043923865300146414, + "grad_norm": 1.895599415585885, + "learning_rate": 1.8000000000000001e-06, + "loss": 1.8474, + "step": 10 + }, + { + "epoch": 0.048316251830161056, + "grad_norm": 1.9304209419460994, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.884, + "step": 11 + }, + { + "epoch": 0.0527086383601757, + "grad_norm": 1.6511439041472846, + "learning_rate": 2.2e-06, + "loss": 1.8452, + "step": 12 + }, + { + "epoch": 0.05710102489019034, + "grad_norm": 1.6727753891125912, + "learning_rate": 2.4000000000000003e-06, + "loss": 1.8632, + "step": 13 + }, + { + "epoch": 0.06149341142020498, + "grad_norm": 1.6571693399048197, + "learning_rate": 2.6e-06, + "loss": 1.8419, + "step": 14 + }, + { + "epoch": 0.06588579795021962, + "grad_norm": 1.5588712723347087, + "learning_rate": 2.8000000000000003e-06, + "loss": 1.8163, + "step": 15 + }, + { + "epoch": 0.07027818448023426, + "grad_norm": 1.57706405792705, + "learning_rate": 3e-06, + "loss": 1.7808, + "step": 16 + }, + { + "epoch": 0.0746705710102489, + "grad_norm": 1.4841447475095337, + "learning_rate": 3.2000000000000003e-06, + "loss": 1.7854, + "step": 17 + }, + { + "epoch": 0.07906295754026355, + "grad_norm": 1.2785291394155553, + "learning_rate": 3.4000000000000005e-06, + "loss": 1.7531, + "step": 18 + }, + { + "epoch": 0.08345534407027819, + "grad_norm": 1.1759630521885338, + "learning_rate": 3.6000000000000003e-06, + "loss": 1.8185, + "step": 19 + }, + { + "epoch": 0.08784773060029283, + "grad_norm": 1.1188613513306425, + "learning_rate": 3.8000000000000005e-06, + "loss": 1.849, + "step": 20 + }, + { + "epoch": 0.09224011713030747, + "grad_norm": 1.008741263333265, + "learning_rate": 4.000000000000001e-06, + "loss": 1.775, + "step": 21 + }, + { + "epoch": 0.09663250366032211, + "grad_norm": 0.9240991279618896, + "learning_rate": 4.2000000000000004e-06, + "loss": 1.8253, + "step": 22 + }, + { + "epoch": 0.10102489019033675, + "grad_norm": 0.9015845652170666, + "learning_rate": 4.4e-06, + "loss": 1.7967, + "step": 23 + }, + { + "epoch": 0.1054172767203514, + "grad_norm": 0.8007281301128234, + "learning_rate": 4.600000000000001e-06, + "loss": 1.7958, + "step": 24 + }, + { + "epoch": 0.10980966325036604, + "grad_norm": 0.7263390923205475, + "learning_rate": 4.800000000000001e-06, + "loss": 1.7744, + "step": 25 + }, + { + "epoch": 0.11420204978038068, + "grad_norm": 0.6527771074756341, + "learning_rate": 5e-06, + "loss": 1.7502, + "step": 26 + }, + { + "epoch": 0.11859443631039532, + "grad_norm": 0.5943189060332841, + "learning_rate": 5.2e-06, + "loss": 1.7378, + "step": 27 + }, + { + "epoch": 0.12298682284040996, + "grad_norm": 0.5426800655727773, + "learning_rate": 5.400000000000001e-06, + "loss": 1.7372, + "step": 28 + }, + { + "epoch": 0.1273792093704246, + "grad_norm": 0.46887045607953654, + "learning_rate": 5.600000000000001e-06, + "loss": 1.7141, + "step": 29 + }, + { + "epoch": 0.13177159590043924, + "grad_norm": 0.4408809706079413, + "learning_rate": 5.8e-06, + "loss": 1.6977, + "step": 30 + }, + { + "epoch": 0.13616398243045388, + "grad_norm": 0.4867065604810947, + "learning_rate": 6e-06, + "loss": 1.7379, + "step": 31 + }, + { + "epoch": 0.14055636896046853, + "grad_norm": 0.40882779662256746, + "learning_rate": 6.200000000000001e-06, + "loss": 1.8064, + "step": 32 + }, + { + "epoch": 0.14494875549048317, + "grad_norm": 0.4363561706655908, + "learning_rate": 6.4000000000000006e-06, + "loss": 1.6835, + "step": 33 + }, + { + "epoch": 0.1493411420204978, + "grad_norm": 0.4443411222267533, + "learning_rate": 6.600000000000001e-06, + "loss": 1.7464, + "step": 34 + }, + { + "epoch": 0.15373352855051245, + "grad_norm": 0.4519702758206936, + "learning_rate": 6.800000000000001e-06, + "loss": 1.6891, + "step": 35 + }, + { + "epoch": 0.1581259150805271, + "grad_norm": 0.4174999907173025, + "learning_rate": 7e-06, + "loss": 1.7262, + "step": 36 + }, + { + "epoch": 0.16251830161054173, + "grad_norm": 0.46742969885702906, + "learning_rate": 7.2000000000000005e-06, + "loss": 1.7608, + "step": 37 + }, + { + "epoch": 0.16691068814055637, + "grad_norm": 0.4155692294314683, + "learning_rate": 7.4e-06, + "loss": 1.7211, + "step": 38 + }, + { + "epoch": 0.17130307467057102, + "grad_norm": 0.4123169421407387, + "learning_rate": 7.600000000000001e-06, + "loss": 1.6956, + "step": 39 + }, + { + "epoch": 0.17569546120058566, + "grad_norm": 0.3887606381731466, + "learning_rate": 7.800000000000002e-06, + "loss": 1.7391, + "step": 40 + }, + { + "epoch": 0.1800878477306003, + "grad_norm": 0.37535703391318914, + "learning_rate": 8.000000000000001e-06, + "loss": 1.751, + "step": 41 + }, + { + "epoch": 0.18448023426061494, + "grad_norm": 0.48621902751310403, + "learning_rate": 8.2e-06, + "loss": 1.7091, + "step": 42 + }, + { + "epoch": 0.18887262079062958, + "grad_norm": 0.39145172998871297, + "learning_rate": 8.400000000000001e-06, + "loss": 1.7764, + "step": 43 + }, + { + "epoch": 0.19326500732064422, + "grad_norm": 0.3735488902443606, + "learning_rate": 8.6e-06, + "loss": 1.7058, + "step": 44 + }, + { + "epoch": 0.19765739385065886, + "grad_norm": 0.33988576479414434, + "learning_rate": 8.8e-06, + "loss": 1.7305, + "step": 45 + }, + { + "epoch": 0.2020497803806735, + "grad_norm": 0.9733163465798257, + "learning_rate": 9e-06, + "loss": 1.7567, + "step": 46 + }, + { + "epoch": 0.20644216691068815, + "grad_norm": 0.3344321176956884, + "learning_rate": 9.200000000000002e-06, + "loss": 1.7277, + "step": 47 + }, + { + "epoch": 0.2108345534407028, + "grad_norm": 0.3647805237752918, + "learning_rate": 9.4e-06, + "loss": 1.7197, + "step": 48 + }, + { + "epoch": 0.21522693997071743, + "grad_norm": 0.36967036325253694, + "learning_rate": 9.600000000000001e-06, + "loss": 1.6994, + "step": 49 + }, + { + "epoch": 0.21961932650073207, + "grad_norm": 0.32386928338357696, + "learning_rate": 9.800000000000001e-06, + "loss": 1.6975, + "step": 50 + }, + { + "epoch": 0.2240117130307467, + "grad_norm": 0.33940794357965215, + "learning_rate": 1e-05, + "loss": 1.7278, + "step": 51 + }, + { + "epoch": 0.22840409956076135, + "grad_norm": 0.3396058372359069, + "learning_rate": 9.999966483034437e-06, + "loss": 1.6342, + "step": 52 + }, + { + "epoch": 0.232796486090776, + "grad_norm": 0.3395876900960321, + "learning_rate": 9.999865932587107e-06, + "loss": 1.6846, + "step": 53 + }, + { + "epoch": 0.23718887262079064, + "grad_norm": 0.3320830942903407, + "learning_rate": 9.999698350006063e-06, + "loss": 1.6744, + "step": 54 + }, + { + "epoch": 0.24158125915080528, + "grad_norm": 0.318642304519145, + "learning_rate": 9.999463737538054e-06, + "loss": 1.6633, + "step": 55 + }, + { + "epoch": 0.24597364568081992, + "grad_norm": 0.3302002388842341, + "learning_rate": 9.999162098328475e-06, + "loss": 1.6994, + "step": 56 + }, + { + "epoch": 0.25036603221083453, + "grad_norm": 0.30428798634415904, + "learning_rate": 9.998793436421342e-06, + "loss": 1.6728, + "step": 57 + }, + { + "epoch": 0.2547584187408492, + "grad_norm": 0.30662673120989614, + "learning_rate": 9.998357756759223e-06, + "loss": 1.6283, + "step": 58 + }, + { + "epoch": 0.2591508052708638, + "grad_norm": 0.34305953470241257, + "learning_rate": 9.997855065183185e-06, + "loss": 1.7269, + "step": 59 + }, + { + "epoch": 0.2635431918008785, + "grad_norm": 0.29647698247102616, + "learning_rate": 9.997285368432704e-06, + "loss": 1.6108, + "step": 60 + }, + { + "epoch": 0.2679355783308931, + "grad_norm": 0.30904144706299363, + "learning_rate": 9.996648674145584e-06, + "loss": 1.5925, + "step": 61 + }, + { + "epoch": 0.27232796486090777, + "grad_norm": 0.28294691041282544, + "learning_rate": 9.995944990857848e-06, + "loss": 1.7004, + "step": 62 + }, + { + "epoch": 0.2767203513909224, + "grad_norm": 0.4537316624659888, + "learning_rate": 9.99517432800363e-06, + "loss": 1.673, + "step": 63 + }, + { + "epoch": 0.28111273792093705, + "grad_norm": 0.33334142649617554, + "learning_rate": 9.994336695915041e-06, + "loss": 1.7305, + "step": 64 + }, + { + "epoch": 0.28550512445095166, + "grad_norm": 0.2906595328556503, + "learning_rate": 9.993432105822034e-06, + "loss": 1.6511, + "step": 65 + }, + { + "epoch": 0.28989751098096633, + "grad_norm": 0.30937353104293036, + "learning_rate": 9.992460569852256e-06, + "loss": 1.7055, + "step": 66 + }, + { + "epoch": 0.29428989751098095, + "grad_norm": 0.29714918983678074, + "learning_rate": 9.99142210103088e-06, + "loss": 1.6022, + "step": 67 + }, + { + "epoch": 0.2986822840409956, + "grad_norm": 0.3077061364082642, + "learning_rate": 9.99031671328044e-06, + "loss": 1.6167, + "step": 68 + }, + { + "epoch": 0.30307467057101023, + "grad_norm": 0.31797999416909106, + "learning_rate": 9.98914442142063e-06, + "loss": 1.6063, + "step": 69 + }, + { + "epoch": 0.3074670571010249, + "grad_norm": 0.28544810434107143, + "learning_rate": 9.987905241168117e-06, + "loss": 1.673, + "step": 70 + }, + { + "epoch": 0.3118594436310395, + "grad_norm": 0.3113516293182134, + "learning_rate": 9.986599189136325e-06, + "loss": 1.6759, + "step": 71 + }, + { + "epoch": 0.3162518301610542, + "grad_norm": 0.32958924621680397, + "learning_rate": 9.985226282835216e-06, + "loss": 1.6894, + "step": 72 + }, + { + "epoch": 0.3206442166910688, + "grad_norm": 0.3189944736085633, + "learning_rate": 9.983786540671052e-06, + "loss": 1.7762, + "step": 73 + }, + { + "epoch": 0.32503660322108346, + "grad_norm": 0.30229970638681786, + "learning_rate": 9.982279981946143e-06, + "loss": 1.709, + "step": 74 + }, + { + "epoch": 0.3294289897510981, + "grad_norm": 0.308085411523194, + "learning_rate": 9.980706626858607e-06, + "loss": 1.6741, + "step": 75 + }, + { + "epoch": 0.33382137628111275, + "grad_norm": 0.2851270169881767, + "learning_rate": 9.979066496502075e-06, + "loss": 1.656, + "step": 76 + }, + { + "epoch": 0.33821376281112736, + "grad_norm": 0.3115977673015286, + "learning_rate": 9.977359612865424e-06, + "loss": 1.6633, + "step": 77 + }, + { + "epoch": 0.34260614934114203, + "grad_norm": 0.2960787226189042, + "learning_rate": 9.97558599883248e-06, + "loss": 1.673, + "step": 78 + }, + { + "epoch": 0.34699853587115664, + "grad_norm": 0.35701812130442995, + "learning_rate": 9.973745678181705e-06, + "loss": 1.6103, + "step": 79 + }, + { + "epoch": 0.3513909224011713, + "grad_norm": 0.367868955812032, + "learning_rate": 9.971838675585888e-06, + "loss": 1.6768, + "step": 80 + }, + { + "epoch": 0.3557833089311859, + "grad_norm": 0.3295428259298103, + "learning_rate": 9.9698650166118e-06, + "loss": 1.6668, + "step": 81 + }, + { + "epoch": 0.3601756954612006, + "grad_norm": 0.3242883305567894, + "learning_rate": 9.96782472771987e-06, + "loss": 1.7152, + "step": 82 + }, + { + "epoch": 0.3645680819912152, + "grad_norm": 0.3070622417895325, + "learning_rate": 9.965717836263813e-06, + "loss": 1.5697, + "step": 83 + }, + { + "epoch": 0.3689604685212299, + "grad_norm": 0.27753629978682975, + "learning_rate": 9.96354437049027e-06, + "loss": 1.6453, + "step": 84 + }, + { + "epoch": 0.3733528550512445, + "grad_norm": 0.2723902519890988, + "learning_rate": 9.961304359538437e-06, + "loss": 1.7165, + "step": 85 + }, + { + "epoch": 0.37774524158125916, + "grad_norm": 0.29061586618994084, + "learning_rate": 9.958997833439658e-06, + "loss": 1.7135, + "step": 86 + }, + { + "epoch": 0.3821376281112738, + "grad_norm": 0.33085661251847487, + "learning_rate": 9.956624823117036e-06, + "loss": 1.6109, + "step": 87 + }, + { + "epoch": 0.38653001464128844, + "grad_norm": 0.2931612046141165, + "learning_rate": 9.954185360385013e-06, + "loss": 1.7048, + "step": 88 + }, + { + "epoch": 0.39092240117130306, + "grad_norm": 0.2900600158100116, + "learning_rate": 9.951679477948946e-06, + "loss": 1.6861, + "step": 89 + }, + { + "epoch": 0.3953147877013177, + "grad_norm": 0.2775954202644971, + "learning_rate": 9.949107209404664e-06, + "loss": 1.5706, + "step": 90 + }, + { + "epoch": 0.39970717423133234, + "grad_norm": 0.2733537235090247, + "learning_rate": 9.946468589238022e-06, + "loss": 1.7146, + "step": 91 + }, + { + "epoch": 0.404099560761347, + "grad_norm": 0.2710139549067813, + "learning_rate": 9.943763652824437e-06, + "loss": 1.6488, + "step": 92 + }, + { + "epoch": 0.4084919472913616, + "grad_norm": 0.27892641103107996, + "learning_rate": 9.94099243642841e-06, + "loss": 1.6002, + "step": 93 + }, + { + "epoch": 0.4128843338213763, + "grad_norm": 0.4570340740318857, + "learning_rate": 9.93815497720305e-06, + "loss": 1.6686, + "step": 94 + }, + { + "epoch": 0.4172767203513909, + "grad_norm": 0.2816582371174757, + "learning_rate": 9.935251313189564e-06, + "loss": 1.7531, + "step": 95 + }, + { + "epoch": 0.4216691068814056, + "grad_norm": 0.2886345479986427, + "learning_rate": 9.932281483316759e-06, + "loss": 1.744, + "step": 96 + }, + { + "epoch": 0.4260614934114202, + "grad_norm": 0.3178449956639589, + "learning_rate": 9.929245527400504e-06, + "loss": 1.6404, + "step": 97 + }, + { + "epoch": 0.43045387994143486, + "grad_norm": 0.31466032997182714, + "learning_rate": 9.926143486143216e-06, + "loss": 1.6583, + "step": 98 + }, + { + "epoch": 0.43484626647144947, + "grad_norm": 0.3406321749653024, + "learning_rate": 9.922975401133292e-06, + "loss": 1.7313, + "step": 99 + }, + { + "epoch": 0.43923865300146414, + "grad_norm": 0.33050248521276526, + "learning_rate": 9.919741314844577e-06, + "loss": 1.6518, + "step": 100 + }, + { + "epoch": 0.44363103953147875, + "grad_norm": 0.2886134577231967, + "learning_rate": 9.916441270635772e-06, + "loss": 1.6957, + "step": 101 + }, + { + "epoch": 0.4480234260614934, + "grad_norm": 0.27520985458354463, + "learning_rate": 9.913075312749867e-06, + "loss": 1.6052, + "step": 102 + }, + { + "epoch": 0.45241581259150804, + "grad_norm": 0.2649316531653303, + "learning_rate": 9.909643486313533e-06, + "loss": 1.6739, + "step": 103 + }, + { + "epoch": 0.4568081991215227, + "grad_norm": 0.27557834576379425, + "learning_rate": 9.90614583733654e-06, + "loss": 1.6584, + "step": 104 + }, + { + "epoch": 0.4612005856515373, + "grad_norm": 0.3243046224133678, + "learning_rate": 9.90258241271112e-06, + "loss": 1.6491, + "step": 105 + }, + { + "epoch": 0.465592972181552, + "grad_norm": 0.2601090357711318, + "learning_rate": 9.89895326021134e-06, + "loss": 1.6662, + "step": 106 + }, + { + "epoch": 0.4699853587115666, + "grad_norm": 0.2842151241183655, + "learning_rate": 9.895258428492475e-06, + "loss": 1.6501, + "step": 107 + }, + { + "epoch": 0.4743777452415813, + "grad_norm": 0.2594313373745908, + "learning_rate": 9.891497967090344e-06, + "loss": 1.6475, + "step": 108 + }, + { + "epoch": 0.4787701317715959, + "grad_norm": 0.2844945420500575, + "learning_rate": 9.887671926420649e-06, + "loss": 1.7204, + "step": 109 + }, + { + "epoch": 0.48316251830161056, + "grad_norm": 0.27655451441999607, + "learning_rate": 9.8837803577783e-06, + "loss": 1.6838, + "step": 110 + }, + { + "epoch": 0.48755490483162517, + "grad_norm": 0.3017150137441088, + "learning_rate": 9.879823313336723e-06, + "loss": 1.6688, + "step": 111 + }, + { + "epoch": 0.49194729136163984, + "grad_norm": 0.26575250791745125, + "learning_rate": 9.87580084614717e-06, + "loss": 1.6753, + "step": 112 + }, + { + "epoch": 0.49633967789165445, + "grad_norm": 0.2581989783149807, + "learning_rate": 9.871713010137998e-06, + "loss": 1.6244, + "step": 113 + }, + { + "epoch": 0.5007320644216691, + "grad_norm": 0.2622294231157268, + "learning_rate": 9.86755986011395e-06, + "loss": 1.6261, + "step": 114 + }, + { + "epoch": 0.5051244509516838, + "grad_norm": 0.26209513370076304, + "learning_rate": 9.86334145175542e-06, + "loss": 1.6397, + "step": 115 + }, + { + "epoch": 0.5095168374816984, + "grad_norm": 0.2846184522412747, + "learning_rate": 9.859057841617709e-06, + "loss": 1.7091, + "step": 116 + }, + { + "epoch": 0.513909224011713, + "grad_norm": 0.2528850826310284, + "learning_rate": 9.854709087130261e-06, + "loss": 1.6831, + "step": 117 + }, + { + "epoch": 0.5183016105417276, + "grad_norm": 0.32698350420324157, + "learning_rate": 9.850295246595898e-06, + "loss": 1.6431, + "step": 118 + }, + { + "epoch": 0.5226939970717424, + "grad_norm": 0.35727071053972576, + "learning_rate": 9.845816379190037e-06, + "loss": 1.7088, + "step": 119 + }, + { + "epoch": 0.527086383601757, + "grad_norm": 0.28971742333278044, + "learning_rate": 9.841272544959892e-06, + "loss": 1.7195, + "step": 120 + }, + { + "epoch": 0.5314787701317716, + "grad_norm": 0.468869869211724, + "learning_rate": 9.836663804823683e-06, + "loss": 1.6896, + "step": 121 + }, + { + "epoch": 0.5358711566617862, + "grad_norm": 0.3908944943187778, + "learning_rate": 9.831990220569801e-06, + "loss": 1.7406, + "step": 122 + }, + { + "epoch": 0.5402635431918009, + "grad_norm": 0.2665924461613152, + "learning_rate": 9.827251854855992e-06, + "loss": 1.6609, + "step": 123 + }, + { + "epoch": 0.5446559297218155, + "grad_norm": 0.28211372742262864, + "learning_rate": 9.82244877120851e-06, + "loss": 1.6684, + "step": 124 + }, + { + "epoch": 0.5490483162518301, + "grad_norm": 0.2623738132657017, + "learning_rate": 9.817581034021273e-06, + "loss": 1.6923, + "step": 125 + }, + { + "epoch": 0.5534407027818448, + "grad_norm": 0.35357676308356323, + "learning_rate": 9.812648708554991e-06, + "loss": 1.6467, + "step": 126 + }, + { + "epoch": 0.5578330893118595, + "grad_norm": 0.2555639734647961, + "learning_rate": 9.807651860936299e-06, + "loss": 1.6893, + "step": 127 + }, + { + "epoch": 0.5622254758418741, + "grad_norm": 0.26746261958597517, + "learning_rate": 9.802590558156863e-06, + "loss": 1.6754, + "step": 128 + }, + { + "epoch": 0.5666178623718887, + "grad_norm": 0.26383031628885084, + "learning_rate": 9.797464868072489e-06, + "loss": 1.6363, + "step": 129 + }, + { + "epoch": 0.5710102489019033, + "grad_norm": 0.319619389966322, + "learning_rate": 9.792274859402205e-06, + "loss": 1.636, + "step": 130 + }, + { + "epoch": 0.575402635431918, + "grad_norm": 0.2639308579990406, + "learning_rate": 9.787020601727353e-06, + "loss": 1.6079, + "step": 131 + }, + { + "epoch": 0.5797950219619327, + "grad_norm": 0.28226380411742236, + "learning_rate": 9.781702165490638e-06, + "loss": 1.657, + "step": 132 + }, + { + "epoch": 0.5841874084919473, + "grad_norm": 0.27157003543418035, + "learning_rate": 9.7763196219952e-06, + "loss": 1.7009, + "step": 133 + }, + { + "epoch": 0.5885797950219619, + "grad_norm": 0.28487209604498265, + "learning_rate": 9.770873043403648e-06, + "loss": 1.6236, + "step": 134 + }, + { + "epoch": 0.5929721815519766, + "grad_norm": 0.3009729998770739, + "learning_rate": 9.765362502737098e-06, + "loss": 1.6812, + "step": 135 + }, + { + "epoch": 0.5973645680819912, + "grad_norm": 0.2576811873616972, + "learning_rate": 9.759788073874188e-06, + "loss": 1.6885, + "step": 136 + }, + { + "epoch": 0.6017569546120058, + "grad_norm": 0.3108368566509119, + "learning_rate": 9.754149831550097e-06, + "loss": 1.6086, + "step": 137 + }, + { + "epoch": 0.6061493411420205, + "grad_norm": 0.3336050806293322, + "learning_rate": 9.748447851355534e-06, + "loss": 1.6773, + "step": 138 + }, + { + "epoch": 0.6105417276720352, + "grad_norm": 0.28918467659013913, + "learning_rate": 9.742682209735727e-06, + "loss": 1.7141, + "step": 139 + }, + { + "epoch": 0.6149341142020498, + "grad_norm": 0.25507889874035206, + "learning_rate": 9.736852983989405e-06, + "loss": 1.6335, + "step": 140 + }, + { + "epoch": 0.6193265007320644, + "grad_norm": 0.2712795343500641, + "learning_rate": 9.730960252267744e-06, + "loss": 1.6186, + "step": 141 + }, + { + "epoch": 0.623718887262079, + "grad_norm": 0.29326296437100124, + "learning_rate": 9.725004093573343e-06, + "loss": 1.6039, + "step": 142 + }, + { + "epoch": 0.6281112737920937, + "grad_norm": 0.27412689936526563, + "learning_rate": 9.718984587759148e-06, + "loss": 1.594, + "step": 143 + }, + { + "epoch": 0.6325036603221084, + "grad_norm": 0.28908560688778157, + "learning_rate": 9.712901815527387e-06, + "loss": 1.6458, + "step": 144 + }, + { + "epoch": 0.636896046852123, + "grad_norm": 0.26538330998431864, + "learning_rate": 9.706755858428487e-06, + "loss": 1.6546, + "step": 145 + }, + { + "epoch": 0.6412884333821376, + "grad_norm": 0.26498403141947585, + "learning_rate": 9.70054679885998e-06, + "loss": 1.6224, + "step": 146 + }, + { + "epoch": 0.6456808199121523, + "grad_norm": 0.25028323667878993, + "learning_rate": 9.6942747200654e-06, + "loss": 1.6554, + "step": 147 + }, + { + "epoch": 0.6500732064421669, + "grad_norm": 0.3077620507501807, + "learning_rate": 9.687939706133168e-06, + "loss": 1.6751, + "step": 148 + }, + { + "epoch": 0.6544655929721815, + "grad_norm": 0.2738251492188122, + "learning_rate": 9.68154184199546e-06, + "loss": 1.6139, + "step": 149 + }, + { + "epoch": 0.6588579795021962, + "grad_norm": 0.300766059001253, + "learning_rate": 9.675081213427076e-06, + "loss": 1.6783, + "step": 150 + }, + { + "epoch": 0.6632503660322109, + "grad_norm": 0.2671963968809055, + "learning_rate": 9.668557907044278e-06, + "loss": 1.7028, + "step": 151 + }, + { + "epoch": 0.6676427525622255, + "grad_norm": 0.27596191143138304, + "learning_rate": 9.661972010303641e-06, + "loss": 1.6279, + "step": 152 + }, + { + "epoch": 0.6720351390922401, + "grad_norm": 0.2719766311854951, + "learning_rate": 9.655323611500876e-06, + "loss": 1.6634, + "step": 153 + }, + { + "epoch": 0.6764275256222547, + "grad_norm": 0.2697612116557389, + "learning_rate": 9.648612799769644e-06, + "loss": 1.617, + "step": 154 + }, + { + "epoch": 0.6808199121522694, + "grad_norm": 0.28137116472179047, + "learning_rate": 9.641839665080363e-06, + "loss": 1.6212, + "step": 155 + }, + { + "epoch": 0.6852122986822841, + "grad_norm": 0.2599526010890989, + "learning_rate": 9.635004298239004e-06, + "loss": 1.6542, + "step": 156 + }, + { + "epoch": 0.6896046852122987, + "grad_norm": 0.662388825119619, + "learning_rate": 9.628106790885866e-06, + "loss": 1.633, + "step": 157 + }, + { + "epoch": 0.6939970717423133, + "grad_norm": 0.2913416852335614, + "learning_rate": 9.621147235494357e-06, + "loss": 1.6904, + "step": 158 + }, + { + "epoch": 0.698389458272328, + "grad_norm": 0.26581120890113497, + "learning_rate": 9.614125725369748e-06, + "loss": 1.6583, + "step": 159 + }, + { + "epoch": 0.7027818448023426, + "grad_norm": 0.26696083510238133, + "learning_rate": 9.607042354647924e-06, + "loss": 1.6354, + "step": 160 + }, + { + "epoch": 0.7071742313323572, + "grad_norm": 0.2913753113559095, + "learning_rate": 9.599897218294122e-06, + "loss": 1.5914, + "step": 161 + }, + { + "epoch": 0.7115666178623719, + "grad_norm": 0.2794317973029827, + "learning_rate": 9.59269041210166e-06, + "loss": 1.5732, + "step": 162 + }, + { + "epoch": 0.7159590043923866, + "grad_norm": 0.27364398897711945, + "learning_rate": 9.585422032690644e-06, + "loss": 1.6796, + "step": 163 + }, + { + "epoch": 0.7203513909224012, + "grad_norm": 0.2924137561125512, + "learning_rate": 9.578092177506683e-06, + "loss": 1.6921, + "step": 164 + }, + { + "epoch": 0.7247437774524158, + "grad_norm": 0.27864643658701665, + "learning_rate": 9.570700944819584e-06, + "loss": 1.6442, + "step": 165 + }, + { + "epoch": 0.7291361639824304, + "grad_norm": 0.3172135018133218, + "learning_rate": 9.56324843372202e-06, + "loss": 1.7003, + "step": 166 + }, + { + "epoch": 0.7335285505124451, + "grad_norm": 0.33119379016577744, + "learning_rate": 9.55573474412821e-06, + "loss": 1.6277, + "step": 167 + }, + { + "epoch": 0.7379209370424598, + "grad_norm": 0.24315818847358012, + "learning_rate": 9.548159976772593e-06, + "loss": 1.621, + "step": 168 + }, + { + "epoch": 0.7423133235724744, + "grad_norm": 0.28066914391093295, + "learning_rate": 9.540524233208449e-06, + "loss": 1.6169, + "step": 169 + }, + { + "epoch": 0.746705710102489, + "grad_norm": 0.3096372829142638, + "learning_rate": 9.532827615806561e-06, + "loss": 1.6425, + "step": 170 + }, + { + "epoch": 0.7510980966325037, + "grad_norm": 0.3050362838238551, + "learning_rate": 9.525070227753835e-06, + "loss": 1.5835, + "step": 171 + }, + { + "epoch": 0.7554904831625183, + "grad_norm": 0.27558198897100084, + "learning_rate": 9.517252173051912e-06, + "loss": 1.6866, + "step": 172 + }, + { + "epoch": 0.7598828696925329, + "grad_norm": 0.3415215102331636, + "learning_rate": 9.509373556515781e-06, + "loss": 1.6954, + "step": 173 + }, + { + "epoch": 0.7642752562225475, + "grad_norm": 0.2833145266859935, + "learning_rate": 9.501434483772371e-06, + "loss": 1.5642, + "step": 174 + }, + { + "epoch": 0.7686676427525623, + "grad_norm": 0.261080979929468, + "learning_rate": 9.49343506125913e-06, + "loss": 1.6481, + "step": 175 + }, + { + "epoch": 0.7730600292825769, + "grad_norm": 0.27528584249968596, + "learning_rate": 9.48537539622261e-06, + "loss": 1.6356, + "step": 176 + }, + { + "epoch": 0.7774524158125915, + "grad_norm": 0.28493642680744213, + "learning_rate": 9.477255596717012e-06, + "loss": 1.617, + "step": 177 + }, + { + "epoch": 0.7818448023426061, + "grad_norm": 0.250792323656803, + "learning_rate": 9.469075771602756e-06, + "loss": 1.7373, + "step": 178 + }, + { + "epoch": 0.7862371888726208, + "grad_norm": 0.2815707976300464, + "learning_rate": 9.460836030545007e-06, + "loss": 1.7134, + "step": 179 + }, + { + "epoch": 0.7906295754026355, + "grad_norm": 0.33965220402585167, + "learning_rate": 9.452536484012212e-06, + "loss": 1.66, + "step": 180 + }, + { + "epoch": 0.7950219619326501, + "grad_norm": 0.337075115730111, + "learning_rate": 9.444177243274619e-06, + "loss": 1.7237, + "step": 181 + }, + { + "epoch": 0.7994143484626647, + "grad_norm": 0.2652773355794464, + "learning_rate": 9.43575842040278e-06, + "loss": 1.7238, + "step": 182 + }, + { + "epoch": 0.8038067349926794, + "grad_norm": 0.29569868187991866, + "learning_rate": 9.427280128266049e-06, + "loss": 1.6484, + "step": 183 + }, + { + "epoch": 0.808199121522694, + "grad_norm": 0.2647366968721245, + "learning_rate": 9.418742480531086e-06, + "loss": 1.6557, + "step": 184 + }, + { + "epoch": 0.8125915080527086, + "grad_norm": 0.2624692157687208, + "learning_rate": 9.410145591660301e-06, + "loss": 1.6501, + "step": 185 + }, + { + "epoch": 0.8169838945827232, + "grad_norm": 0.2604406630602592, + "learning_rate": 9.40148957691035e-06, + "loss": 1.6205, + "step": 186 + }, + { + "epoch": 0.821376281112738, + "grad_norm": 0.3809448031526879, + "learning_rate": 9.392774552330567e-06, + "loss": 1.6014, + "step": 187 + }, + { + "epoch": 0.8257686676427526, + "grad_norm": 0.285104899301775, + "learning_rate": 9.384000634761429e-06, + "loss": 1.7119, + "step": 188 + }, + { + "epoch": 0.8301610541727672, + "grad_norm": 0.3603217046093286, + "learning_rate": 9.375167941832974e-06, + "loss": 1.6129, + "step": 189 + }, + { + "epoch": 0.8345534407027818, + "grad_norm": 0.26192371812461096, + "learning_rate": 9.366276591963222e-06, + "loss": 1.6648, + "step": 190 + }, + { + "epoch": 0.8389458272327965, + "grad_norm": 0.27224376529649075, + "learning_rate": 9.357326704356602e-06, + "loss": 1.7368, + "step": 191 + }, + { + "epoch": 0.8433382137628112, + "grad_norm": 0.267560205737413, + "learning_rate": 9.348318399002347e-06, + "loss": 1.6456, + "step": 192 + }, + { + "epoch": 0.8477306002928258, + "grad_norm": 0.38772390686795116, + "learning_rate": 9.339251796672878e-06, + "loss": 1.6765, + "step": 193 + }, + { + "epoch": 0.8521229868228404, + "grad_norm": 0.28624234868841525, + "learning_rate": 9.330127018922195e-06, + "loss": 1.6397, + "step": 194 + }, + { + "epoch": 0.8565153733528551, + "grad_norm": 0.3517826456437289, + "learning_rate": 9.320944188084241e-06, + "loss": 1.6504, + "step": 195 + }, + { + "epoch": 0.8609077598828697, + "grad_norm": 0.3054896957644684, + "learning_rate": 9.31170342727127e-06, + "loss": 1.6581, + "step": 196 + }, + { + "epoch": 0.8653001464128843, + "grad_norm": 0.47217080493290625, + "learning_rate": 9.302404860372185e-06, + "loss": 1.6614, + "step": 197 + }, + { + "epoch": 0.8696925329428989, + "grad_norm": 0.3744290659075852, + "learning_rate": 9.293048612050883e-06, + "loss": 1.6181, + "step": 198 + }, + { + "epoch": 0.8740849194729137, + "grad_norm": 0.2928672407192581, + "learning_rate": 9.283634807744586e-06, + "loss": 1.5801, + "step": 199 + }, + { + "epoch": 0.8784773060029283, + "grad_norm": 0.5500989172351091, + "learning_rate": 9.274163573662158e-06, + "loss": 1.6213, + "step": 200 + }, + { + "epoch": 0.8828696925329429, + "grad_norm": 0.27844961793523915, + "learning_rate": 9.264635036782406e-06, + "loss": 1.6442, + "step": 201 + }, + { + "epoch": 0.8872620790629575, + "grad_norm": 0.2764381586339949, + "learning_rate": 9.255049324852388e-06, + "loss": 1.6403, + "step": 202 + }, + { + "epoch": 0.8916544655929722, + "grad_norm": 0.28692831842946026, + "learning_rate": 9.245406566385698e-06, + "loss": 1.6204, + "step": 203 + }, + { + "epoch": 0.8960468521229868, + "grad_norm": 0.2854958274623322, + "learning_rate": 9.235706890660735e-06, + "loss": 1.6777, + "step": 204 + }, + { + "epoch": 0.9004392386530015, + "grad_norm": 0.2884758073862357, + "learning_rate": 9.225950427718974e-06, + "loss": 1.6414, + "step": 205 + }, + { + "epoch": 0.9048316251830161, + "grad_norm": 0.2774591809427237, + "learning_rate": 9.216137308363235e-06, + "loss": 1.667, + "step": 206 + }, + { + "epoch": 0.9092240117130308, + "grad_norm": 0.26327408771476135, + "learning_rate": 9.206267664155906e-06, + "loss": 1.6472, + "step": 207 + }, + { + "epoch": 0.9136163982430454, + "grad_norm": 0.3163508666559799, + "learning_rate": 9.1963416274172e-06, + "loss": 1.6898, + "step": 208 + }, + { + "epoch": 0.91800878477306, + "grad_norm": 0.2691312569989342, + "learning_rate": 9.18635933122337e-06, + "loss": 1.6788, + "step": 209 + }, + { + "epoch": 0.9224011713030746, + "grad_norm": 0.329963898141559, + "learning_rate": 9.176320909404925e-06, + "loss": 1.7077, + "step": 210 + }, + { + "epoch": 0.9267935578330894, + "grad_norm": 0.3497378648580076, + "learning_rate": 9.16622649654484e-06, + "loss": 1.7203, + "step": 211 + }, + { + "epoch": 0.931185944363104, + "grad_norm": 0.2750435891659104, + "learning_rate": 9.156076227976752e-06, + "loss": 1.6372, + "step": 212 + }, + { + "epoch": 0.9355783308931186, + "grad_norm": 0.29344217135995226, + "learning_rate": 9.145870239783143e-06, + "loss": 1.6294, + "step": 213 + }, + { + "epoch": 0.9399707174231332, + "grad_norm": 0.2713649627402965, + "learning_rate": 9.135608668793511e-06, + "loss": 1.6391, + "step": 214 + }, + { + "epoch": 0.9443631039531479, + "grad_norm": 0.2995344678229402, + "learning_rate": 9.125291652582549e-06, + "loss": 1.6102, + "step": 215 + }, + { + "epoch": 0.9487554904831625, + "grad_norm": 0.32763546183738235, + "learning_rate": 9.114919329468283e-06, + "loss": 1.6823, + "step": 216 + }, + { + "epoch": 0.9531478770131772, + "grad_norm": 0.29620265372375704, + "learning_rate": 9.104491838510235e-06, + "loss": 1.6905, + "step": 217 + }, + { + "epoch": 0.9575402635431918, + "grad_norm": 0.43487598587006066, + "learning_rate": 9.094009319507547e-06, + "loss": 1.6611, + "step": 218 + }, + { + "epoch": 0.9619326500732065, + "grad_norm": 0.32989859860270077, + "learning_rate": 9.08347191299711e-06, + "loss": 1.634, + "step": 219 + }, + { + "epoch": 0.9663250366032211, + "grad_norm": 0.26622257061624277, + "learning_rate": 9.07287976025168e-06, + "loss": 1.6075, + "step": 220 + }, + { + "epoch": 0.9707174231332357, + "grad_norm": 0.28626358026286186, + "learning_rate": 9.062233003277983e-06, + "loss": 1.6045, + "step": 221 + }, + { + "epoch": 0.9751098096632503, + "grad_norm": 0.5424158794876999, + "learning_rate": 9.051531784814817e-06, + "loss": 1.5825, + "step": 222 + }, + { + "epoch": 0.9795021961932651, + "grad_norm": 0.2771987597060821, + "learning_rate": 9.040776248331128e-06, + "loss": 1.6083, + "step": 223 + }, + { + "epoch": 0.9838945827232797, + "grad_norm": 1.5311321947246777, + "learning_rate": 9.029966538024097e-06, + "loss": 1.5973, + "step": 224 + }, + { + "epoch": 0.9882869692532943, + "grad_norm": 0.3081246423484545, + "learning_rate": 9.019102798817196e-06, + "loss": 1.5915, + "step": 225 + }, + { + "epoch": 0.9926793557833089, + "grad_norm": 0.3066770476121554, + "learning_rate": 9.008185176358257e-06, + "loss": 1.5551, + "step": 226 + }, + { + "epoch": 0.9970717423133236, + "grad_norm": 0.29888292004137484, + "learning_rate": 8.997213817017508e-06, + "loss": 1.7461, + "step": 227 + }, + { + "epoch": 1.0043923865300146, + "grad_norm": 0.5938110979809961, + "learning_rate": 8.986188867885617e-06, + "loss": 3.2509, + "step": 228 + }, + { + "epoch": 1.0087847730600292, + "grad_norm": 0.2899326916339286, + "learning_rate": 8.975110476771724e-06, + "loss": 1.5874, + "step": 229 + }, + { + "epoch": 1.0131771595900438, + "grad_norm": 0.2777095345419943, + "learning_rate": 8.96397879220145e-06, + "loss": 1.6706, + "step": 230 + }, + { + "epoch": 1.0175695461200585, + "grad_norm": 0.26363195429841585, + "learning_rate": 8.952793963414908e-06, + "loss": 1.6317, + "step": 231 + }, + { + "epoch": 1.0219619326500733, + "grad_norm": 0.2688971491502818, + "learning_rate": 8.941556140364707e-06, + "loss": 1.654, + "step": 232 + }, + { + "epoch": 1.026354319180088, + "grad_norm": 0.28487309366468816, + "learning_rate": 8.930265473713939e-06, + "loss": 1.6133, + "step": 233 + }, + { + "epoch": 1.0307467057101025, + "grad_norm": 0.28141744313455036, + "learning_rate": 8.918922114834156e-06, + "loss": 1.589, + "step": 234 + }, + { + "epoch": 1.0351390922401171, + "grad_norm": 0.40308880749849646, + "learning_rate": 8.907526215803351e-06, + "loss": 1.6499, + "step": 235 + }, + { + "epoch": 1.0395314787701317, + "grad_norm": 0.2834055854316979, + "learning_rate": 8.896077929403901e-06, + "loss": 1.658, + "step": 236 + }, + { + "epoch": 1.0439238653001464, + "grad_norm": 0.2887035123274668, + "learning_rate": 8.884577409120535e-06, + "loss": 1.6196, + "step": 237 + }, + { + "epoch": 1.048316251830161, + "grad_norm": 0.2985213212589398, + "learning_rate": 8.873024809138272e-06, + "loss": 1.677, + "step": 238 + }, + { + "epoch": 1.0527086383601758, + "grad_norm": 0.2863532205939482, + "learning_rate": 8.861420284340352e-06, + "loss": 1.6682, + "step": 239 + }, + { + "epoch": 1.0571010248901904, + "grad_norm": 0.29235163398244823, + "learning_rate": 8.849763990306153e-06, + "loss": 1.6009, + "step": 240 + }, + { + "epoch": 1.061493411420205, + "grad_norm": 0.2899336107020558, + "learning_rate": 8.838056083309118e-06, + "loss": 1.6319, + "step": 241 + }, + { + "epoch": 1.0658857979502196, + "grad_norm": 0.3186837918366928, + "learning_rate": 8.826296720314658e-06, + "loss": 1.5875, + "step": 242 + }, + { + "epoch": 1.0702781844802343, + "grad_norm": 0.33938164798671433, + "learning_rate": 8.814486058978035e-06, + "loss": 1.6059, + "step": 243 + }, + { + "epoch": 1.0746705710102489, + "grad_norm": 0.2991115627129074, + "learning_rate": 8.802624257642262e-06, + "loss": 1.6599, + "step": 244 + }, + { + "epoch": 1.0790629575402635, + "grad_norm": 0.28608565865487656, + "learning_rate": 8.790711475335971e-06, + "loss": 1.6364, + "step": 245 + }, + { + "epoch": 1.083455344070278, + "grad_norm": 0.28359019806003394, + "learning_rate": 8.778747871771293e-06, + "loss": 1.6584, + "step": 246 + }, + { + "epoch": 1.0878477306002927, + "grad_norm": 0.34292617248780644, + "learning_rate": 8.766733607341697e-06, + "loss": 1.6613, + "step": 247 + }, + { + "epoch": 1.0922401171303076, + "grad_norm": 0.3469366526291968, + "learning_rate": 8.754668843119865e-06, + "loss": 1.6542, + "step": 248 + }, + { + "epoch": 1.0966325036603222, + "grad_norm": 0.2745752678931537, + "learning_rate": 8.742553740855507e-06, + "loss": 1.6723, + "step": 249 + }, + { + "epoch": 1.1010248901903368, + "grad_norm": 0.3863733971251328, + "learning_rate": 8.730388462973209e-06, + "loss": 1.6268, + "step": 250 + }, + { + "epoch": 1.1054172767203514, + "grad_norm": 0.3196278576328997, + "learning_rate": 8.718173172570254e-06, + "loss": 1.6349, + "step": 251 + }, + { + "epoch": 1.109809663250366, + "grad_norm": 0.27017487934331, + "learning_rate": 8.705908033414426e-06, + "loss": 1.606, + "step": 252 + }, + { + "epoch": 1.1142020497803806, + "grad_norm": 0.6576768607622604, + "learning_rate": 8.693593209941826e-06, + "loss": 1.5936, + "step": 253 + }, + { + "epoch": 1.1185944363103952, + "grad_norm": 0.30260640047372234, + "learning_rate": 8.681228867254657e-06, + "loss": 1.5948, + "step": 254 + }, + { + "epoch": 1.12298682284041, + "grad_norm": 0.3102239482584757, + "learning_rate": 8.66881517111902e-06, + "loss": 1.6928, + "step": 255 + }, + { + "epoch": 1.1273792093704247, + "grad_norm": 0.2818931187575453, + "learning_rate": 8.656352287962687e-06, + "loss": 1.687, + "step": 256 + }, + { + "epoch": 1.1317715959004393, + "grad_norm": 0.30007928577375903, + "learning_rate": 8.643840384872865e-06, + "loss": 1.6628, + "step": 257 + }, + { + "epoch": 1.136163982430454, + "grad_norm": 0.3249258384227434, + "learning_rate": 8.631279629593968e-06, + "loss": 1.6123, + "step": 258 + }, + { + "epoch": 1.1405563689604685, + "grad_norm": 0.2871446068255736, + "learning_rate": 8.61867019052535e-06, + "loss": 1.6608, + "step": 259 + }, + { + "epoch": 1.1449487554904831, + "grad_norm": 0.3001183609451796, + "learning_rate": 8.606012236719073e-06, + "loss": 1.6405, + "step": 260 + }, + { + "epoch": 1.1493411420204978, + "grad_norm": 0.271155672303218, + "learning_rate": 8.593305937877614e-06, + "loss": 1.5819, + "step": 261 + }, + { + "epoch": 1.1537335285505124, + "grad_norm": 0.30999460151747915, + "learning_rate": 8.580551464351603e-06, + "loss": 1.6378, + "step": 262 + }, + { + "epoch": 1.158125915080527, + "grad_norm": 0.2979330701414515, + "learning_rate": 8.567748987137544e-06, + "loss": 1.7342, + "step": 263 + }, + { + "epoch": 1.1625183016105418, + "grad_norm": 0.29037637226651547, + "learning_rate": 8.554898677875509e-06, + "loss": 1.663, + "step": 264 + }, + { + "epoch": 1.1669106881405564, + "grad_norm": 0.29574080692093535, + "learning_rate": 8.542000708846851e-06, + "loss": 1.6937, + "step": 265 + }, + { + "epoch": 1.171303074670571, + "grad_norm": 0.2867463319690563, + "learning_rate": 8.52905525297188e-06, + "loss": 1.5812, + "step": 266 + }, + { + "epoch": 1.1756954612005857, + "grad_norm": 0.3075848284440638, + "learning_rate": 8.516062483807556e-06, + "loss": 1.6421, + "step": 267 + }, + { + "epoch": 1.1800878477306003, + "grad_norm": 0.2745442718449274, + "learning_rate": 8.503022575545159e-06, + "loss": 1.6592, + "step": 268 + }, + { + "epoch": 1.1844802342606149, + "grad_norm": 0.4008583979349149, + "learning_rate": 8.48993570300795e-06, + "loss": 1.6052, + "step": 269 + }, + { + "epoch": 1.1888726207906295, + "grad_norm": 0.5183284694523707, + "learning_rate": 8.476802041648832e-06, + "loss": 1.7352, + "step": 270 + }, + { + "epoch": 1.1932650073206443, + "grad_norm": 0.30560643793660064, + "learning_rate": 8.463621767547998e-06, + "loss": 1.5592, + "step": 271 + }, + { + "epoch": 1.197657393850659, + "grad_norm": 0.5272799941494408, + "learning_rate": 8.450395057410561e-06, + "loss": 1.5471, + "step": 272 + }, + { + "epoch": 1.2020497803806736, + "grad_norm": 0.4281815470371588, + "learning_rate": 8.437122088564197e-06, + "loss": 1.6445, + "step": 273 + }, + { + "epoch": 1.2064421669106882, + "grad_norm": 0.27224728594527714, + "learning_rate": 8.423803038956763e-06, + "loss": 1.6629, + "step": 274 + }, + { + "epoch": 1.2108345534407028, + "grad_norm": 0.3269535080116159, + "learning_rate": 8.41043808715391e-06, + "loss": 1.5722, + "step": 275 + }, + { + "epoch": 1.2152269399707174, + "grad_norm": 0.3054371244574999, + "learning_rate": 8.39702741233669e-06, + "loss": 1.6173, + "step": 276 + }, + { + "epoch": 1.219619326500732, + "grad_norm": 0.36010851057708126, + "learning_rate": 8.383571194299154e-06, + "loss": 1.596, + "step": 277 + }, + { + "epoch": 1.2240117130307466, + "grad_norm": 0.3803533141127172, + "learning_rate": 8.37006961344594e-06, + "loss": 1.6229, + "step": 278 + }, + { + "epoch": 1.2284040995607612, + "grad_norm": 0.28612951080269505, + "learning_rate": 8.356522850789852e-06, + "loss": 1.5549, + "step": 279 + }, + { + "epoch": 1.232796486090776, + "grad_norm": 0.2855964904759227, + "learning_rate": 8.342931087949446e-06, + "loss": 1.6512, + "step": 280 + }, + { + "epoch": 1.2371888726207907, + "grad_norm": 0.3054377930088084, + "learning_rate": 8.32929450714658e-06, + "loss": 1.6274, + "step": 281 + }, + { + "epoch": 1.2415812591508053, + "grad_norm": 0.29343763848165016, + "learning_rate": 8.315613291203977e-06, + "loss": 1.6372, + "step": 282 + }, + { + "epoch": 1.24597364568082, + "grad_norm": 0.40870431747557984, + "learning_rate": 8.301887623542774e-06, + "loss": 1.6667, + "step": 283 + }, + { + "epoch": 1.2503660322108345, + "grad_norm": 0.30881819618442474, + "learning_rate": 8.288117688180064e-06, + "loss": 1.5547, + "step": 284 + }, + { + "epoch": 1.2547584187408491, + "grad_norm": 0.28765815508847925, + "learning_rate": 8.274303669726427e-06, + "loss": 1.6097, + "step": 285 + }, + { + "epoch": 1.2591508052708638, + "grad_norm": 0.33007103881402516, + "learning_rate": 8.260445753383453e-06, + "loss": 1.7016, + "step": 286 + }, + { + "epoch": 1.2635431918008786, + "grad_norm": 0.3628853649167158, + "learning_rate": 8.246544124941266e-06, + "loss": 1.6086, + "step": 287 + }, + { + "epoch": 1.267935578330893, + "grad_norm": 0.40134140387051037, + "learning_rate": 8.232598970776027e-06, + "loss": 1.6789, + "step": 288 + }, + { + "epoch": 1.2723279648609078, + "grad_norm": 0.2920563442488122, + "learning_rate": 8.218610477847435e-06, + "loss": 1.5915, + "step": 289 + }, + { + "epoch": 1.2767203513909224, + "grad_norm": 0.28530665366941144, + "learning_rate": 8.204578833696228e-06, + "loss": 1.5949, + "step": 290 + }, + { + "epoch": 1.281112737920937, + "grad_norm": 0.3155789127902478, + "learning_rate": 8.190504226441654e-06, + "loss": 1.639, + "step": 291 + }, + { + "epoch": 1.2855051244509517, + "grad_norm": 0.3199350667177758, + "learning_rate": 8.176386844778969e-06, + "loss": 1.6818, + "step": 292 + }, + { + "epoch": 1.2898975109809663, + "grad_norm": 0.34596043922632713, + "learning_rate": 8.162226877976886e-06, + "loss": 1.6604, + "step": 293 + }, + { + "epoch": 1.294289897510981, + "grad_norm": 0.32781624288061406, + "learning_rate": 8.148024515875057e-06, + "loss": 1.5971, + "step": 294 + }, + { + "epoch": 1.2986822840409955, + "grad_norm": 0.5927243317332561, + "learning_rate": 8.133779948881514e-06, + "loss": 1.6574, + "step": 295 + }, + { + "epoch": 1.3030746705710103, + "grad_norm": 0.296769298222996, + "learning_rate": 8.11949336797012e-06, + "loss": 1.5505, + "step": 296 + }, + { + "epoch": 1.307467057101025, + "grad_norm": 0.28849112804215943, + "learning_rate": 8.105164964678009e-06, + "loss": 1.6729, + "step": 297 + }, + { + "epoch": 1.3118594436310396, + "grad_norm": 0.3286168384006033, + "learning_rate": 8.090794931103026e-06, + "loss": 1.5949, + "step": 298 + }, + { + "epoch": 1.3162518301610542, + "grad_norm": 0.31185844876531665, + "learning_rate": 8.076383459901138e-06, + "loss": 1.6918, + "step": 299 + }, + { + "epoch": 1.3206442166910688, + "grad_norm": 0.2789921585419944, + "learning_rate": 8.061930744283855e-06, + "loss": 1.6437, + "step": 300 + }, + { + "epoch": 1.3250366032210834, + "grad_norm": 0.30165508316706874, + "learning_rate": 8.04743697801565e-06, + "loss": 1.6555, + "step": 301 + }, + { + "epoch": 1.329428989751098, + "grad_norm": 0.2989059936488396, + "learning_rate": 8.032902355411345e-06, + "loss": 1.5141, + "step": 302 + }, + { + "epoch": 1.3338213762811129, + "grad_norm": 0.29928394703560823, + "learning_rate": 8.018327071333521e-06, + "loss": 1.5859, + "step": 303 + }, + { + "epoch": 1.3382137628111272, + "grad_norm": 0.29795453570714703, + "learning_rate": 8.003711321189895e-06, + "loss": 1.5864, + "step": 304 + }, + { + "epoch": 1.342606149341142, + "grad_norm": 0.30765817351768504, + "learning_rate": 7.989055300930704e-06, + "loss": 1.5743, + "step": 305 + }, + { + "epoch": 1.3469985358711567, + "grad_norm": 0.43955554329120444, + "learning_rate": 7.974359207046081e-06, + "loss": 1.6423, + "step": 306 + }, + { + "epoch": 1.3513909224011713, + "grad_norm": 0.37865307469865994, + "learning_rate": 7.959623236563412e-06, + "loss": 1.6102, + "step": 307 + }, + { + "epoch": 1.355783308931186, + "grad_norm": 0.3074090057367007, + "learning_rate": 7.944847587044704e-06, + "loss": 1.6602, + "step": 308 + }, + { + "epoch": 1.3601756954612005, + "grad_norm": 0.3361817362879421, + "learning_rate": 7.930032456583931e-06, + "loss": 1.6704, + "step": 309 + }, + { + "epoch": 1.3645680819912152, + "grad_norm": 0.2960444587212274, + "learning_rate": 7.915178043804382e-06, + "loss": 1.5536, + "step": 310 + }, + { + "epoch": 1.3689604685212298, + "grad_norm": 0.31080078465435335, + "learning_rate": 7.900284547855992e-06, + "loss": 1.659, + "step": 311 + }, + { + "epoch": 1.3733528550512446, + "grad_norm": 0.29259208311732576, + "learning_rate": 7.885352168412677e-06, + "loss": 1.6324, + "step": 312 + }, + { + "epoch": 1.3777452415812592, + "grad_norm": 0.40704907971444493, + "learning_rate": 7.870381105669657e-06, + "loss": 1.6954, + "step": 313 + }, + { + "epoch": 1.3821376281112738, + "grad_norm": 0.2884678272243352, + "learning_rate": 7.85537156034077e-06, + "loss": 1.6911, + "step": 314 + }, + { + "epoch": 1.3865300146412884, + "grad_norm": 0.3002277679918279, + "learning_rate": 7.84032373365578e-06, + "loss": 1.6384, + "step": 315 + }, + { + "epoch": 1.390922401171303, + "grad_norm": 0.29747656531257255, + "learning_rate": 7.825237827357684e-06, + "loss": 1.5728, + "step": 316 + }, + { + "epoch": 1.3953147877013177, + "grad_norm": 0.2922330023166882, + "learning_rate": 7.810114043700002e-06, + "loss": 1.6876, + "step": 317 + }, + { + "epoch": 1.3997071742313323, + "grad_norm": 0.29216993729442, + "learning_rate": 7.794952585444068e-06, + "loss": 1.7112, + "step": 318 + }, + { + "epoch": 1.4040995607613471, + "grad_norm": 0.2841430060967501, + "learning_rate": 7.779753655856313e-06, + "loss": 1.6352, + "step": 319 + }, + { + "epoch": 1.4084919472913615, + "grad_norm": 0.29535174268674796, + "learning_rate": 7.764517458705536e-06, + "loss": 1.638, + "step": 320 + }, + { + "epoch": 1.4128843338213763, + "grad_norm": 0.3553267662730052, + "learning_rate": 7.749244198260175e-06, + "loss": 1.6262, + "step": 321 + }, + { + "epoch": 1.417276720351391, + "grad_norm": 0.2977673782348279, + "learning_rate": 7.733934079285569e-06, + "loss": 1.6115, + "step": 322 + }, + { + "epoch": 1.4216691068814056, + "grad_norm": 0.2822196746941408, + "learning_rate": 7.718587307041209e-06, + "loss": 1.6468, + "step": 323 + }, + { + "epoch": 1.4260614934114202, + "grad_norm": 0.2837671497162217, + "learning_rate": 7.703204087277989e-06, + "loss": 1.6474, + "step": 324 + }, + { + "epoch": 1.4304538799414348, + "grad_norm": 0.31664451334685184, + "learning_rate": 7.687784626235448e-06, + "loss": 1.5919, + "step": 325 + }, + { + "epoch": 1.4348462664714494, + "grad_norm": 0.33195017711571867, + "learning_rate": 7.672329130639007e-06, + "loss": 1.6941, + "step": 326 + }, + { + "epoch": 1.439238653001464, + "grad_norm": 0.2995108911659512, + "learning_rate": 7.656837807697187e-06, + "loss": 1.6393, + "step": 327 + }, + { + "epoch": 1.4436310395314789, + "grad_norm": 0.2949122695416032, + "learning_rate": 7.641310865098845e-06, + "loss": 1.5135, + "step": 328 + }, + { + "epoch": 1.4480234260614935, + "grad_norm": 0.31946384200509315, + "learning_rate": 7.625748511010382e-06, + "loss": 1.5784, + "step": 329 + }, + { + "epoch": 1.452415812591508, + "grad_norm": 0.29208814812765094, + "learning_rate": 7.610150954072953e-06, + "loss": 1.6794, + "step": 330 + }, + { + "epoch": 1.4568081991215227, + "grad_norm": 0.34475174012170284, + "learning_rate": 7.594518403399666e-06, + "loss": 1.5773, + "step": 331 + }, + { + "epoch": 1.4612005856515373, + "grad_norm": 0.2876304119955936, + "learning_rate": 7.578851068572788e-06, + "loss": 1.5613, + "step": 332 + }, + { + "epoch": 1.465592972181552, + "grad_norm": 0.31542092184200016, + "learning_rate": 7.563149159640929e-06, + "loss": 1.6086, + "step": 333 + }, + { + "epoch": 1.4699853587115665, + "grad_norm": 0.2964161628058562, + "learning_rate": 7.547412887116224e-06, + "loss": 1.5963, + "step": 334 + }, + { + "epoch": 1.4743777452415814, + "grad_norm": 0.32832262869958717, + "learning_rate": 7.531642461971515e-06, + "loss": 1.6054, + "step": 335 + }, + { + "epoch": 1.4787701317715958, + "grad_norm": 0.3209241382730098, + "learning_rate": 7.51583809563752e-06, + "loss": 1.6748, + "step": 336 + }, + { + "epoch": 1.4831625183016106, + "grad_norm": 0.30060262616835987, + "learning_rate": 7.500000000000001e-06, + "loss": 1.6081, + "step": 337 + }, + { + "epoch": 1.4875549048316252, + "grad_norm": 0.3500388762321464, + "learning_rate": 7.4841283873969194e-06, + "loss": 1.5906, + "step": 338 + }, + { + "epoch": 1.4919472913616398, + "grad_norm": 0.28308587203259633, + "learning_rate": 7.468223470615593e-06, + "loss": 1.5879, + "step": 339 + }, + { + "epoch": 1.4963396778916545, + "grad_norm": 0.2901927981331445, + "learning_rate": 7.452285462889841e-06, + "loss": 1.6242, + "step": 340 + }, + { + "epoch": 1.500732064421669, + "grad_norm": 0.3314307283598877, + "learning_rate": 7.436314577897126e-06, + "loss": 1.646, + "step": 341 + }, + { + "epoch": 1.505124450951684, + "grad_norm": 0.3264131109049365, + "learning_rate": 7.420311029755688e-06, + "loss": 1.5855, + "step": 342 + } + ], + "logging_steps": 1, + "max_steps": 908, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 114, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.039402133010514e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-342/training_args.bin b/checkpoint-342/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..282808fde3b579abe8666eba7a8b5732d5eaf3d4 --- /dev/null +++ b/checkpoint-342/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f598588716f36b5db7091989142a430a4fd275bce6f1e0d8e6c17f595272e1ec +size 9041 diff --git a/checkpoint-342/zero_to_fp32.py b/checkpoint-342/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint-342/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoint-456/README.md b/checkpoint-456/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb5cf17f38ede36f3fb46a89995daa3f44167898 --- /dev/null +++ b/checkpoint-456/README.md @@ -0,0 +1,202 @@ +--- +base_model: Gryphe/Codex-24B-Small-3.2 +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/checkpoint-456/adapter_config.json b/checkpoint-456/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa0535ca25a4eef78d9c51576f13234c64c95fb --- /dev/null +++ b/checkpoint-456/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Gryphe/Codex-24B-Small-3.2", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.5, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "o_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/checkpoint-456/adapter_model.safetensors b/checkpoint-456/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..616f2e32ea80e314971a1bc01f5a5c09d081d639 --- /dev/null +++ b/checkpoint-456/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34bb6bf04ffe52f23c7cfdec0bc6ad22e6401c8598a92966e7914ba84549146 +size 1478569432 diff --git a/checkpoint-456/global_step456/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-456/global_step456/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..446e94e1fab2a9041cd23ab25c24024e051902c8 --- /dev/null +++ b/checkpoint-456/global_step456/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ee05ef00b2fd406bc0e6c32170d56c52f20863b8320c9875833bcdffaf74bfd +size 4459517761 diff --git a/checkpoint-456/global_step456/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-456/global_step456/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..221040f3d03ece138cc92dce34467aa29aaae27c --- /dev/null +++ b/checkpoint-456/global_step456/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0726510e6de96d62c6bb2a9ddf7942828e3f0693f4e80f55b2e41b9e2848a12 +size 557481 diff --git a/checkpoint-456/latest b/checkpoint-456/latest new file mode 100644 index 0000000000000000000000000000000000000000..dbd5ff49aa710762c49b97ba3da2fe7861cf8ba3 --- /dev/null +++ b/checkpoint-456/latest @@ -0,0 +1 @@ +global_step456 \ No newline at end of file diff --git a/checkpoint-456/rng_state.pth b/checkpoint-456/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..f6725469039d622ac85cf6a581d83a7edb9aeae8 --- /dev/null +++ b/checkpoint-456/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4daf414073459f2c202276c64795b9b52b1f132919069846c2e24a1894697985 +size 14709 diff --git a/checkpoint-456/scheduler.pt b/checkpoint-456/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c8effb8eed5b26a672e1683a2b0f25879fd3c7b --- /dev/null +++ b/checkpoint-456/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42751c3fda6d7b54652fea9d3d096b4b4ee2931ec6653a079dfc22c1ebc40249 +size 1401 diff --git a/checkpoint-456/special_tokens_map.json b/checkpoint-456/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..4332a4c6d2a485806f55e02260db8a73698e11fb --- /dev/null +++ b/checkpoint-456/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-456/tokenizer.json b/checkpoint-456/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8fab5451b04e019d1612cc5088df62aee9a8208c --- /dev/null +++ b/checkpoint-456/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e5daf1e619597fd56ff036925ea189916417c665d5bdd61b80199cf8d4e6d2 +size 17078029 diff --git a/checkpoint-456/tokenizer_config.json b/checkpoint-456/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..814201fe59040b12e956fb649c749a42bdce1e2f --- /dev/null +++ b/checkpoint-456/tokenizer_config.json @@ -0,0 +1,8021 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "[IMG]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "[IMG_BREAK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "[IMG_END]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "[PREFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "[MIDDLE]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "[SUFFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "[SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "[/SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "[TOOL_CONTENT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "109": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "110": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "111": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "112": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "113": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "114": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "115": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "116": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "117": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "118": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "119": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "120": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "121": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "122": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "123": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "124": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "125": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "126": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "127": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "129": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "130": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "132": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "133": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "134": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "135": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "136": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "137": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "138": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "139": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "140": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "141": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "142": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "143": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "144": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "145": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "146": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "147": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "148": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "149": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "150": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "152": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "153": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "154": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "155": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "156": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "157": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "158": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "159": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "160": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "161": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "162": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "164": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "165": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "166": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "167": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "185": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "186": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "187": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "188": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "189": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "190": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "191": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "222": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "223": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "224": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "225": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "226": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "227": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "228": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "229": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "230": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "231": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "232": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "233": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "234": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "235": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "236": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "237": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "238": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "239": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "240": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "241": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "242": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "243": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "244": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "245": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "246": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "247": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "249": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "251": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "252": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "253": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "254": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "255": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "257": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "258": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "259": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "260": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "262": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "263": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "264": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "265": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "266": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "267": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "268": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "285": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "286": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "287": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "288": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "289": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "290": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "291": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "292": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "293": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "294": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "295": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "296": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "297": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "298": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "299": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "300": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "301": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "302": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "303": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "304": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "305": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "306": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "307": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "308": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "309": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "310": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "311": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "312": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "313": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "314": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "315": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "316": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "317": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "318": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "319": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "320": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "321": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "322": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "323": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "324": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "325": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "326": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "327": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "328": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "329": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "330": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "331": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "332": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "335": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "336": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "337": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "338": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "339": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "340": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "341": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "342": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "343": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "344": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "345": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "346": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "347": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "348": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "349": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "360": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "361": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "362": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "363": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "364": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "365": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "366": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "367": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "368": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "369": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "370": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "371": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "372": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "373": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "374": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "375": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "376": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "377": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "378": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "379": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "380": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "381": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "382": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "383": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "384": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "385": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "386": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "387": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "388": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "389": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "390": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "391": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "392": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "393": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "394": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "395": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "396": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "397": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "398": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "399": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "400": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "401": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "402": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "403": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "404": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "405": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "406": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "407": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "408": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "409": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "410": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "411": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "412": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "413": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "414": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "415": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "416": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "417": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "418": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "419": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "420": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "421": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "422": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "423": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "424": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "425": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "426": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "427": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "428": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "429": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "430": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "431": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "432": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "433": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "434": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "435": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "436": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "437": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "438": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "439": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "440": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "441": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "442": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "443": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "444": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "445": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "446": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "447": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "448": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "449": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "450": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "451": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "452": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "453": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "454": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "455": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "456": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "457": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "458": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "459": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "460": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "461": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "462": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "463": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "464": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "465": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "466": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "467": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "468": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "469": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "470": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "471": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "472": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "473": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "474": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "475": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "476": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "477": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "478": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "479": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "480": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "481": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "482": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "483": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "484": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "485": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "486": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "487": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "488": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "489": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "490": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "491": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "492": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "493": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "494": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "495": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "496": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "497": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "498": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "499": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "500": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "501": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "502": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "503": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "504": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "505": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "506": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "507": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "508": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "509": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "510": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "511": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "512": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "513": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "514": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "515": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "516": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "517": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "518": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "519": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "520": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "521": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "522": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "523": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "524": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "525": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "526": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "527": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "528": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "529": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "530": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "531": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "532": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "533": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "534": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "535": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "536": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "537": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "538": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "539": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "540": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "541": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "542": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "543": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "544": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "545": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "546": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "547": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "548": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "549": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "550": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "551": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "552": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "553": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "554": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "555": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "556": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "557": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "558": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "559": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "560": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "561": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "562": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "563": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "564": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "565": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "566": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "567": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "568": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "569": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "570": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "571": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "572": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "573": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "574": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "575": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "576": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "577": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "578": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "579": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "580": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "581": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "582": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "583": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "584": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "585": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "586": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "587": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "588": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "589": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "590": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "591": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "592": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "593": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "594": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "595": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "596": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "597": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "598": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "599": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "600": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "601": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "602": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "603": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "604": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "605": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "606": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "607": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "608": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "609": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "610": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "611": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "612": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "613": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "614": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "615": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "616": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "617": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "618": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "619": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "620": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "621": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "622": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "623": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "624": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "625": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "626": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "627": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "628": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "629": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "630": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "631": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "632": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "633": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "634": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "635": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "636": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "637": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "638": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "639": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "640": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "641": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "642": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "643": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "644": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "645": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "646": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "647": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "650": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "651": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "652": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "653": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "654": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "655": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "656": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "659": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "660": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "661": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "662": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "663": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "664": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "675": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "676": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "677": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "678": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "679": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "680": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "681": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "682": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "683": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "684": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "685": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "686": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "687": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "688": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "689": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "690": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "691": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "692": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "693": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "694": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "695": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "696": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "697": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "698": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "699": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "700": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "701": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "702": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "703": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "704": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "705": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "706": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "707": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "708": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "709": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "710": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "711": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "712": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "713": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "714": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "715": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "716": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "717": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "718": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "719": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "720": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "721": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "722": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "723": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "724": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "725": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "726": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "727": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "728": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "729": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "730": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "731": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "732": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "733": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "734": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "735": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "736": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "737": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "738": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "739": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "740": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "741": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "742": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "743": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "744": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "745": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "746": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "747": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "748": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "749": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "750": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "751": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "752": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "753": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "754": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "755": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "756": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "757": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "758": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "759": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "760": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "761": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "762": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "763": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "764": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "765": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "766": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "767": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "768": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "769": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "770": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "771": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "772": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "773": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "774": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "775": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "776": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "777": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "778": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "779": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "780": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "781": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "782": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "783": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "784": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "785": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "786": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "787": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "788": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "789": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "790": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "791": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "792": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "793": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "794": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "795": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "796": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "797": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "798": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "799": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "800": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "801": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "802": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "803": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "804": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "805": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "806": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "807": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "808": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "809": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "810": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "811": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "812": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "813": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "814": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "815": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "816": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "817": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "818": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "819": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "820": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "821": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "822": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "823": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "824": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "825": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "826": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "827": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "828": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "829": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "830": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "831": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "832": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "833": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "834": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "835": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "836": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "837": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "838": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "839": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "840": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "841": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "842": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "843": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "844": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "845": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "846": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "847": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "848": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "849": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "850": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "851": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "852": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "853": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "854": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "855": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "856": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "857": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "863": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "864": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "865": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "866": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "867": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "868": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "869": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "870": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "871": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "880": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "881": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "882": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "883": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "884": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "885": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "886": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "887": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "888": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "889": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "890": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "891": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "892": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "893": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "894": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "895": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "896": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "897": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "898": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "899": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "900": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "901": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "902": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "903": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "904": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "905": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "906": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "907": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "908": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "909": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "910": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "911": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "912": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "913": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "914": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "915": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "916": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "917": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "918": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "919": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "920": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "921": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "922": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "923": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "924": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "925": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "926": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "927": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "928": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "929": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "930": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "931": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "932": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "933": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "934": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "935": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "936": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "937": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "938": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "939": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "940": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "941": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "942": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "943": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "944": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "945": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "946": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "947": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "948": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "949": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "950": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "951": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "952": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "953": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "954": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "955": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "956": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "957": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "958": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "959": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "960": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "961": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "962": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "963": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "964": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "965": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "966": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "967": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "968": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "969": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "970": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "971": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "972": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "973": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "974": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "975": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "976": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "977": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "978": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "979": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "980": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "981": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "982": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "983": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "984": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "985": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "986": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "987": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "988": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "989": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "990": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "991": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "992": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "993": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "994": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "995": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "996": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "997": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "998": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "999": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 131072, + "pad_token": "", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-456/trainer_state.json b/checkpoint-456/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2d999c85609d6fd806608b417c8291dcfbddd2aa --- /dev/null +++ b/checkpoint-456/trainer_state.json @@ -0,0 +1,3226 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0087847730600292, + "eval_steps": 500, + "global_step": 456, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004392386530014641, + "grad_norm": 2.023870796411983, + "learning_rate": 0.0, + "loss": 1.8021, + "step": 1 + }, + { + "epoch": 0.008784773060029283, + "grad_norm": 2.0723948799697984, + "learning_rate": 2.0000000000000002e-07, + "loss": 1.8796, + "step": 2 + }, + { + "epoch": 0.013177159590043924, + "grad_norm": 2.0322569672979474, + "learning_rate": 4.0000000000000003e-07, + "loss": 1.855, + "step": 3 + }, + { + "epoch": 0.017569546120058566, + "grad_norm": 2.111472782868084, + "learning_rate": 6.000000000000001e-07, + "loss": 1.8545, + "step": 4 + }, + { + "epoch": 0.021961932650073207, + "grad_norm": 2.096607607233168, + "learning_rate": 8.000000000000001e-07, + "loss": 1.807, + "step": 5 + }, + { + "epoch": 0.02635431918008785, + "grad_norm": 2.0771855831079518, + "learning_rate": 1.0000000000000002e-06, + "loss": 1.8208, + "step": 6 + }, + { + "epoch": 0.03074670571010249, + "grad_norm": 2.0341840619969567, + "learning_rate": 1.2000000000000002e-06, + "loss": 1.8754, + "step": 7 + }, + { + "epoch": 0.03513909224011713, + "grad_norm": 2.0564329237094356, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.8882, + "step": 8 + }, + { + "epoch": 0.03953147877013177, + "grad_norm": 1.9900074702634307, + "learning_rate": 1.6000000000000001e-06, + "loss": 1.8733, + "step": 9 + }, + { + "epoch": 0.043923865300146414, + "grad_norm": 1.895599415585885, + "learning_rate": 1.8000000000000001e-06, + "loss": 1.8474, + "step": 10 + }, + { + "epoch": 0.048316251830161056, + "grad_norm": 1.9304209419460994, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.884, + "step": 11 + }, + { + "epoch": 0.0527086383601757, + "grad_norm": 1.6511439041472846, + "learning_rate": 2.2e-06, + "loss": 1.8452, + "step": 12 + }, + { + "epoch": 0.05710102489019034, + "grad_norm": 1.6727753891125912, + "learning_rate": 2.4000000000000003e-06, + "loss": 1.8632, + "step": 13 + }, + { + "epoch": 0.06149341142020498, + "grad_norm": 1.6571693399048197, + "learning_rate": 2.6e-06, + "loss": 1.8419, + "step": 14 + }, + { + "epoch": 0.06588579795021962, + "grad_norm": 1.5588712723347087, + "learning_rate": 2.8000000000000003e-06, + "loss": 1.8163, + "step": 15 + }, + { + "epoch": 0.07027818448023426, + "grad_norm": 1.57706405792705, + "learning_rate": 3e-06, + "loss": 1.7808, + "step": 16 + }, + { + "epoch": 0.0746705710102489, + "grad_norm": 1.4841447475095337, + "learning_rate": 3.2000000000000003e-06, + "loss": 1.7854, + "step": 17 + }, + { + "epoch": 0.07906295754026355, + "grad_norm": 1.2785291394155553, + "learning_rate": 3.4000000000000005e-06, + "loss": 1.7531, + "step": 18 + }, + { + "epoch": 0.08345534407027819, + "grad_norm": 1.1759630521885338, + "learning_rate": 3.6000000000000003e-06, + "loss": 1.8185, + "step": 19 + }, + { + "epoch": 0.08784773060029283, + "grad_norm": 1.1188613513306425, + "learning_rate": 3.8000000000000005e-06, + "loss": 1.849, + "step": 20 + }, + { + "epoch": 0.09224011713030747, + "grad_norm": 1.008741263333265, + "learning_rate": 4.000000000000001e-06, + "loss": 1.775, + "step": 21 + }, + { + "epoch": 0.09663250366032211, + "grad_norm": 0.9240991279618896, + "learning_rate": 4.2000000000000004e-06, + "loss": 1.8253, + "step": 22 + }, + { + "epoch": 0.10102489019033675, + "grad_norm": 0.9015845652170666, + "learning_rate": 4.4e-06, + "loss": 1.7967, + "step": 23 + }, + { + "epoch": 0.1054172767203514, + "grad_norm": 0.8007281301128234, + "learning_rate": 4.600000000000001e-06, + "loss": 1.7958, + "step": 24 + }, + { + "epoch": 0.10980966325036604, + "grad_norm": 0.7263390923205475, + "learning_rate": 4.800000000000001e-06, + "loss": 1.7744, + "step": 25 + }, + { + "epoch": 0.11420204978038068, + "grad_norm": 0.6527771074756341, + "learning_rate": 5e-06, + "loss": 1.7502, + "step": 26 + }, + { + "epoch": 0.11859443631039532, + "grad_norm": 0.5943189060332841, + "learning_rate": 5.2e-06, + "loss": 1.7378, + "step": 27 + }, + { + "epoch": 0.12298682284040996, + "grad_norm": 0.5426800655727773, + "learning_rate": 5.400000000000001e-06, + "loss": 1.7372, + "step": 28 + }, + { + "epoch": 0.1273792093704246, + "grad_norm": 0.46887045607953654, + "learning_rate": 5.600000000000001e-06, + "loss": 1.7141, + "step": 29 + }, + { + "epoch": 0.13177159590043924, + "grad_norm": 0.4408809706079413, + "learning_rate": 5.8e-06, + "loss": 1.6977, + "step": 30 + }, + { + "epoch": 0.13616398243045388, + "grad_norm": 0.4867065604810947, + "learning_rate": 6e-06, + "loss": 1.7379, + "step": 31 + }, + { + "epoch": 0.14055636896046853, + "grad_norm": 0.40882779662256746, + "learning_rate": 6.200000000000001e-06, + "loss": 1.8064, + "step": 32 + }, + { + "epoch": 0.14494875549048317, + "grad_norm": 0.4363561706655908, + "learning_rate": 6.4000000000000006e-06, + "loss": 1.6835, + "step": 33 + }, + { + "epoch": 0.1493411420204978, + "grad_norm": 0.4443411222267533, + "learning_rate": 6.600000000000001e-06, + "loss": 1.7464, + "step": 34 + }, + { + "epoch": 0.15373352855051245, + "grad_norm": 0.4519702758206936, + "learning_rate": 6.800000000000001e-06, + "loss": 1.6891, + "step": 35 + }, + { + "epoch": 0.1581259150805271, + "grad_norm": 0.4174999907173025, + "learning_rate": 7e-06, + "loss": 1.7262, + "step": 36 + }, + { + "epoch": 0.16251830161054173, + "grad_norm": 0.46742969885702906, + "learning_rate": 7.2000000000000005e-06, + "loss": 1.7608, + "step": 37 + }, + { + "epoch": 0.16691068814055637, + "grad_norm": 0.4155692294314683, + "learning_rate": 7.4e-06, + "loss": 1.7211, + "step": 38 + }, + { + "epoch": 0.17130307467057102, + "grad_norm": 0.4123169421407387, + "learning_rate": 7.600000000000001e-06, + "loss": 1.6956, + "step": 39 + }, + { + "epoch": 0.17569546120058566, + "grad_norm": 0.3887606381731466, + "learning_rate": 7.800000000000002e-06, + "loss": 1.7391, + "step": 40 + }, + { + "epoch": 0.1800878477306003, + "grad_norm": 0.37535703391318914, + "learning_rate": 8.000000000000001e-06, + "loss": 1.751, + "step": 41 + }, + { + "epoch": 0.18448023426061494, + "grad_norm": 0.48621902751310403, + "learning_rate": 8.2e-06, + "loss": 1.7091, + "step": 42 + }, + { + "epoch": 0.18887262079062958, + "grad_norm": 0.39145172998871297, + "learning_rate": 8.400000000000001e-06, + "loss": 1.7764, + "step": 43 + }, + { + "epoch": 0.19326500732064422, + "grad_norm": 0.3735488902443606, + "learning_rate": 8.6e-06, + "loss": 1.7058, + "step": 44 + }, + { + "epoch": 0.19765739385065886, + "grad_norm": 0.33988576479414434, + "learning_rate": 8.8e-06, + "loss": 1.7305, + "step": 45 + }, + { + "epoch": 0.2020497803806735, + "grad_norm": 0.9733163465798257, + "learning_rate": 9e-06, + "loss": 1.7567, + "step": 46 + }, + { + "epoch": 0.20644216691068815, + "grad_norm": 0.3344321176956884, + "learning_rate": 9.200000000000002e-06, + "loss": 1.7277, + "step": 47 + }, + { + "epoch": 0.2108345534407028, + "grad_norm": 0.3647805237752918, + "learning_rate": 9.4e-06, + "loss": 1.7197, + "step": 48 + }, + { + "epoch": 0.21522693997071743, + "grad_norm": 0.36967036325253694, + "learning_rate": 9.600000000000001e-06, + "loss": 1.6994, + "step": 49 + }, + { + "epoch": 0.21961932650073207, + "grad_norm": 0.32386928338357696, + "learning_rate": 9.800000000000001e-06, + "loss": 1.6975, + "step": 50 + }, + { + "epoch": 0.2240117130307467, + "grad_norm": 0.33940794357965215, + "learning_rate": 1e-05, + "loss": 1.7278, + "step": 51 + }, + { + "epoch": 0.22840409956076135, + "grad_norm": 0.3396058372359069, + "learning_rate": 9.999966483034437e-06, + "loss": 1.6342, + "step": 52 + }, + { + "epoch": 0.232796486090776, + "grad_norm": 0.3395876900960321, + "learning_rate": 9.999865932587107e-06, + "loss": 1.6846, + "step": 53 + }, + { + "epoch": 0.23718887262079064, + "grad_norm": 0.3320830942903407, + "learning_rate": 9.999698350006063e-06, + "loss": 1.6744, + "step": 54 + }, + { + "epoch": 0.24158125915080528, + "grad_norm": 0.318642304519145, + "learning_rate": 9.999463737538054e-06, + "loss": 1.6633, + "step": 55 + }, + { + "epoch": 0.24597364568081992, + "grad_norm": 0.3302002388842341, + "learning_rate": 9.999162098328475e-06, + "loss": 1.6994, + "step": 56 + }, + { + "epoch": 0.25036603221083453, + "grad_norm": 0.30428798634415904, + "learning_rate": 9.998793436421342e-06, + "loss": 1.6728, + "step": 57 + }, + { + "epoch": 0.2547584187408492, + "grad_norm": 0.30662673120989614, + "learning_rate": 9.998357756759223e-06, + "loss": 1.6283, + "step": 58 + }, + { + "epoch": 0.2591508052708638, + "grad_norm": 0.34305953470241257, + "learning_rate": 9.997855065183185e-06, + "loss": 1.7269, + "step": 59 + }, + { + "epoch": 0.2635431918008785, + "grad_norm": 0.29647698247102616, + "learning_rate": 9.997285368432704e-06, + "loss": 1.6108, + "step": 60 + }, + { + "epoch": 0.2679355783308931, + "grad_norm": 0.30904144706299363, + "learning_rate": 9.996648674145584e-06, + "loss": 1.5925, + "step": 61 + }, + { + "epoch": 0.27232796486090777, + "grad_norm": 0.28294691041282544, + "learning_rate": 9.995944990857848e-06, + "loss": 1.7004, + "step": 62 + }, + { + "epoch": 0.2767203513909224, + "grad_norm": 0.4537316624659888, + "learning_rate": 9.99517432800363e-06, + "loss": 1.673, + "step": 63 + }, + { + "epoch": 0.28111273792093705, + "grad_norm": 0.33334142649617554, + "learning_rate": 9.994336695915041e-06, + "loss": 1.7305, + "step": 64 + }, + { + "epoch": 0.28550512445095166, + "grad_norm": 0.2906595328556503, + "learning_rate": 9.993432105822034e-06, + "loss": 1.6511, + "step": 65 + }, + { + "epoch": 0.28989751098096633, + "grad_norm": 0.30937353104293036, + "learning_rate": 9.992460569852256e-06, + "loss": 1.7055, + "step": 66 + }, + { + "epoch": 0.29428989751098095, + "grad_norm": 0.29714918983678074, + "learning_rate": 9.99142210103088e-06, + "loss": 1.6022, + "step": 67 + }, + { + "epoch": 0.2986822840409956, + "grad_norm": 0.3077061364082642, + "learning_rate": 9.99031671328044e-06, + "loss": 1.6167, + "step": 68 + }, + { + "epoch": 0.30307467057101023, + "grad_norm": 0.31797999416909106, + "learning_rate": 9.98914442142063e-06, + "loss": 1.6063, + "step": 69 + }, + { + "epoch": 0.3074670571010249, + "grad_norm": 0.28544810434107143, + "learning_rate": 9.987905241168117e-06, + "loss": 1.673, + "step": 70 + }, + { + "epoch": 0.3118594436310395, + "grad_norm": 0.3113516293182134, + "learning_rate": 9.986599189136325e-06, + "loss": 1.6759, + "step": 71 + }, + { + "epoch": 0.3162518301610542, + "grad_norm": 0.32958924621680397, + "learning_rate": 9.985226282835216e-06, + "loss": 1.6894, + "step": 72 + }, + { + "epoch": 0.3206442166910688, + "grad_norm": 0.3189944736085633, + "learning_rate": 9.983786540671052e-06, + "loss": 1.7762, + "step": 73 + }, + { + "epoch": 0.32503660322108346, + "grad_norm": 0.30229970638681786, + "learning_rate": 9.982279981946143e-06, + "loss": 1.709, + "step": 74 + }, + { + "epoch": 0.3294289897510981, + "grad_norm": 0.308085411523194, + "learning_rate": 9.980706626858607e-06, + "loss": 1.6741, + "step": 75 + }, + { + "epoch": 0.33382137628111275, + "grad_norm": 0.2851270169881767, + "learning_rate": 9.979066496502075e-06, + "loss": 1.656, + "step": 76 + }, + { + "epoch": 0.33821376281112736, + "grad_norm": 0.3115977673015286, + "learning_rate": 9.977359612865424e-06, + "loss": 1.6633, + "step": 77 + }, + { + "epoch": 0.34260614934114203, + "grad_norm": 0.2960787226189042, + "learning_rate": 9.97558599883248e-06, + "loss": 1.673, + "step": 78 + }, + { + "epoch": 0.34699853587115664, + "grad_norm": 0.35701812130442995, + "learning_rate": 9.973745678181705e-06, + "loss": 1.6103, + "step": 79 + }, + { + "epoch": 0.3513909224011713, + "grad_norm": 0.367868955812032, + "learning_rate": 9.971838675585888e-06, + "loss": 1.6768, + "step": 80 + }, + { + "epoch": 0.3557833089311859, + "grad_norm": 0.3295428259298103, + "learning_rate": 9.9698650166118e-06, + "loss": 1.6668, + "step": 81 + }, + { + "epoch": 0.3601756954612006, + "grad_norm": 0.3242883305567894, + "learning_rate": 9.96782472771987e-06, + "loss": 1.7152, + "step": 82 + }, + { + "epoch": 0.3645680819912152, + "grad_norm": 0.3070622417895325, + "learning_rate": 9.965717836263813e-06, + "loss": 1.5697, + "step": 83 + }, + { + "epoch": 0.3689604685212299, + "grad_norm": 0.27753629978682975, + "learning_rate": 9.96354437049027e-06, + "loss": 1.6453, + "step": 84 + }, + { + "epoch": 0.3733528550512445, + "grad_norm": 0.2723902519890988, + "learning_rate": 9.961304359538437e-06, + "loss": 1.7165, + "step": 85 + }, + { + "epoch": 0.37774524158125916, + "grad_norm": 0.29061586618994084, + "learning_rate": 9.958997833439658e-06, + "loss": 1.7135, + "step": 86 + }, + { + "epoch": 0.3821376281112738, + "grad_norm": 0.33085661251847487, + "learning_rate": 9.956624823117036e-06, + "loss": 1.6109, + "step": 87 + }, + { + "epoch": 0.38653001464128844, + "grad_norm": 0.2931612046141165, + "learning_rate": 9.954185360385013e-06, + "loss": 1.7048, + "step": 88 + }, + { + "epoch": 0.39092240117130306, + "grad_norm": 0.2900600158100116, + "learning_rate": 9.951679477948946e-06, + "loss": 1.6861, + "step": 89 + }, + { + "epoch": 0.3953147877013177, + "grad_norm": 0.2775954202644971, + "learning_rate": 9.949107209404664e-06, + "loss": 1.5706, + "step": 90 + }, + { + "epoch": 0.39970717423133234, + "grad_norm": 0.2733537235090247, + "learning_rate": 9.946468589238022e-06, + "loss": 1.7146, + "step": 91 + }, + { + "epoch": 0.404099560761347, + "grad_norm": 0.2710139549067813, + "learning_rate": 9.943763652824437e-06, + "loss": 1.6488, + "step": 92 + }, + { + "epoch": 0.4084919472913616, + "grad_norm": 0.27892641103107996, + "learning_rate": 9.94099243642841e-06, + "loss": 1.6002, + "step": 93 + }, + { + "epoch": 0.4128843338213763, + "grad_norm": 0.4570340740318857, + "learning_rate": 9.93815497720305e-06, + "loss": 1.6686, + "step": 94 + }, + { + "epoch": 0.4172767203513909, + "grad_norm": 0.2816582371174757, + "learning_rate": 9.935251313189564e-06, + "loss": 1.7531, + "step": 95 + }, + { + "epoch": 0.4216691068814056, + "grad_norm": 0.2886345479986427, + "learning_rate": 9.932281483316759e-06, + "loss": 1.744, + "step": 96 + }, + { + "epoch": 0.4260614934114202, + "grad_norm": 0.3178449956639589, + "learning_rate": 9.929245527400504e-06, + "loss": 1.6404, + "step": 97 + }, + { + "epoch": 0.43045387994143486, + "grad_norm": 0.31466032997182714, + "learning_rate": 9.926143486143216e-06, + "loss": 1.6583, + "step": 98 + }, + { + "epoch": 0.43484626647144947, + "grad_norm": 0.3406321749653024, + "learning_rate": 9.922975401133292e-06, + "loss": 1.7313, + "step": 99 + }, + { + "epoch": 0.43923865300146414, + "grad_norm": 0.33050248521276526, + "learning_rate": 9.919741314844577e-06, + "loss": 1.6518, + "step": 100 + }, + { + "epoch": 0.44363103953147875, + "grad_norm": 0.2886134577231967, + "learning_rate": 9.916441270635772e-06, + "loss": 1.6957, + "step": 101 + }, + { + "epoch": 0.4480234260614934, + "grad_norm": 0.27520985458354463, + "learning_rate": 9.913075312749867e-06, + "loss": 1.6052, + "step": 102 + }, + { + "epoch": 0.45241581259150804, + "grad_norm": 0.2649316531653303, + "learning_rate": 9.909643486313533e-06, + "loss": 1.6739, + "step": 103 + }, + { + "epoch": 0.4568081991215227, + "grad_norm": 0.27557834576379425, + "learning_rate": 9.90614583733654e-06, + "loss": 1.6584, + "step": 104 + }, + { + "epoch": 0.4612005856515373, + "grad_norm": 0.3243046224133678, + "learning_rate": 9.90258241271112e-06, + "loss": 1.6491, + "step": 105 + }, + { + "epoch": 0.465592972181552, + "grad_norm": 0.2601090357711318, + "learning_rate": 9.89895326021134e-06, + "loss": 1.6662, + "step": 106 + }, + { + "epoch": 0.4699853587115666, + "grad_norm": 0.2842151241183655, + "learning_rate": 9.895258428492475e-06, + "loss": 1.6501, + "step": 107 + }, + { + "epoch": 0.4743777452415813, + "grad_norm": 0.2594313373745908, + "learning_rate": 9.891497967090344e-06, + "loss": 1.6475, + "step": 108 + }, + { + "epoch": 0.4787701317715959, + "grad_norm": 0.2844945420500575, + "learning_rate": 9.887671926420649e-06, + "loss": 1.7204, + "step": 109 + }, + { + "epoch": 0.48316251830161056, + "grad_norm": 0.27655451441999607, + "learning_rate": 9.8837803577783e-06, + "loss": 1.6838, + "step": 110 + }, + { + "epoch": 0.48755490483162517, + "grad_norm": 0.3017150137441088, + "learning_rate": 9.879823313336723e-06, + "loss": 1.6688, + "step": 111 + }, + { + "epoch": 0.49194729136163984, + "grad_norm": 0.26575250791745125, + "learning_rate": 9.87580084614717e-06, + "loss": 1.6753, + "step": 112 + }, + { + "epoch": 0.49633967789165445, + "grad_norm": 0.2581989783149807, + "learning_rate": 9.871713010137998e-06, + "loss": 1.6244, + "step": 113 + }, + { + "epoch": 0.5007320644216691, + "grad_norm": 0.2622294231157268, + "learning_rate": 9.86755986011395e-06, + "loss": 1.6261, + "step": 114 + }, + { + "epoch": 0.5051244509516838, + "grad_norm": 0.26209513370076304, + "learning_rate": 9.86334145175542e-06, + "loss": 1.6397, + "step": 115 + }, + { + "epoch": 0.5095168374816984, + "grad_norm": 0.2846184522412747, + "learning_rate": 9.859057841617709e-06, + "loss": 1.7091, + "step": 116 + }, + { + "epoch": 0.513909224011713, + "grad_norm": 0.2528850826310284, + "learning_rate": 9.854709087130261e-06, + "loss": 1.6831, + "step": 117 + }, + { + "epoch": 0.5183016105417276, + "grad_norm": 0.32698350420324157, + "learning_rate": 9.850295246595898e-06, + "loss": 1.6431, + "step": 118 + }, + { + "epoch": 0.5226939970717424, + "grad_norm": 0.35727071053972576, + "learning_rate": 9.845816379190037e-06, + "loss": 1.7088, + "step": 119 + }, + { + "epoch": 0.527086383601757, + "grad_norm": 0.28971742333278044, + "learning_rate": 9.841272544959892e-06, + "loss": 1.7195, + "step": 120 + }, + { + "epoch": 0.5314787701317716, + "grad_norm": 0.468869869211724, + "learning_rate": 9.836663804823683e-06, + "loss": 1.6896, + "step": 121 + }, + { + "epoch": 0.5358711566617862, + "grad_norm": 0.3908944943187778, + "learning_rate": 9.831990220569801e-06, + "loss": 1.7406, + "step": 122 + }, + { + "epoch": 0.5402635431918009, + "grad_norm": 0.2665924461613152, + "learning_rate": 9.827251854855992e-06, + "loss": 1.6609, + "step": 123 + }, + { + "epoch": 0.5446559297218155, + "grad_norm": 0.28211372742262864, + "learning_rate": 9.82244877120851e-06, + "loss": 1.6684, + "step": 124 + }, + { + "epoch": 0.5490483162518301, + "grad_norm": 0.2623738132657017, + "learning_rate": 9.817581034021273e-06, + "loss": 1.6923, + "step": 125 + }, + { + "epoch": 0.5534407027818448, + "grad_norm": 0.35357676308356323, + "learning_rate": 9.812648708554991e-06, + "loss": 1.6467, + "step": 126 + }, + { + "epoch": 0.5578330893118595, + "grad_norm": 0.2555639734647961, + "learning_rate": 9.807651860936299e-06, + "loss": 1.6893, + "step": 127 + }, + { + "epoch": 0.5622254758418741, + "grad_norm": 0.26746261958597517, + "learning_rate": 9.802590558156863e-06, + "loss": 1.6754, + "step": 128 + }, + { + "epoch": 0.5666178623718887, + "grad_norm": 0.26383031628885084, + "learning_rate": 9.797464868072489e-06, + "loss": 1.6363, + "step": 129 + }, + { + "epoch": 0.5710102489019033, + "grad_norm": 0.319619389966322, + "learning_rate": 9.792274859402205e-06, + "loss": 1.636, + "step": 130 + }, + { + "epoch": 0.575402635431918, + "grad_norm": 0.2639308579990406, + "learning_rate": 9.787020601727353e-06, + "loss": 1.6079, + "step": 131 + }, + { + "epoch": 0.5797950219619327, + "grad_norm": 0.28226380411742236, + "learning_rate": 9.781702165490638e-06, + "loss": 1.657, + "step": 132 + }, + { + "epoch": 0.5841874084919473, + "grad_norm": 0.27157003543418035, + "learning_rate": 9.7763196219952e-06, + "loss": 1.7009, + "step": 133 + }, + { + "epoch": 0.5885797950219619, + "grad_norm": 0.28487209604498265, + "learning_rate": 9.770873043403648e-06, + "loss": 1.6236, + "step": 134 + }, + { + "epoch": 0.5929721815519766, + "grad_norm": 0.3009729998770739, + "learning_rate": 9.765362502737098e-06, + "loss": 1.6812, + "step": 135 + }, + { + "epoch": 0.5973645680819912, + "grad_norm": 0.2576811873616972, + "learning_rate": 9.759788073874188e-06, + "loss": 1.6885, + "step": 136 + }, + { + "epoch": 0.6017569546120058, + "grad_norm": 0.3108368566509119, + "learning_rate": 9.754149831550097e-06, + "loss": 1.6086, + "step": 137 + }, + { + "epoch": 0.6061493411420205, + "grad_norm": 0.3336050806293322, + "learning_rate": 9.748447851355534e-06, + "loss": 1.6773, + "step": 138 + }, + { + "epoch": 0.6105417276720352, + "grad_norm": 0.28918467659013913, + "learning_rate": 9.742682209735727e-06, + "loss": 1.7141, + "step": 139 + }, + { + "epoch": 0.6149341142020498, + "grad_norm": 0.25507889874035206, + "learning_rate": 9.736852983989405e-06, + "loss": 1.6335, + "step": 140 + }, + { + "epoch": 0.6193265007320644, + "grad_norm": 0.2712795343500641, + "learning_rate": 9.730960252267744e-06, + "loss": 1.6186, + "step": 141 + }, + { + "epoch": 0.623718887262079, + "grad_norm": 0.29326296437100124, + "learning_rate": 9.725004093573343e-06, + "loss": 1.6039, + "step": 142 + }, + { + "epoch": 0.6281112737920937, + "grad_norm": 0.27412689936526563, + "learning_rate": 9.718984587759148e-06, + "loss": 1.594, + "step": 143 + }, + { + "epoch": 0.6325036603221084, + "grad_norm": 0.28908560688778157, + "learning_rate": 9.712901815527387e-06, + "loss": 1.6458, + "step": 144 + }, + { + "epoch": 0.636896046852123, + "grad_norm": 0.26538330998431864, + "learning_rate": 9.706755858428487e-06, + "loss": 1.6546, + "step": 145 + }, + { + "epoch": 0.6412884333821376, + "grad_norm": 0.26498403141947585, + "learning_rate": 9.70054679885998e-06, + "loss": 1.6224, + "step": 146 + }, + { + "epoch": 0.6456808199121523, + "grad_norm": 0.25028323667878993, + "learning_rate": 9.6942747200654e-06, + "loss": 1.6554, + "step": 147 + }, + { + "epoch": 0.6500732064421669, + "grad_norm": 0.3077620507501807, + "learning_rate": 9.687939706133168e-06, + "loss": 1.6751, + "step": 148 + }, + { + "epoch": 0.6544655929721815, + "grad_norm": 0.2738251492188122, + "learning_rate": 9.68154184199546e-06, + "loss": 1.6139, + "step": 149 + }, + { + "epoch": 0.6588579795021962, + "grad_norm": 0.300766059001253, + "learning_rate": 9.675081213427076e-06, + "loss": 1.6783, + "step": 150 + }, + { + "epoch": 0.6632503660322109, + "grad_norm": 0.2671963968809055, + "learning_rate": 9.668557907044278e-06, + "loss": 1.7028, + "step": 151 + }, + { + "epoch": 0.6676427525622255, + "grad_norm": 0.27596191143138304, + "learning_rate": 9.661972010303641e-06, + "loss": 1.6279, + "step": 152 + }, + { + "epoch": 0.6720351390922401, + "grad_norm": 0.2719766311854951, + "learning_rate": 9.655323611500876e-06, + "loss": 1.6634, + "step": 153 + }, + { + "epoch": 0.6764275256222547, + "grad_norm": 0.2697612116557389, + "learning_rate": 9.648612799769644e-06, + "loss": 1.617, + "step": 154 + }, + { + "epoch": 0.6808199121522694, + "grad_norm": 0.28137116472179047, + "learning_rate": 9.641839665080363e-06, + "loss": 1.6212, + "step": 155 + }, + { + "epoch": 0.6852122986822841, + "grad_norm": 0.2599526010890989, + "learning_rate": 9.635004298239004e-06, + "loss": 1.6542, + "step": 156 + }, + { + "epoch": 0.6896046852122987, + "grad_norm": 0.662388825119619, + "learning_rate": 9.628106790885866e-06, + "loss": 1.633, + "step": 157 + }, + { + "epoch": 0.6939970717423133, + "grad_norm": 0.2913416852335614, + "learning_rate": 9.621147235494357e-06, + "loss": 1.6904, + "step": 158 + }, + { + "epoch": 0.698389458272328, + "grad_norm": 0.26581120890113497, + "learning_rate": 9.614125725369748e-06, + "loss": 1.6583, + "step": 159 + }, + { + "epoch": 0.7027818448023426, + "grad_norm": 0.26696083510238133, + "learning_rate": 9.607042354647924e-06, + "loss": 1.6354, + "step": 160 + }, + { + "epoch": 0.7071742313323572, + "grad_norm": 0.2913753113559095, + "learning_rate": 9.599897218294122e-06, + "loss": 1.5914, + "step": 161 + }, + { + "epoch": 0.7115666178623719, + "grad_norm": 0.2794317973029827, + "learning_rate": 9.59269041210166e-06, + "loss": 1.5732, + "step": 162 + }, + { + "epoch": 0.7159590043923866, + "grad_norm": 0.27364398897711945, + "learning_rate": 9.585422032690644e-06, + "loss": 1.6796, + "step": 163 + }, + { + "epoch": 0.7203513909224012, + "grad_norm": 0.2924137561125512, + "learning_rate": 9.578092177506683e-06, + "loss": 1.6921, + "step": 164 + }, + { + "epoch": 0.7247437774524158, + "grad_norm": 0.27864643658701665, + "learning_rate": 9.570700944819584e-06, + "loss": 1.6442, + "step": 165 + }, + { + "epoch": 0.7291361639824304, + "grad_norm": 0.3172135018133218, + "learning_rate": 9.56324843372202e-06, + "loss": 1.7003, + "step": 166 + }, + { + "epoch": 0.7335285505124451, + "grad_norm": 0.33119379016577744, + "learning_rate": 9.55573474412821e-06, + "loss": 1.6277, + "step": 167 + }, + { + "epoch": 0.7379209370424598, + "grad_norm": 0.24315818847358012, + "learning_rate": 9.548159976772593e-06, + "loss": 1.621, + "step": 168 + }, + { + "epoch": 0.7423133235724744, + "grad_norm": 0.28066914391093295, + "learning_rate": 9.540524233208449e-06, + "loss": 1.6169, + "step": 169 + }, + { + "epoch": 0.746705710102489, + "grad_norm": 0.3096372829142638, + "learning_rate": 9.532827615806561e-06, + "loss": 1.6425, + "step": 170 + }, + { + "epoch": 0.7510980966325037, + "grad_norm": 0.3050362838238551, + "learning_rate": 9.525070227753835e-06, + "loss": 1.5835, + "step": 171 + }, + { + "epoch": 0.7554904831625183, + "grad_norm": 0.27558198897100084, + "learning_rate": 9.517252173051912e-06, + "loss": 1.6866, + "step": 172 + }, + { + "epoch": 0.7598828696925329, + "grad_norm": 0.3415215102331636, + "learning_rate": 9.509373556515781e-06, + "loss": 1.6954, + "step": 173 + }, + { + "epoch": 0.7642752562225475, + "grad_norm": 0.2833145266859935, + "learning_rate": 9.501434483772371e-06, + "loss": 1.5642, + "step": 174 + }, + { + "epoch": 0.7686676427525623, + "grad_norm": 0.261080979929468, + "learning_rate": 9.49343506125913e-06, + "loss": 1.6481, + "step": 175 + }, + { + "epoch": 0.7730600292825769, + "grad_norm": 0.27528584249968596, + "learning_rate": 9.48537539622261e-06, + "loss": 1.6356, + "step": 176 + }, + { + "epoch": 0.7774524158125915, + "grad_norm": 0.28493642680744213, + "learning_rate": 9.477255596717012e-06, + "loss": 1.617, + "step": 177 + }, + { + "epoch": 0.7818448023426061, + "grad_norm": 0.250792323656803, + "learning_rate": 9.469075771602756e-06, + "loss": 1.7373, + "step": 178 + }, + { + "epoch": 0.7862371888726208, + "grad_norm": 0.2815707976300464, + "learning_rate": 9.460836030545007e-06, + "loss": 1.7134, + "step": 179 + }, + { + "epoch": 0.7906295754026355, + "grad_norm": 0.33965220402585167, + "learning_rate": 9.452536484012212e-06, + "loss": 1.66, + "step": 180 + }, + { + "epoch": 0.7950219619326501, + "grad_norm": 0.337075115730111, + "learning_rate": 9.444177243274619e-06, + "loss": 1.7237, + "step": 181 + }, + { + "epoch": 0.7994143484626647, + "grad_norm": 0.2652773355794464, + "learning_rate": 9.43575842040278e-06, + "loss": 1.7238, + "step": 182 + }, + { + "epoch": 0.8038067349926794, + "grad_norm": 0.29569868187991866, + "learning_rate": 9.427280128266049e-06, + "loss": 1.6484, + "step": 183 + }, + { + "epoch": 0.808199121522694, + "grad_norm": 0.2647366968721245, + "learning_rate": 9.418742480531086e-06, + "loss": 1.6557, + "step": 184 + }, + { + "epoch": 0.8125915080527086, + "grad_norm": 0.2624692157687208, + "learning_rate": 9.410145591660301e-06, + "loss": 1.6501, + "step": 185 + }, + { + "epoch": 0.8169838945827232, + "grad_norm": 0.2604406630602592, + "learning_rate": 9.40148957691035e-06, + "loss": 1.6205, + "step": 186 + }, + { + "epoch": 0.821376281112738, + "grad_norm": 0.3809448031526879, + "learning_rate": 9.392774552330567e-06, + "loss": 1.6014, + "step": 187 + }, + { + "epoch": 0.8257686676427526, + "grad_norm": 0.285104899301775, + "learning_rate": 9.384000634761429e-06, + "loss": 1.7119, + "step": 188 + }, + { + "epoch": 0.8301610541727672, + "grad_norm": 0.3603217046093286, + "learning_rate": 9.375167941832974e-06, + "loss": 1.6129, + "step": 189 + }, + { + "epoch": 0.8345534407027818, + "grad_norm": 0.26192371812461096, + "learning_rate": 9.366276591963222e-06, + "loss": 1.6648, + "step": 190 + }, + { + "epoch": 0.8389458272327965, + "grad_norm": 0.27224376529649075, + "learning_rate": 9.357326704356602e-06, + "loss": 1.7368, + "step": 191 + }, + { + "epoch": 0.8433382137628112, + "grad_norm": 0.267560205737413, + "learning_rate": 9.348318399002347e-06, + "loss": 1.6456, + "step": 192 + }, + { + "epoch": 0.8477306002928258, + "grad_norm": 0.38772390686795116, + "learning_rate": 9.339251796672878e-06, + "loss": 1.6765, + "step": 193 + }, + { + "epoch": 0.8521229868228404, + "grad_norm": 0.28624234868841525, + "learning_rate": 9.330127018922195e-06, + "loss": 1.6397, + "step": 194 + }, + { + "epoch": 0.8565153733528551, + "grad_norm": 0.3517826456437289, + "learning_rate": 9.320944188084241e-06, + "loss": 1.6504, + "step": 195 + }, + { + "epoch": 0.8609077598828697, + "grad_norm": 0.3054896957644684, + "learning_rate": 9.31170342727127e-06, + "loss": 1.6581, + "step": 196 + }, + { + "epoch": 0.8653001464128843, + "grad_norm": 0.47217080493290625, + "learning_rate": 9.302404860372185e-06, + "loss": 1.6614, + "step": 197 + }, + { + "epoch": 0.8696925329428989, + "grad_norm": 0.3744290659075852, + "learning_rate": 9.293048612050883e-06, + "loss": 1.6181, + "step": 198 + }, + { + "epoch": 0.8740849194729137, + "grad_norm": 0.2928672407192581, + "learning_rate": 9.283634807744586e-06, + "loss": 1.5801, + "step": 199 + }, + { + "epoch": 0.8784773060029283, + "grad_norm": 0.5500989172351091, + "learning_rate": 9.274163573662158e-06, + "loss": 1.6213, + "step": 200 + }, + { + "epoch": 0.8828696925329429, + "grad_norm": 0.27844961793523915, + "learning_rate": 9.264635036782406e-06, + "loss": 1.6442, + "step": 201 + }, + { + "epoch": 0.8872620790629575, + "grad_norm": 0.2764381586339949, + "learning_rate": 9.255049324852388e-06, + "loss": 1.6403, + "step": 202 + }, + { + "epoch": 0.8916544655929722, + "grad_norm": 0.28692831842946026, + "learning_rate": 9.245406566385698e-06, + "loss": 1.6204, + "step": 203 + }, + { + "epoch": 0.8960468521229868, + "grad_norm": 0.2854958274623322, + "learning_rate": 9.235706890660735e-06, + "loss": 1.6777, + "step": 204 + }, + { + "epoch": 0.9004392386530015, + "grad_norm": 0.2884758073862357, + "learning_rate": 9.225950427718974e-06, + "loss": 1.6414, + "step": 205 + }, + { + "epoch": 0.9048316251830161, + "grad_norm": 0.2774591809427237, + "learning_rate": 9.216137308363235e-06, + "loss": 1.667, + "step": 206 + }, + { + "epoch": 0.9092240117130308, + "grad_norm": 0.26327408771476135, + "learning_rate": 9.206267664155906e-06, + "loss": 1.6472, + "step": 207 + }, + { + "epoch": 0.9136163982430454, + "grad_norm": 0.3163508666559799, + "learning_rate": 9.1963416274172e-06, + "loss": 1.6898, + "step": 208 + }, + { + "epoch": 0.91800878477306, + "grad_norm": 0.2691312569989342, + "learning_rate": 9.18635933122337e-06, + "loss": 1.6788, + "step": 209 + }, + { + "epoch": 0.9224011713030746, + "grad_norm": 0.329963898141559, + "learning_rate": 9.176320909404925e-06, + "loss": 1.7077, + "step": 210 + }, + { + "epoch": 0.9267935578330894, + "grad_norm": 0.3497378648580076, + "learning_rate": 9.16622649654484e-06, + "loss": 1.7203, + "step": 211 + }, + { + "epoch": 0.931185944363104, + "grad_norm": 0.2750435891659104, + "learning_rate": 9.156076227976752e-06, + "loss": 1.6372, + "step": 212 + }, + { + "epoch": 0.9355783308931186, + "grad_norm": 0.29344217135995226, + "learning_rate": 9.145870239783143e-06, + "loss": 1.6294, + "step": 213 + }, + { + "epoch": 0.9399707174231332, + "grad_norm": 0.2713649627402965, + "learning_rate": 9.135608668793511e-06, + "loss": 1.6391, + "step": 214 + }, + { + "epoch": 0.9443631039531479, + "grad_norm": 0.2995344678229402, + "learning_rate": 9.125291652582549e-06, + "loss": 1.6102, + "step": 215 + }, + { + "epoch": 0.9487554904831625, + "grad_norm": 0.32763546183738235, + "learning_rate": 9.114919329468283e-06, + "loss": 1.6823, + "step": 216 + }, + { + "epoch": 0.9531478770131772, + "grad_norm": 0.29620265372375704, + "learning_rate": 9.104491838510235e-06, + "loss": 1.6905, + "step": 217 + }, + { + "epoch": 0.9575402635431918, + "grad_norm": 0.43487598587006066, + "learning_rate": 9.094009319507547e-06, + "loss": 1.6611, + "step": 218 + }, + { + "epoch": 0.9619326500732065, + "grad_norm": 0.32989859860270077, + "learning_rate": 9.08347191299711e-06, + "loss": 1.634, + "step": 219 + }, + { + "epoch": 0.9663250366032211, + "grad_norm": 0.26622257061624277, + "learning_rate": 9.07287976025168e-06, + "loss": 1.6075, + "step": 220 + }, + { + "epoch": 0.9707174231332357, + "grad_norm": 0.28626358026286186, + "learning_rate": 9.062233003277983e-06, + "loss": 1.6045, + "step": 221 + }, + { + "epoch": 0.9751098096632503, + "grad_norm": 0.5424158794876999, + "learning_rate": 9.051531784814817e-06, + "loss": 1.5825, + "step": 222 + }, + { + "epoch": 0.9795021961932651, + "grad_norm": 0.2771987597060821, + "learning_rate": 9.040776248331128e-06, + "loss": 1.6083, + "step": 223 + }, + { + "epoch": 0.9838945827232797, + "grad_norm": 1.5311321947246777, + "learning_rate": 9.029966538024097e-06, + "loss": 1.5973, + "step": 224 + }, + { + "epoch": 0.9882869692532943, + "grad_norm": 0.3081246423484545, + "learning_rate": 9.019102798817196e-06, + "loss": 1.5915, + "step": 225 + }, + { + "epoch": 0.9926793557833089, + "grad_norm": 0.3066770476121554, + "learning_rate": 9.008185176358257e-06, + "loss": 1.5551, + "step": 226 + }, + { + "epoch": 0.9970717423133236, + "grad_norm": 0.29888292004137484, + "learning_rate": 8.997213817017508e-06, + "loss": 1.7461, + "step": 227 + }, + { + "epoch": 1.0043923865300146, + "grad_norm": 0.5938110979809961, + "learning_rate": 8.986188867885617e-06, + "loss": 3.2509, + "step": 228 + }, + { + "epoch": 1.0087847730600292, + "grad_norm": 0.2899326916339286, + "learning_rate": 8.975110476771724e-06, + "loss": 1.5874, + "step": 229 + }, + { + "epoch": 1.0131771595900438, + "grad_norm": 0.2777095345419943, + "learning_rate": 8.96397879220145e-06, + "loss": 1.6706, + "step": 230 + }, + { + "epoch": 1.0175695461200585, + "grad_norm": 0.26363195429841585, + "learning_rate": 8.952793963414908e-06, + "loss": 1.6317, + "step": 231 + }, + { + "epoch": 1.0219619326500733, + "grad_norm": 0.2688971491502818, + "learning_rate": 8.941556140364707e-06, + "loss": 1.654, + "step": 232 + }, + { + "epoch": 1.026354319180088, + "grad_norm": 0.28487309366468816, + "learning_rate": 8.930265473713939e-06, + "loss": 1.6133, + "step": 233 + }, + { + "epoch": 1.0307467057101025, + "grad_norm": 0.28141744313455036, + "learning_rate": 8.918922114834156e-06, + "loss": 1.589, + "step": 234 + }, + { + "epoch": 1.0351390922401171, + "grad_norm": 0.40308880749849646, + "learning_rate": 8.907526215803351e-06, + "loss": 1.6499, + "step": 235 + }, + { + "epoch": 1.0395314787701317, + "grad_norm": 0.2834055854316979, + "learning_rate": 8.896077929403901e-06, + "loss": 1.658, + "step": 236 + }, + { + "epoch": 1.0439238653001464, + "grad_norm": 0.2887035123274668, + "learning_rate": 8.884577409120535e-06, + "loss": 1.6196, + "step": 237 + }, + { + "epoch": 1.048316251830161, + "grad_norm": 0.2985213212589398, + "learning_rate": 8.873024809138272e-06, + "loss": 1.677, + "step": 238 + }, + { + "epoch": 1.0527086383601758, + "grad_norm": 0.2863532205939482, + "learning_rate": 8.861420284340352e-06, + "loss": 1.6682, + "step": 239 + }, + { + "epoch": 1.0571010248901904, + "grad_norm": 0.29235163398244823, + "learning_rate": 8.849763990306153e-06, + "loss": 1.6009, + "step": 240 + }, + { + "epoch": 1.061493411420205, + "grad_norm": 0.2899336107020558, + "learning_rate": 8.838056083309118e-06, + "loss": 1.6319, + "step": 241 + }, + { + "epoch": 1.0658857979502196, + "grad_norm": 0.3186837918366928, + "learning_rate": 8.826296720314658e-06, + "loss": 1.5875, + "step": 242 + }, + { + "epoch": 1.0702781844802343, + "grad_norm": 0.33938164798671433, + "learning_rate": 8.814486058978035e-06, + "loss": 1.6059, + "step": 243 + }, + { + "epoch": 1.0746705710102489, + "grad_norm": 0.2991115627129074, + "learning_rate": 8.802624257642262e-06, + "loss": 1.6599, + "step": 244 + }, + { + "epoch": 1.0790629575402635, + "grad_norm": 0.28608565865487656, + "learning_rate": 8.790711475335971e-06, + "loss": 1.6364, + "step": 245 + }, + { + "epoch": 1.083455344070278, + "grad_norm": 0.28359019806003394, + "learning_rate": 8.778747871771293e-06, + "loss": 1.6584, + "step": 246 + }, + { + "epoch": 1.0878477306002927, + "grad_norm": 0.34292617248780644, + "learning_rate": 8.766733607341697e-06, + "loss": 1.6613, + "step": 247 + }, + { + "epoch": 1.0922401171303076, + "grad_norm": 0.3469366526291968, + "learning_rate": 8.754668843119865e-06, + "loss": 1.6542, + "step": 248 + }, + { + "epoch": 1.0966325036603222, + "grad_norm": 0.2745752678931537, + "learning_rate": 8.742553740855507e-06, + "loss": 1.6723, + "step": 249 + }, + { + "epoch": 1.1010248901903368, + "grad_norm": 0.3863733971251328, + "learning_rate": 8.730388462973209e-06, + "loss": 1.6268, + "step": 250 + }, + { + "epoch": 1.1054172767203514, + "grad_norm": 0.3196278576328997, + "learning_rate": 8.718173172570254e-06, + "loss": 1.6349, + "step": 251 + }, + { + "epoch": 1.109809663250366, + "grad_norm": 0.27017487934331, + "learning_rate": 8.705908033414426e-06, + "loss": 1.606, + "step": 252 + }, + { + "epoch": 1.1142020497803806, + "grad_norm": 0.6576768607622604, + "learning_rate": 8.693593209941826e-06, + "loss": 1.5936, + "step": 253 + }, + { + "epoch": 1.1185944363103952, + "grad_norm": 0.30260640047372234, + "learning_rate": 8.681228867254657e-06, + "loss": 1.5948, + "step": 254 + }, + { + "epoch": 1.12298682284041, + "grad_norm": 0.3102239482584757, + "learning_rate": 8.66881517111902e-06, + "loss": 1.6928, + "step": 255 + }, + { + "epoch": 1.1273792093704247, + "grad_norm": 0.2818931187575453, + "learning_rate": 8.656352287962687e-06, + "loss": 1.687, + "step": 256 + }, + { + "epoch": 1.1317715959004393, + "grad_norm": 0.30007928577375903, + "learning_rate": 8.643840384872865e-06, + "loss": 1.6628, + "step": 257 + }, + { + "epoch": 1.136163982430454, + "grad_norm": 0.3249258384227434, + "learning_rate": 8.631279629593968e-06, + "loss": 1.6123, + "step": 258 + }, + { + "epoch": 1.1405563689604685, + "grad_norm": 0.2871446068255736, + "learning_rate": 8.61867019052535e-06, + "loss": 1.6608, + "step": 259 + }, + { + "epoch": 1.1449487554904831, + "grad_norm": 0.3001183609451796, + "learning_rate": 8.606012236719073e-06, + "loss": 1.6405, + "step": 260 + }, + { + "epoch": 1.1493411420204978, + "grad_norm": 0.271155672303218, + "learning_rate": 8.593305937877614e-06, + "loss": 1.5819, + "step": 261 + }, + { + "epoch": 1.1537335285505124, + "grad_norm": 0.30999460151747915, + "learning_rate": 8.580551464351603e-06, + "loss": 1.6378, + "step": 262 + }, + { + "epoch": 1.158125915080527, + "grad_norm": 0.2979330701414515, + "learning_rate": 8.567748987137544e-06, + "loss": 1.7342, + "step": 263 + }, + { + "epoch": 1.1625183016105418, + "grad_norm": 0.29037637226651547, + "learning_rate": 8.554898677875509e-06, + "loss": 1.663, + "step": 264 + }, + { + "epoch": 1.1669106881405564, + "grad_norm": 0.29574080692093535, + "learning_rate": 8.542000708846851e-06, + "loss": 1.6937, + "step": 265 + }, + { + "epoch": 1.171303074670571, + "grad_norm": 0.2867463319690563, + "learning_rate": 8.52905525297188e-06, + "loss": 1.5812, + "step": 266 + }, + { + "epoch": 1.1756954612005857, + "grad_norm": 0.3075848284440638, + "learning_rate": 8.516062483807556e-06, + "loss": 1.6421, + "step": 267 + }, + { + "epoch": 1.1800878477306003, + "grad_norm": 0.2745442718449274, + "learning_rate": 8.503022575545159e-06, + "loss": 1.6592, + "step": 268 + }, + { + "epoch": 1.1844802342606149, + "grad_norm": 0.4008583979349149, + "learning_rate": 8.48993570300795e-06, + "loss": 1.6052, + "step": 269 + }, + { + "epoch": 1.1888726207906295, + "grad_norm": 0.5183284694523707, + "learning_rate": 8.476802041648832e-06, + "loss": 1.7352, + "step": 270 + }, + { + "epoch": 1.1932650073206443, + "grad_norm": 0.30560643793660064, + "learning_rate": 8.463621767547998e-06, + "loss": 1.5592, + "step": 271 + }, + { + "epoch": 1.197657393850659, + "grad_norm": 0.5272799941494408, + "learning_rate": 8.450395057410561e-06, + "loss": 1.5471, + "step": 272 + }, + { + "epoch": 1.2020497803806736, + "grad_norm": 0.4281815470371588, + "learning_rate": 8.437122088564197e-06, + "loss": 1.6445, + "step": 273 + }, + { + "epoch": 1.2064421669106882, + "grad_norm": 0.27224728594527714, + "learning_rate": 8.423803038956763e-06, + "loss": 1.6629, + "step": 274 + }, + { + "epoch": 1.2108345534407028, + "grad_norm": 0.3269535080116159, + "learning_rate": 8.41043808715391e-06, + "loss": 1.5722, + "step": 275 + }, + { + "epoch": 1.2152269399707174, + "grad_norm": 0.3054371244574999, + "learning_rate": 8.39702741233669e-06, + "loss": 1.6173, + "step": 276 + }, + { + "epoch": 1.219619326500732, + "grad_norm": 0.36010851057708126, + "learning_rate": 8.383571194299154e-06, + "loss": 1.596, + "step": 277 + }, + { + "epoch": 1.2240117130307466, + "grad_norm": 0.3803533141127172, + "learning_rate": 8.37006961344594e-06, + "loss": 1.6229, + "step": 278 + }, + { + "epoch": 1.2284040995607612, + "grad_norm": 0.28612951080269505, + "learning_rate": 8.356522850789852e-06, + "loss": 1.5549, + "step": 279 + }, + { + "epoch": 1.232796486090776, + "grad_norm": 0.2855964904759227, + "learning_rate": 8.342931087949446e-06, + "loss": 1.6512, + "step": 280 + }, + { + "epoch": 1.2371888726207907, + "grad_norm": 0.3054377930088084, + "learning_rate": 8.32929450714658e-06, + "loss": 1.6274, + "step": 281 + }, + { + "epoch": 1.2415812591508053, + "grad_norm": 0.29343763848165016, + "learning_rate": 8.315613291203977e-06, + "loss": 1.6372, + "step": 282 + }, + { + "epoch": 1.24597364568082, + "grad_norm": 0.40870431747557984, + "learning_rate": 8.301887623542774e-06, + "loss": 1.6667, + "step": 283 + }, + { + "epoch": 1.2503660322108345, + "grad_norm": 0.30881819618442474, + "learning_rate": 8.288117688180064e-06, + "loss": 1.5547, + "step": 284 + }, + { + "epoch": 1.2547584187408491, + "grad_norm": 0.28765815508847925, + "learning_rate": 8.274303669726427e-06, + "loss": 1.6097, + "step": 285 + }, + { + "epoch": 1.2591508052708638, + "grad_norm": 0.33007103881402516, + "learning_rate": 8.260445753383453e-06, + "loss": 1.7016, + "step": 286 + }, + { + "epoch": 1.2635431918008786, + "grad_norm": 0.3628853649167158, + "learning_rate": 8.246544124941266e-06, + "loss": 1.6086, + "step": 287 + }, + { + "epoch": 1.267935578330893, + "grad_norm": 0.40134140387051037, + "learning_rate": 8.232598970776027e-06, + "loss": 1.6789, + "step": 288 + }, + { + "epoch": 1.2723279648609078, + "grad_norm": 0.2920563442488122, + "learning_rate": 8.218610477847435e-06, + "loss": 1.5915, + "step": 289 + }, + { + "epoch": 1.2767203513909224, + "grad_norm": 0.28530665366941144, + "learning_rate": 8.204578833696228e-06, + "loss": 1.5949, + "step": 290 + }, + { + "epoch": 1.281112737920937, + "grad_norm": 0.3155789127902478, + "learning_rate": 8.190504226441654e-06, + "loss": 1.639, + "step": 291 + }, + { + "epoch": 1.2855051244509517, + "grad_norm": 0.3199350667177758, + "learning_rate": 8.176386844778969e-06, + "loss": 1.6818, + "step": 292 + }, + { + "epoch": 1.2898975109809663, + "grad_norm": 0.34596043922632713, + "learning_rate": 8.162226877976886e-06, + "loss": 1.6604, + "step": 293 + }, + { + "epoch": 1.294289897510981, + "grad_norm": 0.32781624288061406, + "learning_rate": 8.148024515875057e-06, + "loss": 1.5971, + "step": 294 + }, + { + "epoch": 1.2986822840409955, + "grad_norm": 0.5927243317332561, + "learning_rate": 8.133779948881514e-06, + "loss": 1.6574, + "step": 295 + }, + { + "epoch": 1.3030746705710103, + "grad_norm": 0.296769298222996, + "learning_rate": 8.11949336797012e-06, + "loss": 1.5505, + "step": 296 + }, + { + "epoch": 1.307467057101025, + "grad_norm": 0.28849112804215943, + "learning_rate": 8.105164964678009e-06, + "loss": 1.6729, + "step": 297 + }, + { + "epoch": 1.3118594436310396, + "grad_norm": 0.3286168384006033, + "learning_rate": 8.090794931103026e-06, + "loss": 1.5949, + "step": 298 + }, + { + "epoch": 1.3162518301610542, + "grad_norm": 0.31185844876531665, + "learning_rate": 8.076383459901138e-06, + "loss": 1.6918, + "step": 299 + }, + { + "epoch": 1.3206442166910688, + "grad_norm": 0.2789921585419944, + "learning_rate": 8.061930744283855e-06, + "loss": 1.6437, + "step": 300 + }, + { + "epoch": 1.3250366032210834, + "grad_norm": 0.30165508316706874, + "learning_rate": 8.04743697801565e-06, + "loss": 1.6555, + "step": 301 + }, + { + "epoch": 1.329428989751098, + "grad_norm": 0.2989059936488396, + "learning_rate": 8.032902355411345e-06, + "loss": 1.5141, + "step": 302 + }, + { + "epoch": 1.3338213762811129, + "grad_norm": 0.29928394703560823, + "learning_rate": 8.018327071333521e-06, + "loss": 1.5859, + "step": 303 + }, + { + "epoch": 1.3382137628111272, + "grad_norm": 0.29795453570714703, + "learning_rate": 8.003711321189895e-06, + "loss": 1.5864, + "step": 304 + }, + { + "epoch": 1.342606149341142, + "grad_norm": 0.30765817351768504, + "learning_rate": 7.989055300930704e-06, + "loss": 1.5743, + "step": 305 + }, + { + "epoch": 1.3469985358711567, + "grad_norm": 0.43955554329120444, + "learning_rate": 7.974359207046081e-06, + "loss": 1.6423, + "step": 306 + }, + { + "epoch": 1.3513909224011713, + "grad_norm": 0.37865307469865994, + "learning_rate": 7.959623236563412e-06, + "loss": 1.6102, + "step": 307 + }, + { + "epoch": 1.355783308931186, + "grad_norm": 0.3074090057367007, + "learning_rate": 7.944847587044704e-06, + "loss": 1.6602, + "step": 308 + }, + { + "epoch": 1.3601756954612005, + "grad_norm": 0.3361817362879421, + "learning_rate": 7.930032456583931e-06, + "loss": 1.6704, + "step": 309 + }, + { + "epoch": 1.3645680819912152, + "grad_norm": 0.2960444587212274, + "learning_rate": 7.915178043804382e-06, + "loss": 1.5536, + "step": 310 + }, + { + "epoch": 1.3689604685212298, + "grad_norm": 0.31080078465435335, + "learning_rate": 7.900284547855992e-06, + "loss": 1.659, + "step": 311 + }, + { + "epoch": 1.3733528550512446, + "grad_norm": 0.29259208311732576, + "learning_rate": 7.885352168412677e-06, + "loss": 1.6324, + "step": 312 + }, + { + "epoch": 1.3777452415812592, + "grad_norm": 0.40704907971444493, + "learning_rate": 7.870381105669657e-06, + "loss": 1.6954, + "step": 313 + }, + { + "epoch": 1.3821376281112738, + "grad_norm": 0.2884678272243352, + "learning_rate": 7.85537156034077e-06, + "loss": 1.6911, + "step": 314 + }, + { + "epoch": 1.3865300146412884, + "grad_norm": 0.3002277679918279, + "learning_rate": 7.84032373365578e-06, + "loss": 1.6384, + "step": 315 + }, + { + "epoch": 1.390922401171303, + "grad_norm": 0.29747656531257255, + "learning_rate": 7.825237827357684e-06, + "loss": 1.5728, + "step": 316 + }, + { + "epoch": 1.3953147877013177, + "grad_norm": 0.2922330023166882, + "learning_rate": 7.810114043700002e-06, + "loss": 1.6876, + "step": 317 + }, + { + "epoch": 1.3997071742313323, + "grad_norm": 0.29216993729442, + "learning_rate": 7.794952585444068e-06, + "loss": 1.7112, + "step": 318 + }, + { + "epoch": 1.4040995607613471, + "grad_norm": 0.2841430060967501, + "learning_rate": 7.779753655856313e-06, + "loss": 1.6352, + "step": 319 + }, + { + "epoch": 1.4084919472913615, + "grad_norm": 0.29535174268674796, + "learning_rate": 7.764517458705536e-06, + "loss": 1.638, + "step": 320 + }, + { + "epoch": 1.4128843338213763, + "grad_norm": 0.3553267662730052, + "learning_rate": 7.749244198260175e-06, + "loss": 1.6262, + "step": 321 + }, + { + "epoch": 1.417276720351391, + "grad_norm": 0.2977673782348279, + "learning_rate": 7.733934079285569e-06, + "loss": 1.6115, + "step": 322 + }, + { + "epoch": 1.4216691068814056, + "grad_norm": 0.2822196746941408, + "learning_rate": 7.718587307041209e-06, + "loss": 1.6468, + "step": 323 + }, + { + "epoch": 1.4260614934114202, + "grad_norm": 0.2837671497162217, + "learning_rate": 7.703204087277989e-06, + "loss": 1.6474, + "step": 324 + }, + { + "epoch": 1.4304538799414348, + "grad_norm": 0.31664451334685184, + "learning_rate": 7.687784626235448e-06, + "loss": 1.5919, + "step": 325 + }, + { + "epoch": 1.4348462664714494, + "grad_norm": 0.33195017711571867, + "learning_rate": 7.672329130639007e-06, + "loss": 1.6941, + "step": 326 + }, + { + "epoch": 1.439238653001464, + "grad_norm": 0.2995108911659512, + "learning_rate": 7.656837807697187e-06, + "loss": 1.6393, + "step": 327 + }, + { + "epoch": 1.4436310395314789, + "grad_norm": 0.2949122695416032, + "learning_rate": 7.641310865098845e-06, + "loss": 1.5135, + "step": 328 + }, + { + "epoch": 1.4480234260614935, + "grad_norm": 0.31946384200509315, + "learning_rate": 7.625748511010382e-06, + "loss": 1.5784, + "step": 329 + }, + { + "epoch": 1.452415812591508, + "grad_norm": 0.29208814812765094, + "learning_rate": 7.610150954072953e-06, + "loss": 1.6794, + "step": 330 + }, + { + "epoch": 1.4568081991215227, + "grad_norm": 0.34475174012170284, + "learning_rate": 7.594518403399666e-06, + "loss": 1.5773, + "step": 331 + }, + { + "epoch": 1.4612005856515373, + "grad_norm": 0.2876304119955936, + "learning_rate": 7.578851068572788e-06, + "loss": 1.5613, + "step": 332 + }, + { + "epoch": 1.465592972181552, + "grad_norm": 0.31542092184200016, + "learning_rate": 7.563149159640929e-06, + "loss": 1.6086, + "step": 333 + }, + { + "epoch": 1.4699853587115665, + "grad_norm": 0.2964161628058562, + "learning_rate": 7.547412887116224e-06, + "loss": 1.5963, + "step": 334 + }, + { + "epoch": 1.4743777452415814, + "grad_norm": 0.32832262869958717, + "learning_rate": 7.531642461971515e-06, + "loss": 1.6054, + "step": 335 + }, + { + "epoch": 1.4787701317715958, + "grad_norm": 0.3209241382730098, + "learning_rate": 7.51583809563752e-06, + "loss": 1.6748, + "step": 336 + }, + { + "epoch": 1.4831625183016106, + "grad_norm": 0.30060262616835987, + "learning_rate": 7.500000000000001e-06, + "loss": 1.6081, + "step": 337 + }, + { + "epoch": 1.4875549048316252, + "grad_norm": 0.3500388762321464, + "learning_rate": 7.4841283873969194e-06, + "loss": 1.5906, + "step": 338 + }, + { + "epoch": 1.4919472913616398, + "grad_norm": 0.28308587203259633, + "learning_rate": 7.468223470615593e-06, + "loss": 1.5879, + "step": 339 + }, + { + "epoch": 1.4963396778916545, + "grad_norm": 0.2901927981331445, + "learning_rate": 7.452285462889841e-06, + "loss": 1.6242, + "step": 340 + }, + { + "epoch": 1.500732064421669, + "grad_norm": 0.3314307283598877, + "learning_rate": 7.436314577897126e-06, + "loss": 1.646, + "step": 341 + }, + { + "epoch": 1.505124450951684, + "grad_norm": 0.3264131109049365, + "learning_rate": 7.420311029755688e-06, + "loss": 1.5855, + "step": 342 + }, + { + "epoch": 1.5095168374816983, + "grad_norm": 0.3519561244527488, + "learning_rate": 7.404275033021676e-06, + "loss": 1.6415, + "step": 343 + }, + { + "epoch": 1.5139092240117131, + "grad_norm": 0.3231864119230437, + "learning_rate": 7.388206802686272e-06, + "loss": 1.6038, + "step": 344 + }, + { + "epoch": 1.5183016105417275, + "grad_norm": 0.4648117736859699, + "learning_rate": 7.372106554172802e-06, + "loss": 1.6327, + "step": 345 + }, + { + "epoch": 1.5226939970717424, + "grad_norm": 0.30819599956019383, + "learning_rate": 7.355974503333859e-06, + "loss": 1.7179, + "step": 346 + }, + { + "epoch": 1.527086383601757, + "grad_norm": 0.3425241683440226, + "learning_rate": 7.339810866448398e-06, + "loss": 1.6486, + "step": 347 + }, + { + "epoch": 1.5314787701317716, + "grad_norm": 0.3229436387251198, + "learning_rate": 7.323615860218844e-06, + "loss": 1.5028, + "step": 348 + }, + { + "epoch": 1.5358711566617862, + "grad_norm": 0.32420322558759, + "learning_rate": 7.307389701768183e-06, + "loss": 1.517, + "step": 349 + }, + { + "epoch": 1.5402635431918008, + "grad_norm": 0.33308925785501975, + "learning_rate": 7.291132608637053e-06, + "loss": 1.6278, + "step": 350 + }, + { + "epoch": 1.5446559297218156, + "grad_norm": 0.33961176568859514, + "learning_rate": 7.274844798780826e-06, + "loss": 1.6651, + "step": 351 + }, + { + "epoch": 1.54904831625183, + "grad_norm": 0.32173930273995155, + "learning_rate": 7.258526490566687e-06, + "loss": 1.491, + "step": 352 + }, + { + "epoch": 1.5534407027818449, + "grad_norm": 0.3093139646127846, + "learning_rate": 7.242177902770706e-06, + "loss": 1.5796, + "step": 353 + }, + { + "epoch": 1.5578330893118595, + "grad_norm": 0.32919345439390313, + "learning_rate": 7.2257992545749045e-06, + "loss": 1.5865, + "step": 354 + }, + { + "epoch": 1.562225475841874, + "grad_norm": 0.2835010953118983, + "learning_rate": 7.209390765564318e-06, + "loss": 1.6417, + "step": 355 + }, + { + "epoch": 1.5666178623718887, + "grad_norm": 1.7445414238062138, + "learning_rate": 7.192952655724049e-06, + "loss": 1.6597, + "step": 356 + }, + { + "epoch": 1.5710102489019033, + "grad_norm": 0.5860196285629298, + "learning_rate": 7.176485145436325e-06, + "loss": 1.5387, + "step": 357 + }, + { + "epoch": 1.5754026354319182, + "grad_norm": 0.3537384385833807, + "learning_rate": 7.159988455477534e-06, + "loss": 1.6013, + "step": 358 + }, + { + "epoch": 1.5797950219619326, + "grad_norm": 0.3286829503374976, + "learning_rate": 7.143462807015271e-06, + "loss": 1.6264, + "step": 359 + }, + { + "epoch": 1.5841874084919474, + "grad_norm": 0.3197218822335203, + "learning_rate": 7.1269084216053756e-06, + "loss": 1.5552, + "step": 360 + }, + { + "epoch": 1.5885797950219618, + "grad_norm": 0.31209150272027425, + "learning_rate": 7.11032552118895e-06, + "loss": 1.6108, + "step": 361 + }, + { + "epoch": 1.5929721815519766, + "grad_norm": 0.3227458575298131, + "learning_rate": 7.093714328089398e-06, + "loss": 1.649, + "step": 362 + }, + { + "epoch": 1.5973645680819912, + "grad_norm": 0.3637374362635078, + "learning_rate": 7.0770750650094335e-06, + "loss": 1.6739, + "step": 363 + }, + { + "epoch": 1.6017569546120058, + "grad_norm": 0.31236210751966154, + "learning_rate": 7.060407955028098e-06, + "loss": 1.5946, + "step": 364 + }, + { + "epoch": 1.6061493411420205, + "grad_norm": 0.35651724986710415, + "learning_rate": 7.0437132215977744e-06, + "loss": 1.6337, + "step": 365 + }, + { + "epoch": 1.610541727672035, + "grad_norm": 0.31143483852268733, + "learning_rate": 7.026991088541184e-06, + "loss": 1.67, + "step": 366 + }, + { + "epoch": 1.61493411420205, + "grad_norm": 0.33050222076540253, + "learning_rate": 7.01024178004839e-06, + "loss": 1.6204, + "step": 367 + }, + { + "epoch": 1.6193265007320643, + "grad_norm": 0.3284547022539121, + "learning_rate": 6.99346552067379e-06, + "loss": 1.573, + "step": 368 + }, + { + "epoch": 1.6237188872620791, + "grad_norm": 0.3603027916184282, + "learning_rate": 6.976662535333107e-06, + "loss": 1.6553, + "step": 369 + }, + { + "epoch": 1.6281112737920937, + "grad_norm": 0.8725979374632243, + "learning_rate": 6.959833049300376e-06, + "loss": 1.6279, + "step": 370 + }, + { + "epoch": 1.6325036603221084, + "grad_norm": 0.28597465971107255, + "learning_rate": 6.942977288204915e-06, + "loss": 1.5074, + "step": 371 + }, + { + "epoch": 1.636896046852123, + "grad_norm": 0.31388416897723714, + "learning_rate": 6.926095478028312e-06, + "loss": 1.6896, + "step": 372 + }, + { + "epoch": 1.6412884333821376, + "grad_norm": 0.3008107505835411, + "learning_rate": 6.909187845101387e-06, + "loss": 1.5688, + "step": 373 + }, + { + "epoch": 1.6456808199121524, + "grad_norm": 0.32252769956105753, + "learning_rate": 6.892254616101158e-06, + "loss": 1.7391, + "step": 374 + }, + { + "epoch": 1.6500732064421668, + "grad_norm": 0.3253332219188239, + "learning_rate": 6.87529601804781e-06, + "loss": 1.5479, + "step": 375 + }, + { + "epoch": 1.6544655929721817, + "grad_norm": 0.3354516401894849, + "learning_rate": 6.858312278301638e-06, + "loss": 1.5944, + "step": 376 + }, + { + "epoch": 1.658857979502196, + "grad_norm": 0.3101157670728667, + "learning_rate": 6.841303624560012e-06, + "loss": 1.5979, + "step": 377 + }, + { + "epoch": 1.6632503660322109, + "grad_norm": 0.3263157294243148, + "learning_rate": 6.8242702848543185e-06, + "loss": 1.6651, + "step": 378 + }, + { + "epoch": 1.6676427525622255, + "grad_norm": 0.2963170265612591, + "learning_rate": 6.807212487546897e-06, + "loss": 1.6414, + "step": 379 + }, + { + "epoch": 1.67203513909224, + "grad_norm": 0.3436329546570268, + "learning_rate": 6.790130461327993e-06, + "loss": 1.5237, + "step": 380 + }, + { + "epoch": 1.6764275256222547, + "grad_norm": 0.30951635226199037, + "learning_rate": 6.773024435212678e-06, + "loss": 1.5608, + "step": 381 + }, + { + "epoch": 1.6808199121522693, + "grad_norm": 0.3024875350743093, + "learning_rate": 6.755894638537791e-06, + "loss": 1.5595, + "step": 382 + }, + { + "epoch": 1.6852122986822842, + "grad_norm": 0.30964409063235376, + "learning_rate": 6.73874130095885e-06, + "loss": 1.6166, + "step": 383 + }, + { + "epoch": 1.6896046852122986, + "grad_norm": 0.31495863587091266, + "learning_rate": 6.721564652446987e-06, + "loss": 1.5279, + "step": 384 + }, + { + "epoch": 1.6939970717423134, + "grad_norm": 0.3115937718660612, + "learning_rate": 6.704364923285858e-06, + "loss": 1.5694, + "step": 385 + }, + { + "epoch": 1.698389458272328, + "grad_norm": 0.29768001900234425, + "learning_rate": 6.687142344068552e-06, + "loss": 1.5578, + "step": 386 + }, + { + "epoch": 1.7027818448023426, + "grad_norm": 0.32146567067323123, + "learning_rate": 6.669897145694507e-06, + "loss": 1.6368, + "step": 387 + }, + { + "epoch": 1.7071742313323572, + "grad_norm": 0.29522637791127904, + "learning_rate": 6.6526295593664136e-06, + "loss": 1.6121, + "step": 388 + }, + { + "epoch": 1.7115666178623719, + "grad_norm": 0.30160303491140916, + "learning_rate": 6.635339816587109e-06, + "loss": 1.6238, + "step": 389 + }, + { + "epoch": 1.7159590043923867, + "grad_norm": 0.4208009773656739, + "learning_rate": 6.618028149156479e-06, + "loss": 1.62, + "step": 390 + }, + { + "epoch": 1.720351390922401, + "grad_norm": 0.2881943555017053, + "learning_rate": 6.600694789168345e-06, + "loss": 1.6364, + "step": 391 + }, + { + "epoch": 1.724743777452416, + "grad_norm": 0.3077307710744344, + "learning_rate": 6.583339969007364e-06, + "loss": 1.5677, + "step": 392 + }, + { + "epoch": 1.7291361639824303, + "grad_norm": 0.35235096673751176, + "learning_rate": 6.565963921345896e-06, + "loss": 1.6163, + "step": 393 + }, + { + "epoch": 1.7335285505124451, + "grad_norm": 0.3186476335255873, + "learning_rate": 6.548566879140897e-06, + "loss": 1.6289, + "step": 394 + }, + { + "epoch": 1.7379209370424598, + "grad_norm": 0.31476614898776, + "learning_rate": 6.531149075630796e-06, + "loss": 1.5874, + "step": 395 + }, + { + "epoch": 1.7423133235724744, + "grad_norm": 0.3469536206500234, + "learning_rate": 6.513710744332361e-06, + "loss": 1.6621, + "step": 396 + }, + { + "epoch": 1.746705710102489, + "grad_norm": 0.30680438680917854, + "learning_rate": 6.49625211903757e-06, + "loss": 1.595, + "step": 397 + }, + { + "epoch": 1.7510980966325036, + "grad_norm": 0.34458603376189034, + "learning_rate": 6.478773433810478e-06, + "loss": 1.5886, + "step": 398 + }, + { + "epoch": 1.7554904831625184, + "grad_norm": 0.34083953962031827, + "learning_rate": 6.461274922984087e-06, + "loss": 1.5705, + "step": 399 + }, + { + "epoch": 1.7598828696925328, + "grad_norm": 0.3345685083562738, + "learning_rate": 6.4437568211571864e-06, + "loss": 1.6181, + "step": 400 + }, + { + "epoch": 1.7642752562225477, + "grad_norm": 0.28767837558253834, + "learning_rate": 6.426219363191224e-06, + "loss": 1.6273, + "step": 401 + }, + { + "epoch": 1.7686676427525623, + "grad_norm": 0.3309056934700509, + "learning_rate": 6.408662784207149e-06, + "loss": 1.5575, + "step": 402 + }, + { + "epoch": 1.7730600292825769, + "grad_norm": 0.33936552783693097, + "learning_rate": 6.391087319582264e-06, + "loss": 1.5653, + "step": 403 + }, + { + "epoch": 1.7774524158125915, + "grad_norm": 0.38728001848935395, + "learning_rate": 6.373493204947065e-06, + "loss": 1.5962, + "step": 404 + }, + { + "epoch": 1.7818448023426061, + "grad_norm": 0.33731817557741206, + "learning_rate": 6.355880676182086e-06, + "loss": 1.5267, + "step": 405 + }, + { + "epoch": 1.786237188872621, + "grad_norm": 0.3073610397325029, + "learning_rate": 6.3382499694147345e-06, + "loss": 1.5735, + "step": 406 + }, + { + "epoch": 1.7906295754026353, + "grad_norm": 0.5666929994713482, + "learning_rate": 6.3206013210161285e-06, + "loss": 1.6164, + "step": 407 + }, + { + "epoch": 1.7950219619326502, + "grad_norm": 0.3423706928777578, + "learning_rate": 6.302934967597922e-06, + "loss": 1.6694, + "step": 408 + }, + { + "epoch": 1.7994143484626646, + "grad_norm": 0.3100272828530065, + "learning_rate": 6.2852511460091406e-06, + "loss": 1.6322, + "step": 409 + }, + { + "epoch": 1.8038067349926794, + "grad_norm": 0.3710603928040935, + "learning_rate": 6.267550093333e-06, + "loss": 1.5405, + "step": 410 + }, + { + "epoch": 1.808199121522694, + "grad_norm": 0.3024234235421662, + "learning_rate": 6.249832046883729e-06, + "loss": 1.6389, + "step": 411 + }, + { + "epoch": 1.8125915080527086, + "grad_norm": 0.2923050985231585, + "learning_rate": 6.232097244203388e-06, + "loss": 1.563, + "step": 412 + }, + { + "epoch": 1.8169838945827232, + "grad_norm": 0.37742465142395615, + "learning_rate": 6.214345923058686e-06, + "loss": 1.5605, + "step": 413 + }, + { + "epoch": 1.8213762811127379, + "grad_norm": 0.2763099605085667, + "learning_rate": 6.1965783214377895e-06, + "loss": 1.5757, + "step": 414 + }, + { + "epoch": 1.8257686676427527, + "grad_norm": 0.29226253974509314, + "learning_rate": 6.178794677547138e-06, + "loss": 1.6213, + "step": 415 + }, + { + "epoch": 1.830161054172767, + "grad_norm": 0.3892056822790433, + "learning_rate": 6.16099522980824e-06, + "loss": 1.6471, + "step": 416 + }, + { + "epoch": 1.834553440702782, + "grad_norm": 0.3547953314773269, + "learning_rate": 6.143180216854488e-06, + "loss": 1.5933, + "step": 417 + }, + { + "epoch": 1.8389458272327965, + "grad_norm": 0.2876417607221798, + "learning_rate": 6.125349877527952e-06, + "loss": 1.6056, + "step": 418 + }, + { + "epoch": 1.8433382137628112, + "grad_norm": 0.29632892867662236, + "learning_rate": 6.1075044508761804e-06, + "loss": 1.5798, + "step": 419 + }, + { + "epoch": 1.8477306002928258, + "grad_norm": 0.3351398469787539, + "learning_rate": 6.0896441761489925e-06, + "loss": 1.5775, + "step": 420 + }, + { + "epoch": 1.8521229868228404, + "grad_norm": 0.32572738949136165, + "learning_rate": 6.071769292795274e-06, + "loss": 1.5065, + "step": 421 + }, + { + "epoch": 1.8565153733528552, + "grad_norm": 0.32922431311553646, + "learning_rate": 6.053880040459765e-06, + "loss": 1.5768, + "step": 422 + }, + { + "epoch": 1.8609077598828696, + "grad_norm": 0.3229086221064645, + "learning_rate": 6.035976658979846e-06, + "loss": 1.6242, + "step": 423 + }, + { + "epoch": 1.8653001464128844, + "grad_norm": 0.3114352291982662, + "learning_rate": 6.018059388382327e-06, + "loss": 1.6473, + "step": 424 + }, + { + "epoch": 1.8696925329428988, + "grad_norm": 0.31045445243058734, + "learning_rate": 6.000128468880223e-06, + "loss": 1.535, + "step": 425 + }, + { + "epoch": 1.8740849194729137, + "grad_norm": 0.33523008129662585, + "learning_rate": 5.982184140869539e-06, + "loss": 1.6344, + "step": 426 + }, + { + "epoch": 1.8784773060029283, + "grad_norm": 0.3081522900734568, + "learning_rate": 5.964226644926045e-06, + "loss": 1.6386, + "step": 427 + }, + { + "epoch": 1.882869692532943, + "grad_norm": 0.2900227624443104, + "learning_rate": 5.946256221802052e-06, + "loss": 1.5843, + "step": 428 + }, + { + "epoch": 1.8872620790629575, + "grad_norm": 0.354099020882711, + "learning_rate": 5.928273112423177e-06, + "loss": 1.5758, + "step": 429 + }, + { + "epoch": 1.8916544655929721, + "grad_norm": 0.34160211290628684, + "learning_rate": 5.910277557885128e-06, + "loss": 1.5551, + "step": 430 + }, + { + "epoch": 1.896046852122987, + "grad_norm": 0.3162945077147358, + "learning_rate": 5.892269799450453e-06, + "loss": 1.6204, + "step": 431 + }, + { + "epoch": 1.9004392386530014, + "grad_norm": 0.3509407679369851, + "learning_rate": 5.8742500785453226e-06, + "loss": 1.6224, + "step": 432 + }, + { + "epoch": 1.9048316251830162, + "grad_norm": 0.2848983911128527, + "learning_rate": 5.856218636756281e-06, + "loss": 1.6059, + "step": 433 + }, + { + "epoch": 1.9092240117130308, + "grad_norm": 0.38193324539008777, + "learning_rate": 5.838175715827015e-06, + "loss": 1.5511, + "step": 434 + }, + { + "epoch": 1.9136163982430454, + "grad_norm": 0.31934508340416906, + "learning_rate": 5.820121557655109e-06, + "loss": 1.7028, + "step": 435 + }, + { + "epoch": 1.91800878477306, + "grad_norm": 0.33670681289920085, + "learning_rate": 5.8020564042888015e-06, + "loss": 1.5687, + "step": 436 + }, + { + "epoch": 1.9224011713030746, + "grad_norm": 0.318844847843566, + "learning_rate": 5.783980497923743e-06, + "loss": 1.567, + "step": 437 + }, + { + "epoch": 1.9267935578330895, + "grad_norm": 0.29610949062035014, + "learning_rate": 5.76589408089974e-06, + "loss": 1.5166, + "step": 438 + }, + { + "epoch": 1.9311859443631039, + "grad_norm": 0.28728684262143106, + "learning_rate": 5.747797395697525e-06, + "loss": 1.5813, + "step": 439 + }, + { + "epoch": 1.9355783308931187, + "grad_norm": 0.32309431631814955, + "learning_rate": 5.729690684935487e-06, + "loss": 1.641, + "step": 440 + }, + { + "epoch": 1.939970717423133, + "grad_norm": 0.2969130322032886, + "learning_rate": 5.711574191366427e-06, + "loss": 1.5608, + "step": 441 + }, + { + "epoch": 1.944363103953148, + "grad_norm": 0.42872939564364393, + "learning_rate": 5.693448157874299e-06, + "loss": 1.5983, + "step": 442 + }, + { + "epoch": 1.9487554904831625, + "grad_norm": 0.3798270075015977, + "learning_rate": 5.675312827470959e-06, + "loss": 1.5896, + "step": 443 + }, + { + "epoch": 1.9531478770131772, + "grad_norm": 0.30174438725527203, + "learning_rate": 5.657168443292909e-06, + "loss": 1.6043, + "step": 444 + }, + { + "epoch": 1.9575402635431918, + "grad_norm": 0.4119857865917871, + "learning_rate": 5.639015248598025e-06, + "loss": 1.5754, + "step": 445 + }, + { + "epoch": 1.9619326500732064, + "grad_norm": 0.32221313787015915, + "learning_rate": 5.620853486762307e-06, + "loss": 1.6569, + "step": 446 + }, + { + "epoch": 1.9663250366032212, + "grad_norm": 0.3547729959901477, + "learning_rate": 5.6026834012766155e-06, + "loss": 1.6228, + "step": 447 + }, + { + "epoch": 1.9707174231332356, + "grad_norm": 0.28279646337718173, + "learning_rate": 5.584505235743404e-06, + "loss": 1.5874, + "step": 448 + }, + { + "epoch": 1.9751098096632504, + "grad_norm": 0.3264218725678291, + "learning_rate": 5.566319233873446e-06, + "loss": 1.6313, + "step": 449 + }, + { + "epoch": 1.979502196193265, + "grad_norm": 0.28955653787826696, + "learning_rate": 5.548125639482587e-06, + "loss": 1.5989, + "step": 450 + }, + { + "epoch": 1.9838945827232797, + "grad_norm": 0.881097673855719, + "learning_rate": 5.529924696488456e-06, + "loss": 1.5796, + "step": 451 + }, + { + "epoch": 1.9882869692532943, + "grad_norm": 0.32048202284049887, + "learning_rate": 5.5117166489072014e-06, + "loss": 1.578, + "step": 452 + }, + { + "epoch": 1.992679355783309, + "grad_norm": 0.2896791795835667, + "learning_rate": 5.493501740850228e-06, + "loss": 1.6146, + "step": 453 + }, + { + "epoch": 1.9970717423133237, + "grad_norm": 0.35142442191521966, + "learning_rate": 5.475280216520913e-06, + "loss": 1.5522, + "step": 454 + }, + { + "epoch": 2.004392386530015, + "grad_norm": 0.6164551609296155, + "learning_rate": 5.45705232021134e-06, + "loss": 3.059, + "step": 455 + }, + { + "epoch": 2.0087847730600292, + "grad_norm": 0.33044889056247584, + "learning_rate": 5.438818296299015e-06, + "loss": 1.5982, + "step": 456 + } + ], + "logging_steps": 1, + "max_steps": 908, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 114, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2060458640914514e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-456/training_args.bin b/checkpoint-456/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..282808fde3b579abe8666eba7a8b5732d5eaf3d4 --- /dev/null +++ b/checkpoint-456/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f598588716f36b5db7091989142a430a4fd275bce6f1e0d8e6c17f595272e1ec +size 9041 diff --git a/checkpoint-456/zero_to_fp32.py b/checkpoint-456/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint-456/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoint-570/README.md b/checkpoint-570/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb5cf17f38ede36f3fb46a89995daa3f44167898 --- /dev/null +++ b/checkpoint-570/README.md @@ -0,0 +1,202 @@ +--- +base_model: Gryphe/Codex-24B-Small-3.2 +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/checkpoint-570/adapter_config.json b/checkpoint-570/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa0535ca25a4eef78d9c51576f13234c64c95fb --- /dev/null +++ b/checkpoint-570/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Gryphe/Codex-24B-Small-3.2", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.5, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "o_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/checkpoint-570/adapter_model.safetensors b/checkpoint-570/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b01da38c3667c965052e258cd837db281b2b0012 --- /dev/null +++ b/checkpoint-570/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a9f141f348f72b528162cb577a0d8c657fda394ac1ab696f91f99ac2c1f64cc +size 1478569432 diff --git a/checkpoint-570/global_step570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-570/global_step570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb49476ee71f22c120355822de6a0f10bc894f68 --- /dev/null +++ b/checkpoint-570/global_step570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889244b5f6c7e86ce818be8067213e46387f5e3f61ec4fd531405754ffb8fc83 +size 4459517761 diff --git a/checkpoint-570/global_step570/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-570/global_step570/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..308493e54d41104f383467274182b8f3cfc46ef7 --- /dev/null +++ b/checkpoint-570/global_step570/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e9a748ed2ac4dda2c9f3ee324252b9f4fa777c6265fd122ea5e213c730ccdf +size 557481 diff --git a/checkpoint-570/latest b/checkpoint-570/latest new file mode 100644 index 0000000000000000000000000000000000000000..0433d1c81a4b69bdd8533de1f0573850078819c8 --- /dev/null +++ b/checkpoint-570/latest @@ -0,0 +1 @@ +global_step570 \ No newline at end of file diff --git a/checkpoint-570/rng_state.pth b/checkpoint-570/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..6783313964d1faa282f193eb91dd42529f7c18ea --- /dev/null +++ b/checkpoint-570/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7fd0793e58de5669c1d25b5b9959c87020e8bda27d8446e46995387e3dfc06b +size 14709 diff --git a/checkpoint-570/scheduler.pt b/checkpoint-570/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..77e008a081ed1ef155ccf6f01d24f01ea87d016e --- /dev/null +++ b/checkpoint-570/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ebb975a51dcfbe2d034002dc39db3e6d4d5b8acd046c47cf2b4b7ede80bf991 +size 1401 diff --git a/checkpoint-570/special_tokens_map.json b/checkpoint-570/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..4332a4c6d2a485806f55e02260db8a73698e11fb --- /dev/null +++ b/checkpoint-570/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-570/tokenizer.json b/checkpoint-570/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8fab5451b04e019d1612cc5088df62aee9a8208c --- /dev/null +++ b/checkpoint-570/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e5daf1e619597fd56ff036925ea189916417c665d5bdd61b80199cf8d4e6d2 +size 17078029 diff --git a/checkpoint-570/tokenizer_config.json b/checkpoint-570/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..814201fe59040b12e956fb649c749a42bdce1e2f --- /dev/null +++ b/checkpoint-570/tokenizer_config.json @@ -0,0 +1,8021 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "[IMG]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "[IMG_BREAK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "[IMG_END]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "[PREFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "[MIDDLE]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "[SUFFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "[SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "[/SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "[TOOL_CONTENT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "109": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "110": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "111": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "112": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "113": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "114": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "115": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "116": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "117": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "118": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "119": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "120": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "121": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "122": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "123": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "124": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "125": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "126": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "127": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "129": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "130": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "132": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "133": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "134": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "135": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "136": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "137": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "138": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "139": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "140": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "141": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "142": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "143": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "144": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "145": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "146": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "147": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "148": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "149": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "150": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "152": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "153": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "154": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "155": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "156": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "157": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "158": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "159": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "160": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "161": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "162": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "164": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "165": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "166": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "167": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "185": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "186": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "187": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "188": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "189": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "190": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "191": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "222": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "223": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "224": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "225": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "226": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "227": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "228": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "229": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "230": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "231": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "232": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "233": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "234": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "235": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "236": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "237": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "238": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "239": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "240": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "241": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "242": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "243": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "244": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "245": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "246": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "247": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "249": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "251": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "252": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "253": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "254": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "255": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "257": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "258": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "259": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "260": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "262": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "263": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "264": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "265": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "266": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "267": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "268": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "285": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "286": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "287": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "288": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "289": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "290": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "291": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "292": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "293": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "294": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "295": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "296": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "297": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "298": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "299": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "300": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "301": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "302": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "303": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "304": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "305": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "306": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "307": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "308": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "309": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "310": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "311": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "312": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "313": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "314": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "315": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "316": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "317": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "318": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "319": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "320": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "321": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "322": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "323": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "324": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "325": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "326": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "327": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "328": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "329": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "330": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "331": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "332": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "335": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "336": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "337": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "338": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "339": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "340": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "341": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "342": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "343": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "344": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "345": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "346": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "347": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "348": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "349": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "360": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "361": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "362": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "363": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "364": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "365": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "366": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "367": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "368": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "369": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "370": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "371": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "372": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "373": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "374": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "375": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "376": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "377": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "378": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "379": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "380": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "381": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "382": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "383": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "384": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "385": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "386": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "387": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "388": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "389": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "390": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "391": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "392": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "393": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "394": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "395": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "396": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "397": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "398": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "399": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "400": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "401": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "402": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "403": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "404": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "405": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "406": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "407": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "408": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "409": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "410": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "411": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "412": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "413": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "414": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "415": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "416": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "417": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "418": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "419": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "420": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "421": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "422": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "423": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "424": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "425": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "426": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "427": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "428": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "429": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "430": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "431": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "432": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "433": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "434": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "435": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "436": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "437": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "438": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "439": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "440": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "441": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "442": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "443": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "444": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "445": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "446": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "447": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "448": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "449": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "450": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "451": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "452": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "453": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "454": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "455": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "456": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "457": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "458": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "459": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "460": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "461": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "462": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "463": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "464": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "465": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "466": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "467": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "468": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "469": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "470": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "471": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "472": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "473": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "474": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "475": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "476": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "477": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "478": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "479": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "480": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "481": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "482": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "483": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "484": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "485": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "486": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "487": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "488": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "489": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "490": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "491": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "492": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "493": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "494": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "495": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "496": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "497": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "498": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "499": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "500": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "501": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "502": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "503": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "504": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "505": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "506": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "507": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "508": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "509": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "510": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "511": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "512": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "513": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "514": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "515": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "516": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "517": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "518": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "519": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "520": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "521": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "522": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "523": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "524": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "525": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "526": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "527": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "528": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "529": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "530": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "531": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "532": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "533": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "534": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "535": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "536": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "537": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "538": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "539": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "540": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "541": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "542": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "543": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "544": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "545": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "546": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "547": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "548": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "549": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "550": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "551": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "552": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "553": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "554": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "555": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "556": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "557": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "558": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "559": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "560": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "561": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "562": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "563": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "564": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "565": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "566": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "567": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "568": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "569": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "570": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "571": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "572": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "573": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "574": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "575": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "576": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "577": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "578": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "579": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "580": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "581": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "582": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "583": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "584": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "585": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "586": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "587": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "588": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "589": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "590": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "591": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "592": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "593": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "594": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "595": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "596": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "597": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "598": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "599": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "600": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "601": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "602": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "603": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "604": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "605": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "606": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "607": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "608": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "609": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "610": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "611": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "612": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "613": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "614": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "615": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "616": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "617": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "618": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "619": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "620": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "621": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "622": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "623": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "624": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "625": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "626": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "627": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "628": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "629": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "630": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "631": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "632": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "633": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "634": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "635": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "636": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "637": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "638": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "639": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "640": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "641": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "642": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "643": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "644": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "645": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "646": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "647": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "650": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "651": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "652": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "653": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "654": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "655": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "656": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "659": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "660": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "661": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "662": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "663": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "664": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "675": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "676": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "677": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "678": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "679": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "680": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "681": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "682": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "683": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "684": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "685": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "686": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "687": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "688": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "689": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "690": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "691": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "692": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "693": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "694": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "695": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "696": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "697": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "698": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "699": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "700": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "701": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "702": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "703": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "704": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "705": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "706": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "707": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "708": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "709": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "710": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "711": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "712": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "713": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "714": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "715": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "716": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "717": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "718": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "719": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "720": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "721": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "722": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "723": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "724": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "725": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "726": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "727": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "728": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "729": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "730": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "731": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "732": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "733": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "734": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "735": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "736": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "737": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "738": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "739": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "740": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "741": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "742": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "743": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "744": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "745": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "746": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "747": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "748": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "749": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "750": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "751": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "752": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "753": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "754": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "755": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "756": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "757": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "758": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "759": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "760": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "761": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "762": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "763": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "764": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "765": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "766": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "767": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "768": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "769": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "770": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "771": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "772": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "773": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "774": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "775": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "776": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "777": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "778": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "779": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "780": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "781": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "782": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "783": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "784": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "785": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "786": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "787": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "788": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "789": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "790": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "791": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "792": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "793": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "794": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "795": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "796": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "797": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "798": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "799": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "800": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "801": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "802": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "803": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "804": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "805": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "806": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "807": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "808": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "809": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "810": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "811": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "812": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "813": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "814": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "815": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "816": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "817": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "818": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "819": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "820": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "821": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "822": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "823": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "824": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "825": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "826": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "827": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "828": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "829": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "830": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "831": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "832": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "833": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "834": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "835": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "836": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "837": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "838": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "839": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "840": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "841": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "842": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "843": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "844": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "845": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "846": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "847": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "848": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "849": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "850": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "851": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "852": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "853": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "854": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "855": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "856": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "857": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "863": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "864": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "865": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "866": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "867": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "868": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "869": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "870": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "871": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "880": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "881": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "882": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "883": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "884": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "885": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "886": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "887": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "888": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "889": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "890": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "891": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "892": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "893": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "894": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "895": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "896": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "897": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "898": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "899": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "900": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "901": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "902": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "903": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "904": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "905": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "906": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "907": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "908": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "909": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "910": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "911": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "912": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "913": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "914": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "915": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "916": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "917": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "918": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "919": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "920": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "921": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "922": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "923": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "924": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "925": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "926": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "927": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "928": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "929": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "930": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "931": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "932": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "933": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "934": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "935": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "936": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "937": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "938": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "939": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "940": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "941": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "942": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "943": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "944": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "945": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "946": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "947": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "948": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "949": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "950": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "951": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "952": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "953": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "954": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "955": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "956": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "957": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "958": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "959": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "960": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "961": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "962": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "963": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "964": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "965": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "966": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "967": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "968": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "969": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "970": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "971": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "972": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "973": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "974": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "975": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "976": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "977": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "978": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "979": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "980": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "981": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "982": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "983": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "984": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "985": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "986": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "987": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "988": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "989": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "990": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "991": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "992": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "993": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "994": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "995": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "996": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "997": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "998": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "999": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 131072, + "pad_token": "", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-570/trainer_state.json b/checkpoint-570/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d108e52b434818452c3d2cec68a2dcff26858aee --- /dev/null +++ b/checkpoint-570/trainer_state.json @@ -0,0 +1,4024 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.5095168374816983, + "eval_steps": 500, + "global_step": 570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004392386530014641, + "grad_norm": 2.023870796411983, + "learning_rate": 0.0, + "loss": 1.8021, + "step": 1 + }, + { + "epoch": 0.008784773060029283, + "grad_norm": 2.0723948799697984, + "learning_rate": 2.0000000000000002e-07, + "loss": 1.8796, + "step": 2 + }, + { + "epoch": 0.013177159590043924, + "grad_norm": 2.0322569672979474, + "learning_rate": 4.0000000000000003e-07, + "loss": 1.855, + "step": 3 + }, + { + "epoch": 0.017569546120058566, + "grad_norm": 2.111472782868084, + "learning_rate": 6.000000000000001e-07, + "loss": 1.8545, + "step": 4 + }, + { + "epoch": 0.021961932650073207, + "grad_norm": 2.096607607233168, + "learning_rate": 8.000000000000001e-07, + "loss": 1.807, + "step": 5 + }, + { + "epoch": 0.02635431918008785, + "grad_norm": 2.0771855831079518, + "learning_rate": 1.0000000000000002e-06, + "loss": 1.8208, + "step": 6 + }, + { + "epoch": 0.03074670571010249, + "grad_norm": 2.0341840619969567, + "learning_rate": 1.2000000000000002e-06, + "loss": 1.8754, + "step": 7 + }, + { + "epoch": 0.03513909224011713, + "grad_norm": 2.0564329237094356, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.8882, + "step": 8 + }, + { + "epoch": 0.03953147877013177, + "grad_norm": 1.9900074702634307, + "learning_rate": 1.6000000000000001e-06, + "loss": 1.8733, + "step": 9 + }, + { + "epoch": 0.043923865300146414, + "grad_norm": 1.895599415585885, + "learning_rate": 1.8000000000000001e-06, + "loss": 1.8474, + "step": 10 + }, + { + "epoch": 0.048316251830161056, + "grad_norm": 1.9304209419460994, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.884, + "step": 11 + }, + { + "epoch": 0.0527086383601757, + "grad_norm": 1.6511439041472846, + "learning_rate": 2.2e-06, + "loss": 1.8452, + "step": 12 + }, + { + "epoch": 0.05710102489019034, + "grad_norm": 1.6727753891125912, + "learning_rate": 2.4000000000000003e-06, + "loss": 1.8632, + "step": 13 + }, + { + "epoch": 0.06149341142020498, + "grad_norm": 1.6571693399048197, + "learning_rate": 2.6e-06, + "loss": 1.8419, + "step": 14 + }, + { + "epoch": 0.06588579795021962, + "grad_norm": 1.5588712723347087, + "learning_rate": 2.8000000000000003e-06, + "loss": 1.8163, + "step": 15 + }, + { + "epoch": 0.07027818448023426, + "grad_norm": 1.57706405792705, + "learning_rate": 3e-06, + "loss": 1.7808, + "step": 16 + }, + { + "epoch": 0.0746705710102489, + "grad_norm": 1.4841447475095337, + "learning_rate": 3.2000000000000003e-06, + "loss": 1.7854, + "step": 17 + }, + { + "epoch": 0.07906295754026355, + "grad_norm": 1.2785291394155553, + "learning_rate": 3.4000000000000005e-06, + "loss": 1.7531, + "step": 18 + }, + { + "epoch": 0.08345534407027819, + "grad_norm": 1.1759630521885338, + "learning_rate": 3.6000000000000003e-06, + "loss": 1.8185, + "step": 19 + }, + { + "epoch": 0.08784773060029283, + "grad_norm": 1.1188613513306425, + "learning_rate": 3.8000000000000005e-06, + "loss": 1.849, + "step": 20 + }, + { + "epoch": 0.09224011713030747, + "grad_norm": 1.008741263333265, + "learning_rate": 4.000000000000001e-06, + "loss": 1.775, + "step": 21 + }, + { + "epoch": 0.09663250366032211, + "grad_norm": 0.9240991279618896, + "learning_rate": 4.2000000000000004e-06, + "loss": 1.8253, + "step": 22 + }, + { + "epoch": 0.10102489019033675, + "grad_norm": 0.9015845652170666, + "learning_rate": 4.4e-06, + "loss": 1.7967, + "step": 23 + }, + { + "epoch": 0.1054172767203514, + "grad_norm": 0.8007281301128234, + "learning_rate": 4.600000000000001e-06, + "loss": 1.7958, + "step": 24 + }, + { + "epoch": 0.10980966325036604, + "grad_norm": 0.7263390923205475, + "learning_rate": 4.800000000000001e-06, + "loss": 1.7744, + "step": 25 + }, + { + "epoch": 0.11420204978038068, + "grad_norm": 0.6527771074756341, + "learning_rate": 5e-06, + "loss": 1.7502, + "step": 26 + }, + { + "epoch": 0.11859443631039532, + "grad_norm": 0.5943189060332841, + "learning_rate": 5.2e-06, + "loss": 1.7378, + "step": 27 + }, + { + "epoch": 0.12298682284040996, + "grad_norm": 0.5426800655727773, + "learning_rate": 5.400000000000001e-06, + "loss": 1.7372, + "step": 28 + }, + { + "epoch": 0.1273792093704246, + "grad_norm": 0.46887045607953654, + "learning_rate": 5.600000000000001e-06, + "loss": 1.7141, + "step": 29 + }, + { + "epoch": 0.13177159590043924, + "grad_norm": 0.4408809706079413, + "learning_rate": 5.8e-06, + "loss": 1.6977, + "step": 30 + }, + { + "epoch": 0.13616398243045388, + "grad_norm": 0.4867065604810947, + "learning_rate": 6e-06, + "loss": 1.7379, + "step": 31 + }, + { + "epoch": 0.14055636896046853, + "grad_norm": 0.40882779662256746, + "learning_rate": 6.200000000000001e-06, + "loss": 1.8064, + "step": 32 + }, + { + "epoch": 0.14494875549048317, + "grad_norm": 0.4363561706655908, + "learning_rate": 6.4000000000000006e-06, + "loss": 1.6835, + "step": 33 + }, + { + "epoch": 0.1493411420204978, + "grad_norm": 0.4443411222267533, + "learning_rate": 6.600000000000001e-06, + "loss": 1.7464, + "step": 34 + }, + { + "epoch": 0.15373352855051245, + "grad_norm": 0.4519702758206936, + "learning_rate": 6.800000000000001e-06, + "loss": 1.6891, + "step": 35 + }, + { + "epoch": 0.1581259150805271, + "grad_norm": 0.4174999907173025, + "learning_rate": 7e-06, + "loss": 1.7262, + "step": 36 + }, + { + "epoch": 0.16251830161054173, + "grad_norm": 0.46742969885702906, + "learning_rate": 7.2000000000000005e-06, + "loss": 1.7608, + "step": 37 + }, + { + "epoch": 0.16691068814055637, + "grad_norm": 0.4155692294314683, + "learning_rate": 7.4e-06, + "loss": 1.7211, + "step": 38 + }, + { + "epoch": 0.17130307467057102, + "grad_norm": 0.4123169421407387, + "learning_rate": 7.600000000000001e-06, + "loss": 1.6956, + "step": 39 + }, + { + "epoch": 0.17569546120058566, + "grad_norm": 0.3887606381731466, + "learning_rate": 7.800000000000002e-06, + "loss": 1.7391, + "step": 40 + }, + { + "epoch": 0.1800878477306003, + "grad_norm": 0.37535703391318914, + "learning_rate": 8.000000000000001e-06, + "loss": 1.751, + "step": 41 + }, + { + "epoch": 0.18448023426061494, + "grad_norm": 0.48621902751310403, + "learning_rate": 8.2e-06, + "loss": 1.7091, + "step": 42 + }, + { + "epoch": 0.18887262079062958, + "grad_norm": 0.39145172998871297, + "learning_rate": 8.400000000000001e-06, + "loss": 1.7764, + "step": 43 + }, + { + "epoch": 0.19326500732064422, + "grad_norm": 0.3735488902443606, + "learning_rate": 8.6e-06, + "loss": 1.7058, + "step": 44 + }, + { + "epoch": 0.19765739385065886, + "grad_norm": 0.33988576479414434, + "learning_rate": 8.8e-06, + "loss": 1.7305, + "step": 45 + }, + { + "epoch": 0.2020497803806735, + "grad_norm": 0.9733163465798257, + "learning_rate": 9e-06, + "loss": 1.7567, + "step": 46 + }, + { + "epoch": 0.20644216691068815, + "grad_norm": 0.3344321176956884, + "learning_rate": 9.200000000000002e-06, + "loss": 1.7277, + "step": 47 + }, + { + "epoch": 0.2108345534407028, + "grad_norm": 0.3647805237752918, + "learning_rate": 9.4e-06, + "loss": 1.7197, + "step": 48 + }, + { + "epoch": 0.21522693997071743, + "grad_norm": 0.36967036325253694, + "learning_rate": 9.600000000000001e-06, + "loss": 1.6994, + "step": 49 + }, + { + "epoch": 0.21961932650073207, + "grad_norm": 0.32386928338357696, + "learning_rate": 9.800000000000001e-06, + "loss": 1.6975, + "step": 50 + }, + { + "epoch": 0.2240117130307467, + "grad_norm": 0.33940794357965215, + "learning_rate": 1e-05, + "loss": 1.7278, + "step": 51 + }, + { + "epoch": 0.22840409956076135, + "grad_norm": 0.3396058372359069, + "learning_rate": 9.999966483034437e-06, + "loss": 1.6342, + "step": 52 + }, + { + "epoch": 0.232796486090776, + "grad_norm": 0.3395876900960321, + "learning_rate": 9.999865932587107e-06, + "loss": 1.6846, + "step": 53 + }, + { + "epoch": 0.23718887262079064, + "grad_norm": 0.3320830942903407, + "learning_rate": 9.999698350006063e-06, + "loss": 1.6744, + "step": 54 + }, + { + "epoch": 0.24158125915080528, + "grad_norm": 0.318642304519145, + "learning_rate": 9.999463737538054e-06, + "loss": 1.6633, + "step": 55 + }, + { + "epoch": 0.24597364568081992, + "grad_norm": 0.3302002388842341, + "learning_rate": 9.999162098328475e-06, + "loss": 1.6994, + "step": 56 + }, + { + "epoch": 0.25036603221083453, + "grad_norm": 0.30428798634415904, + "learning_rate": 9.998793436421342e-06, + "loss": 1.6728, + "step": 57 + }, + { + "epoch": 0.2547584187408492, + "grad_norm": 0.30662673120989614, + "learning_rate": 9.998357756759223e-06, + "loss": 1.6283, + "step": 58 + }, + { + "epoch": 0.2591508052708638, + "grad_norm": 0.34305953470241257, + "learning_rate": 9.997855065183185e-06, + "loss": 1.7269, + "step": 59 + }, + { + "epoch": 0.2635431918008785, + "grad_norm": 0.29647698247102616, + "learning_rate": 9.997285368432704e-06, + "loss": 1.6108, + "step": 60 + }, + { + "epoch": 0.2679355783308931, + "grad_norm": 0.30904144706299363, + "learning_rate": 9.996648674145584e-06, + "loss": 1.5925, + "step": 61 + }, + { + "epoch": 0.27232796486090777, + "grad_norm": 0.28294691041282544, + "learning_rate": 9.995944990857848e-06, + "loss": 1.7004, + "step": 62 + }, + { + "epoch": 0.2767203513909224, + "grad_norm": 0.4537316624659888, + "learning_rate": 9.99517432800363e-06, + "loss": 1.673, + "step": 63 + }, + { + "epoch": 0.28111273792093705, + "grad_norm": 0.33334142649617554, + "learning_rate": 9.994336695915041e-06, + "loss": 1.7305, + "step": 64 + }, + { + "epoch": 0.28550512445095166, + "grad_norm": 0.2906595328556503, + "learning_rate": 9.993432105822034e-06, + "loss": 1.6511, + "step": 65 + }, + { + "epoch": 0.28989751098096633, + "grad_norm": 0.30937353104293036, + "learning_rate": 9.992460569852256e-06, + "loss": 1.7055, + "step": 66 + }, + { + "epoch": 0.29428989751098095, + "grad_norm": 0.29714918983678074, + "learning_rate": 9.99142210103088e-06, + "loss": 1.6022, + "step": 67 + }, + { + "epoch": 0.2986822840409956, + "grad_norm": 0.3077061364082642, + "learning_rate": 9.99031671328044e-06, + "loss": 1.6167, + "step": 68 + }, + { + "epoch": 0.30307467057101023, + "grad_norm": 0.31797999416909106, + "learning_rate": 9.98914442142063e-06, + "loss": 1.6063, + "step": 69 + }, + { + "epoch": 0.3074670571010249, + "grad_norm": 0.28544810434107143, + "learning_rate": 9.987905241168117e-06, + "loss": 1.673, + "step": 70 + }, + { + "epoch": 0.3118594436310395, + "grad_norm": 0.3113516293182134, + "learning_rate": 9.986599189136325e-06, + "loss": 1.6759, + "step": 71 + }, + { + "epoch": 0.3162518301610542, + "grad_norm": 0.32958924621680397, + "learning_rate": 9.985226282835216e-06, + "loss": 1.6894, + "step": 72 + }, + { + "epoch": 0.3206442166910688, + "grad_norm": 0.3189944736085633, + "learning_rate": 9.983786540671052e-06, + "loss": 1.7762, + "step": 73 + }, + { + "epoch": 0.32503660322108346, + "grad_norm": 0.30229970638681786, + "learning_rate": 9.982279981946143e-06, + "loss": 1.709, + "step": 74 + }, + { + "epoch": 0.3294289897510981, + "grad_norm": 0.308085411523194, + "learning_rate": 9.980706626858607e-06, + "loss": 1.6741, + "step": 75 + }, + { + "epoch": 0.33382137628111275, + "grad_norm": 0.2851270169881767, + "learning_rate": 9.979066496502075e-06, + "loss": 1.656, + "step": 76 + }, + { + "epoch": 0.33821376281112736, + "grad_norm": 0.3115977673015286, + "learning_rate": 9.977359612865424e-06, + "loss": 1.6633, + "step": 77 + }, + { + "epoch": 0.34260614934114203, + "grad_norm": 0.2960787226189042, + "learning_rate": 9.97558599883248e-06, + "loss": 1.673, + "step": 78 + }, + { + "epoch": 0.34699853587115664, + "grad_norm": 0.35701812130442995, + "learning_rate": 9.973745678181705e-06, + "loss": 1.6103, + "step": 79 + }, + { + "epoch": 0.3513909224011713, + "grad_norm": 0.367868955812032, + "learning_rate": 9.971838675585888e-06, + "loss": 1.6768, + "step": 80 + }, + { + "epoch": 0.3557833089311859, + "grad_norm": 0.3295428259298103, + "learning_rate": 9.9698650166118e-06, + "loss": 1.6668, + "step": 81 + }, + { + "epoch": 0.3601756954612006, + "grad_norm": 0.3242883305567894, + "learning_rate": 9.96782472771987e-06, + "loss": 1.7152, + "step": 82 + }, + { + "epoch": 0.3645680819912152, + "grad_norm": 0.3070622417895325, + "learning_rate": 9.965717836263813e-06, + "loss": 1.5697, + "step": 83 + }, + { + "epoch": 0.3689604685212299, + "grad_norm": 0.27753629978682975, + "learning_rate": 9.96354437049027e-06, + "loss": 1.6453, + "step": 84 + }, + { + "epoch": 0.3733528550512445, + "grad_norm": 0.2723902519890988, + "learning_rate": 9.961304359538437e-06, + "loss": 1.7165, + "step": 85 + }, + { + "epoch": 0.37774524158125916, + "grad_norm": 0.29061586618994084, + "learning_rate": 9.958997833439658e-06, + "loss": 1.7135, + "step": 86 + }, + { + "epoch": 0.3821376281112738, + "grad_norm": 0.33085661251847487, + "learning_rate": 9.956624823117036e-06, + "loss": 1.6109, + "step": 87 + }, + { + "epoch": 0.38653001464128844, + "grad_norm": 0.2931612046141165, + "learning_rate": 9.954185360385013e-06, + "loss": 1.7048, + "step": 88 + }, + { + "epoch": 0.39092240117130306, + "grad_norm": 0.2900600158100116, + "learning_rate": 9.951679477948946e-06, + "loss": 1.6861, + "step": 89 + }, + { + "epoch": 0.3953147877013177, + "grad_norm": 0.2775954202644971, + "learning_rate": 9.949107209404664e-06, + "loss": 1.5706, + "step": 90 + }, + { + "epoch": 0.39970717423133234, + "grad_norm": 0.2733537235090247, + "learning_rate": 9.946468589238022e-06, + "loss": 1.7146, + "step": 91 + }, + { + "epoch": 0.404099560761347, + "grad_norm": 0.2710139549067813, + "learning_rate": 9.943763652824437e-06, + "loss": 1.6488, + "step": 92 + }, + { + "epoch": 0.4084919472913616, + "grad_norm": 0.27892641103107996, + "learning_rate": 9.94099243642841e-06, + "loss": 1.6002, + "step": 93 + }, + { + "epoch": 0.4128843338213763, + "grad_norm": 0.4570340740318857, + "learning_rate": 9.93815497720305e-06, + "loss": 1.6686, + "step": 94 + }, + { + "epoch": 0.4172767203513909, + "grad_norm": 0.2816582371174757, + "learning_rate": 9.935251313189564e-06, + "loss": 1.7531, + "step": 95 + }, + { + "epoch": 0.4216691068814056, + "grad_norm": 0.2886345479986427, + "learning_rate": 9.932281483316759e-06, + "loss": 1.744, + "step": 96 + }, + { + "epoch": 0.4260614934114202, + "grad_norm": 0.3178449956639589, + "learning_rate": 9.929245527400504e-06, + "loss": 1.6404, + "step": 97 + }, + { + "epoch": 0.43045387994143486, + "grad_norm": 0.31466032997182714, + "learning_rate": 9.926143486143216e-06, + "loss": 1.6583, + "step": 98 + }, + { + "epoch": 0.43484626647144947, + "grad_norm": 0.3406321749653024, + "learning_rate": 9.922975401133292e-06, + "loss": 1.7313, + "step": 99 + }, + { + "epoch": 0.43923865300146414, + "grad_norm": 0.33050248521276526, + "learning_rate": 9.919741314844577e-06, + "loss": 1.6518, + "step": 100 + }, + { + "epoch": 0.44363103953147875, + "grad_norm": 0.2886134577231967, + "learning_rate": 9.916441270635772e-06, + "loss": 1.6957, + "step": 101 + }, + { + "epoch": 0.4480234260614934, + "grad_norm": 0.27520985458354463, + "learning_rate": 9.913075312749867e-06, + "loss": 1.6052, + "step": 102 + }, + { + "epoch": 0.45241581259150804, + "grad_norm": 0.2649316531653303, + "learning_rate": 9.909643486313533e-06, + "loss": 1.6739, + "step": 103 + }, + { + "epoch": 0.4568081991215227, + "grad_norm": 0.27557834576379425, + "learning_rate": 9.90614583733654e-06, + "loss": 1.6584, + "step": 104 + }, + { + "epoch": 0.4612005856515373, + "grad_norm": 0.3243046224133678, + "learning_rate": 9.90258241271112e-06, + "loss": 1.6491, + "step": 105 + }, + { + "epoch": 0.465592972181552, + "grad_norm": 0.2601090357711318, + "learning_rate": 9.89895326021134e-06, + "loss": 1.6662, + "step": 106 + }, + { + "epoch": 0.4699853587115666, + "grad_norm": 0.2842151241183655, + "learning_rate": 9.895258428492475e-06, + "loss": 1.6501, + "step": 107 + }, + { + "epoch": 0.4743777452415813, + "grad_norm": 0.2594313373745908, + "learning_rate": 9.891497967090344e-06, + "loss": 1.6475, + "step": 108 + }, + { + "epoch": 0.4787701317715959, + "grad_norm": 0.2844945420500575, + "learning_rate": 9.887671926420649e-06, + "loss": 1.7204, + "step": 109 + }, + { + "epoch": 0.48316251830161056, + "grad_norm": 0.27655451441999607, + "learning_rate": 9.8837803577783e-06, + "loss": 1.6838, + "step": 110 + }, + { + "epoch": 0.48755490483162517, + "grad_norm": 0.3017150137441088, + "learning_rate": 9.879823313336723e-06, + "loss": 1.6688, + "step": 111 + }, + { + "epoch": 0.49194729136163984, + "grad_norm": 0.26575250791745125, + "learning_rate": 9.87580084614717e-06, + "loss": 1.6753, + "step": 112 + }, + { + "epoch": 0.49633967789165445, + "grad_norm": 0.2581989783149807, + "learning_rate": 9.871713010137998e-06, + "loss": 1.6244, + "step": 113 + }, + { + "epoch": 0.5007320644216691, + "grad_norm": 0.2622294231157268, + "learning_rate": 9.86755986011395e-06, + "loss": 1.6261, + "step": 114 + }, + { + "epoch": 0.5051244509516838, + "grad_norm": 0.26209513370076304, + "learning_rate": 9.86334145175542e-06, + "loss": 1.6397, + "step": 115 + }, + { + "epoch": 0.5095168374816984, + "grad_norm": 0.2846184522412747, + "learning_rate": 9.859057841617709e-06, + "loss": 1.7091, + "step": 116 + }, + { + "epoch": 0.513909224011713, + "grad_norm": 0.2528850826310284, + "learning_rate": 9.854709087130261e-06, + "loss": 1.6831, + "step": 117 + }, + { + "epoch": 0.5183016105417276, + "grad_norm": 0.32698350420324157, + "learning_rate": 9.850295246595898e-06, + "loss": 1.6431, + "step": 118 + }, + { + "epoch": 0.5226939970717424, + "grad_norm": 0.35727071053972576, + "learning_rate": 9.845816379190037e-06, + "loss": 1.7088, + "step": 119 + }, + { + "epoch": 0.527086383601757, + "grad_norm": 0.28971742333278044, + "learning_rate": 9.841272544959892e-06, + "loss": 1.7195, + "step": 120 + }, + { + "epoch": 0.5314787701317716, + "grad_norm": 0.468869869211724, + "learning_rate": 9.836663804823683e-06, + "loss": 1.6896, + "step": 121 + }, + { + "epoch": 0.5358711566617862, + "grad_norm": 0.3908944943187778, + "learning_rate": 9.831990220569801e-06, + "loss": 1.7406, + "step": 122 + }, + { + "epoch": 0.5402635431918009, + "grad_norm": 0.2665924461613152, + "learning_rate": 9.827251854855992e-06, + "loss": 1.6609, + "step": 123 + }, + { + "epoch": 0.5446559297218155, + "grad_norm": 0.28211372742262864, + "learning_rate": 9.82244877120851e-06, + "loss": 1.6684, + "step": 124 + }, + { + "epoch": 0.5490483162518301, + "grad_norm": 0.2623738132657017, + "learning_rate": 9.817581034021273e-06, + "loss": 1.6923, + "step": 125 + }, + { + "epoch": 0.5534407027818448, + "grad_norm": 0.35357676308356323, + "learning_rate": 9.812648708554991e-06, + "loss": 1.6467, + "step": 126 + }, + { + "epoch": 0.5578330893118595, + "grad_norm": 0.2555639734647961, + "learning_rate": 9.807651860936299e-06, + "loss": 1.6893, + "step": 127 + }, + { + "epoch": 0.5622254758418741, + "grad_norm": 0.26746261958597517, + "learning_rate": 9.802590558156863e-06, + "loss": 1.6754, + "step": 128 + }, + { + "epoch": 0.5666178623718887, + "grad_norm": 0.26383031628885084, + "learning_rate": 9.797464868072489e-06, + "loss": 1.6363, + "step": 129 + }, + { + "epoch": 0.5710102489019033, + "grad_norm": 0.319619389966322, + "learning_rate": 9.792274859402205e-06, + "loss": 1.636, + "step": 130 + }, + { + "epoch": 0.575402635431918, + "grad_norm": 0.2639308579990406, + "learning_rate": 9.787020601727353e-06, + "loss": 1.6079, + "step": 131 + }, + { + "epoch": 0.5797950219619327, + "grad_norm": 0.28226380411742236, + "learning_rate": 9.781702165490638e-06, + "loss": 1.657, + "step": 132 + }, + { + "epoch": 0.5841874084919473, + "grad_norm": 0.27157003543418035, + "learning_rate": 9.7763196219952e-06, + "loss": 1.7009, + "step": 133 + }, + { + "epoch": 0.5885797950219619, + "grad_norm": 0.28487209604498265, + "learning_rate": 9.770873043403648e-06, + "loss": 1.6236, + "step": 134 + }, + { + "epoch": 0.5929721815519766, + "grad_norm": 0.3009729998770739, + "learning_rate": 9.765362502737098e-06, + "loss": 1.6812, + "step": 135 + }, + { + "epoch": 0.5973645680819912, + "grad_norm": 0.2576811873616972, + "learning_rate": 9.759788073874188e-06, + "loss": 1.6885, + "step": 136 + }, + { + "epoch": 0.6017569546120058, + "grad_norm": 0.3108368566509119, + "learning_rate": 9.754149831550097e-06, + "loss": 1.6086, + "step": 137 + }, + { + "epoch": 0.6061493411420205, + "grad_norm": 0.3336050806293322, + "learning_rate": 9.748447851355534e-06, + "loss": 1.6773, + "step": 138 + }, + { + "epoch": 0.6105417276720352, + "grad_norm": 0.28918467659013913, + "learning_rate": 9.742682209735727e-06, + "loss": 1.7141, + "step": 139 + }, + { + "epoch": 0.6149341142020498, + "grad_norm": 0.25507889874035206, + "learning_rate": 9.736852983989405e-06, + "loss": 1.6335, + "step": 140 + }, + { + "epoch": 0.6193265007320644, + "grad_norm": 0.2712795343500641, + "learning_rate": 9.730960252267744e-06, + "loss": 1.6186, + "step": 141 + }, + { + "epoch": 0.623718887262079, + "grad_norm": 0.29326296437100124, + "learning_rate": 9.725004093573343e-06, + "loss": 1.6039, + "step": 142 + }, + { + "epoch": 0.6281112737920937, + "grad_norm": 0.27412689936526563, + "learning_rate": 9.718984587759148e-06, + "loss": 1.594, + "step": 143 + }, + { + "epoch": 0.6325036603221084, + "grad_norm": 0.28908560688778157, + "learning_rate": 9.712901815527387e-06, + "loss": 1.6458, + "step": 144 + }, + { + "epoch": 0.636896046852123, + "grad_norm": 0.26538330998431864, + "learning_rate": 9.706755858428487e-06, + "loss": 1.6546, + "step": 145 + }, + { + "epoch": 0.6412884333821376, + "grad_norm": 0.26498403141947585, + "learning_rate": 9.70054679885998e-06, + "loss": 1.6224, + "step": 146 + }, + { + "epoch": 0.6456808199121523, + "grad_norm": 0.25028323667878993, + "learning_rate": 9.6942747200654e-06, + "loss": 1.6554, + "step": 147 + }, + { + "epoch": 0.6500732064421669, + "grad_norm": 0.3077620507501807, + "learning_rate": 9.687939706133168e-06, + "loss": 1.6751, + "step": 148 + }, + { + "epoch": 0.6544655929721815, + "grad_norm": 0.2738251492188122, + "learning_rate": 9.68154184199546e-06, + "loss": 1.6139, + "step": 149 + }, + { + "epoch": 0.6588579795021962, + "grad_norm": 0.300766059001253, + "learning_rate": 9.675081213427076e-06, + "loss": 1.6783, + "step": 150 + }, + { + "epoch": 0.6632503660322109, + "grad_norm": 0.2671963968809055, + "learning_rate": 9.668557907044278e-06, + "loss": 1.7028, + "step": 151 + }, + { + "epoch": 0.6676427525622255, + "grad_norm": 0.27596191143138304, + "learning_rate": 9.661972010303641e-06, + "loss": 1.6279, + "step": 152 + }, + { + "epoch": 0.6720351390922401, + "grad_norm": 0.2719766311854951, + "learning_rate": 9.655323611500876e-06, + "loss": 1.6634, + "step": 153 + }, + { + "epoch": 0.6764275256222547, + "grad_norm": 0.2697612116557389, + "learning_rate": 9.648612799769644e-06, + "loss": 1.617, + "step": 154 + }, + { + "epoch": 0.6808199121522694, + "grad_norm": 0.28137116472179047, + "learning_rate": 9.641839665080363e-06, + "loss": 1.6212, + "step": 155 + }, + { + "epoch": 0.6852122986822841, + "grad_norm": 0.2599526010890989, + "learning_rate": 9.635004298239004e-06, + "loss": 1.6542, + "step": 156 + }, + { + "epoch": 0.6896046852122987, + "grad_norm": 0.662388825119619, + "learning_rate": 9.628106790885866e-06, + "loss": 1.633, + "step": 157 + }, + { + "epoch": 0.6939970717423133, + "grad_norm": 0.2913416852335614, + "learning_rate": 9.621147235494357e-06, + "loss": 1.6904, + "step": 158 + }, + { + "epoch": 0.698389458272328, + "grad_norm": 0.26581120890113497, + "learning_rate": 9.614125725369748e-06, + "loss": 1.6583, + "step": 159 + }, + { + "epoch": 0.7027818448023426, + "grad_norm": 0.26696083510238133, + "learning_rate": 9.607042354647924e-06, + "loss": 1.6354, + "step": 160 + }, + { + "epoch": 0.7071742313323572, + "grad_norm": 0.2913753113559095, + "learning_rate": 9.599897218294122e-06, + "loss": 1.5914, + "step": 161 + }, + { + "epoch": 0.7115666178623719, + "grad_norm": 0.2794317973029827, + "learning_rate": 9.59269041210166e-06, + "loss": 1.5732, + "step": 162 + }, + { + "epoch": 0.7159590043923866, + "grad_norm": 0.27364398897711945, + "learning_rate": 9.585422032690644e-06, + "loss": 1.6796, + "step": 163 + }, + { + "epoch": 0.7203513909224012, + "grad_norm": 0.2924137561125512, + "learning_rate": 9.578092177506683e-06, + "loss": 1.6921, + "step": 164 + }, + { + "epoch": 0.7247437774524158, + "grad_norm": 0.27864643658701665, + "learning_rate": 9.570700944819584e-06, + "loss": 1.6442, + "step": 165 + }, + { + "epoch": 0.7291361639824304, + "grad_norm": 0.3172135018133218, + "learning_rate": 9.56324843372202e-06, + "loss": 1.7003, + "step": 166 + }, + { + "epoch": 0.7335285505124451, + "grad_norm": 0.33119379016577744, + "learning_rate": 9.55573474412821e-06, + "loss": 1.6277, + "step": 167 + }, + { + "epoch": 0.7379209370424598, + "grad_norm": 0.24315818847358012, + "learning_rate": 9.548159976772593e-06, + "loss": 1.621, + "step": 168 + }, + { + "epoch": 0.7423133235724744, + "grad_norm": 0.28066914391093295, + "learning_rate": 9.540524233208449e-06, + "loss": 1.6169, + "step": 169 + }, + { + "epoch": 0.746705710102489, + "grad_norm": 0.3096372829142638, + "learning_rate": 9.532827615806561e-06, + "loss": 1.6425, + "step": 170 + }, + { + "epoch": 0.7510980966325037, + "grad_norm": 0.3050362838238551, + "learning_rate": 9.525070227753835e-06, + "loss": 1.5835, + "step": 171 + }, + { + "epoch": 0.7554904831625183, + "grad_norm": 0.27558198897100084, + "learning_rate": 9.517252173051912e-06, + "loss": 1.6866, + "step": 172 + }, + { + "epoch": 0.7598828696925329, + "grad_norm": 0.3415215102331636, + "learning_rate": 9.509373556515781e-06, + "loss": 1.6954, + "step": 173 + }, + { + "epoch": 0.7642752562225475, + "grad_norm": 0.2833145266859935, + "learning_rate": 9.501434483772371e-06, + "loss": 1.5642, + "step": 174 + }, + { + "epoch": 0.7686676427525623, + "grad_norm": 0.261080979929468, + "learning_rate": 9.49343506125913e-06, + "loss": 1.6481, + "step": 175 + }, + { + "epoch": 0.7730600292825769, + "grad_norm": 0.27528584249968596, + "learning_rate": 9.48537539622261e-06, + "loss": 1.6356, + "step": 176 + }, + { + "epoch": 0.7774524158125915, + "grad_norm": 0.28493642680744213, + "learning_rate": 9.477255596717012e-06, + "loss": 1.617, + "step": 177 + }, + { + "epoch": 0.7818448023426061, + "grad_norm": 0.250792323656803, + "learning_rate": 9.469075771602756e-06, + "loss": 1.7373, + "step": 178 + }, + { + "epoch": 0.7862371888726208, + "grad_norm": 0.2815707976300464, + "learning_rate": 9.460836030545007e-06, + "loss": 1.7134, + "step": 179 + }, + { + "epoch": 0.7906295754026355, + "grad_norm": 0.33965220402585167, + "learning_rate": 9.452536484012212e-06, + "loss": 1.66, + "step": 180 + }, + { + "epoch": 0.7950219619326501, + "grad_norm": 0.337075115730111, + "learning_rate": 9.444177243274619e-06, + "loss": 1.7237, + "step": 181 + }, + { + "epoch": 0.7994143484626647, + "grad_norm": 0.2652773355794464, + "learning_rate": 9.43575842040278e-06, + "loss": 1.7238, + "step": 182 + }, + { + "epoch": 0.8038067349926794, + "grad_norm": 0.29569868187991866, + "learning_rate": 9.427280128266049e-06, + "loss": 1.6484, + "step": 183 + }, + { + "epoch": 0.808199121522694, + "grad_norm": 0.2647366968721245, + "learning_rate": 9.418742480531086e-06, + "loss": 1.6557, + "step": 184 + }, + { + "epoch": 0.8125915080527086, + "grad_norm": 0.2624692157687208, + "learning_rate": 9.410145591660301e-06, + "loss": 1.6501, + "step": 185 + }, + { + "epoch": 0.8169838945827232, + "grad_norm": 0.2604406630602592, + "learning_rate": 9.40148957691035e-06, + "loss": 1.6205, + "step": 186 + }, + { + "epoch": 0.821376281112738, + "grad_norm": 0.3809448031526879, + "learning_rate": 9.392774552330567e-06, + "loss": 1.6014, + "step": 187 + }, + { + "epoch": 0.8257686676427526, + "grad_norm": 0.285104899301775, + "learning_rate": 9.384000634761429e-06, + "loss": 1.7119, + "step": 188 + }, + { + "epoch": 0.8301610541727672, + "grad_norm": 0.3603217046093286, + "learning_rate": 9.375167941832974e-06, + "loss": 1.6129, + "step": 189 + }, + { + "epoch": 0.8345534407027818, + "grad_norm": 0.26192371812461096, + "learning_rate": 9.366276591963222e-06, + "loss": 1.6648, + "step": 190 + }, + { + "epoch": 0.8389458272327965, + "grad_norm": 0.27224376529649075, + "learning_rate": 9.357326704356602e-06, + "loss": 1.7368, + "step": 191 + }, + { + "epoch": 0.8433382137628112, + "grad_norm": 0.267560205737413, + "learning_rate": 9.348318399002347e-06, + "loss": 1.6456, + "step": 192 + }, + { + "epoch": 0.8477306002928258, + "grad_norm": 0.38772390686795116, + "learning_rate": 9.339251796672878e-06, + "loss": 1.6765, + "step": 193 + }, + { + "epoch": 0.8521229868228404, + "grad_norm": 0.28624234868841525, + "learning_rate": 9.330127018922195e-06, + "loss": 1.6397, + "step": 194 + }, + { + "epoch": 0.8565153733528551, + "grad_norm": 0.3517826456437289, + "learning_rate": 9.320944188084241e-06, + "loss": 1.6504, + "step": 195 + }, + { + "epoch": 0.8609077598828697, + "grad_norm": 0.3054896957644684, + "learning_rate": 9.31170342727127e-06, + "loss": 1.6581, + "step": 196 + }, + { + "epoch": 0.8653001464128843, + "grad_norm": 0.47217080493290625, + "learning_rate": 9.302404860372185e-06, + "loss": 1.6614, + "step": 197 + }, + { + "epoch": 0.8696925329428989, + "grad_norm": 0.3744290659075852, + "learning_rate": 9.293048612050883e-06, + "loss": 1.6181, + "step": 198 + }, + { + "epoch": 0.8740849194729137, + "grad_norm": 0.2928672407192581, + "learning_rate": 9.283634807744586e-06, + "loss": 1.5801, + "step": 199 + }, + { + "epoch": 0.8784773060029283, + "grad_norm": 0.5500989172351091, + "learning_rate": 9.274163573662158e-06, + "loss": 1.6213, + "step": 200 + }, + { + "epoch": 0.8828696925329429, + "grad_norm": 0.27844961793523915, + "learning_rate": 9.264635036782406e-06, + "loss": 1.6442, + "step": 201 + }, + { + "epoch": 0.8872620790629575, + "grad_norm": 0.2764381586339949, + "learning_rate": 9.255049324852388e-06, + "loss": 1.6403, + "step": 202 + }, + { + "epoch": 0.8916544655929722, + "grad_norm": 0.28692831842946026, + "learning_rate": 9.245406566385698e-06, + "loss": 1.6204, + "step": 203 + }, + { + "epoch": 0.8960468521229868, + "grad_norm": 0.2854958274623322, + "learning_rate": 9.235706890660735e-06, + "loss": 1.6777, + "step": 204 + }, + { + "epoch": 0.9004392386530015, + "grad_norm": 0.2884758073862357, + "learning_rate": 9.225950427718974e-06, + "loss": 1.6414, + "step": 205 + }, + { + "epoch": 0.9048316251830161, + "grad_norm": 0.2774591809427237, + "learning_rate": 9.216137308363235e-06, + "loss": 1.667, + "step": 206 + }, + { + "epoch": 0.9092240117130308, + "grad_norm": 0.26327408771476135, + "learning_rate": 9.206267664155906e-06, + "loss": 1.6472, + "step": 207 + }, + { + "epoch": 0.9136163982430454, + "grad_norm": 0.3163508666559799, + "learning_rate": 9.1963416274172e-06, + "loss": 1.6898, + "step": 208 + }, + { + "epoch": 0.91800878477306, + "grad_norm": 0.2691312569989342, + "learning_rate": 9.18635933122337e-06, + "loss": 1.6788, + "step": 209 + }, + { + "epoch": 0.9224011713030746, + "grad_norm": 0.329963898141559, + "learning_rate": 9.176320909404925e-06, + "loss": 1.7077, + "step": 210 + }, + { + "epoch": 0.9267935578330894, + "grad_norm": 0.3497378648580076, + "learning_rate": 9.16622649654484e-06, + "loss": 1.7203, + "step": 211 + }, + { + "epoch": 0.931185944363104, + "grad_norm": 0.2750435891659104, + "learning_rate": 9.156076227976752e-06, + "loss": 1.6372, + "step": 212 + }, + { + "epoch": 0.9355783308931186, + "grad_norm": 0.29344217135995226, + "learning_rate": 9.145870239783143e-06, + "loss": 1.6294, + "step": 213 + }, + { + "epoch": 0.9399707174231332, + "grad_norm": 0.2713649627402965, + "learning_rate": 9.135608668793511e-06, + "loss": 1.6391, + "step": 214 + }, + { + "epoch": 0.9443631039531479, + "grad_norm": 0.2995344678229402, + "learning_rate": 9.125291652582549e-06, + "loss": 1.6102, + "step": 215 + }, + { + "epoch": 0.9487554904831625, + "grad_norm": 0.32763546183738235, + "learning_rate": 9.114919329468283e-06, + "loss": 1.6823, + "step": 216 + }, + { + "epoch": 0.9531478770131772, + "grad_norm": 0.29620265372375704, + "learning_rate": 9.104491838510235e-06, + "loss": 1.6905, + "step": 217 + }, + { + "epoch": 0.9575402635431918, + "grad_norm": 0.43487598587006066, + "learning_rate": 9.094009319507547e-06, + "loss": 1.6611, + "step": 218 + }, + { + "epoch": 0.9619326500732065, + "grad_norm": 0.32989859860270077, + "learning_rate": 9.08347191299711e-06, + "loss": 1.634, + "step": 219 + }, + { + "epoch": 0.9663250366032211, + "grad_norm": 0.26622257061624277, + "learning_rate": 9.07287976025168e-06, + "loss": 1.6075, + "step": 220 + }, + { + "epoch": 0.9707174231332357, + "grad_norm": 0.28626358026286186, + "learning_rate": 9.062233003277983e-06, + "loss": 1.6045, + "step": 221 + }, + { + "epoch": 0.9751098096632503, + "grad_norm": 0.5424158794876999, + "learning_rate": 9.051531784814817e-06, + "loss": 1.5825, + "step": 222 + }, + { + "epoch": 0.9795021961932651, + "grad_norm": 0.2771987597060821, + "learning_rate": 9.040776248331128e-06, + "loss": 1.6083, + "step": 223 + }, + { + "epoch": 0.9838945827232797, + "grad_norm": 1.5311321947246777, + "learning_rate": 9.029966538024097e-06, + "loss": 1.5973, + "step": 224 + }, + { + "epoch": 0.9882869692532943, + "grad_norm": 0.3081246423484545, + "learning_rate": 9.019102798817196e-06, + "loss": 1.5915, + "step": 225 + }, + { + "epoch": 0.9926793557833089, + "grad_norm": 0.3066770476121554, + "learning_rate": 9.008185176358257e-06, + "loss": 1.5551, + "step": 226 + }, + { + "epoch": 0.9970717423133236, + "grad_norm": 0.29888292004137484, + "learning_rate": 8.997213817017508e-06, + "loss": 1.7461, + "step": 227 + }, + { + "epoch": 1.0043923865300146, + "grad_norm": 0.5938110979809961, + "learning_rate": 8.986188867885617e-06, + "loss": 3.2509, + "step": 228 + }, + { + "epoch": 1.0087847730600292, + "grad_norm": 0.2899326916339286, + "learning_rate": 8.975110476771724e-06, + "loss": 1.5874, + "step": 229 + }, + { + "epoch": 1.0131771595900438, + "grad_norm": 0.2777095345419943, + "learning_rate": 8.96397879220145e-06, + "loss": 1.6706, + "step": 230 + }, + { + "epoch": 1.0175695461200585, + "grad_norm": 0.26363195429841585, + "learning_rate": 8.952793963414908e-06, + "loss": 1.6317, + "step": 231 + }, + { + "epoch": 1.0219619326500733, + "grad_norm": 0.2688971491502818, + "learning_rate": 8.941556140364707e-06, + "loss": 1.654, + "step": 232 + }, + { + "epoch": 1.026354319180088, + "grad_norm": 0.28487309366468816, + "learning_rate": 8.930265473713939e-06, + "loss": 1.6133, + "step": 233 + }, + { + "epoch": 1.0307467057101025, + "grad_norm": 0.28141744313455036, + "learning_rate": 8.918922114834156e-06, + "loss": 1.589, + "step": 234 + }, + { + "epoch": 1.0351390922401171, + "grad_norm": 0.40308880749849646, + "learning_rate": 8.907526215803351e-06, + "loss": 1.6499, + "step": 235 + }, + { + "epoch": 1.0395314787701317, + "grad_norm": 0.2834055854316979, + "learning_rate": 8.896077929403901e-06, + "loss": 1.658, + "step": 236 + }, + { + "epoch": 1.0439238653001464, + "grad_norm": 0.2887035123274668, + "learning_rate": 8.884577409120535e-06, + "loss": 1.6196, + "step": 237 + }, + { + "epoch": 1.048316251830161, + "grad_norm": 0.2985213212589398, + "learning_rate": 8.873024809138272e-06, + "loss": 1.677, + "step": 238 + }, + { + "epoch": 1.0527086383601758, + "grad_norm": 0.2863532205939482, + "learning_rate": 8.861420284340352e-06, + "loss": 1.6682, + "step": 239 + }, + { + "epoch": 1.0571010248901904, + "grad_norm": 0.29235163398244823, + "learning_rate": 8.849763990306153e-06, + "loss": 1.6009, + "step": 240 + }, + { + "epoch": 1.061493411420205, + "grad_norm": 0.2899336107020558, + "learning_rate": 8.838056083309118e-06, + "loss": 1.6319, + "step": 241 + }, + { + "epoch": 1.0658857979502196, + "grad_norm": 0.3186837918366928, + "learning_rate": 8.826296720314658e-06, + "loss": 1.5875, + "step": 242 + }, + { + "epoch": 1.0702781844802343, + "grad_norm": 0.33938164798671433, + "learning_rate": 8.814486058978035e-06, + "loss": 1.6059, + "step": 243 + }, + { + "epoch": 1.0746705710102489, + "grad_norm": 0.2991115627129074, + "learning_rate": 8.802624257642262e-06, + "loss": 1.6599, + "step": 244 + }, + { + "epoch": 1.0790629575402635, + "grad_norm": 0.28608565865487656, + "learning_rate": 8.790711475335971e-06, + "loss": 1.6364, + "step": 245 + }, + { + "epoch": 1.083455344070278, + "grad_norm": 0.28359019806003394, + "learning_rate": 8.778747871771293e-06, + "loss": 1.6584, + "step": 246 + }, + { + "epoch": 1.0878477306002927, + "grad_norm": 0.34292617248780644, + "learning_rate": 8.766733607341697e-06, + "loss": 1.6613, + "step": 247 + }, + { + "epoch": 1.0922401171303076, + "grad_norm": 0.3469366526291968, + "learning_rate": 8.754668843119865e-06, + "loss": 1.6542, + "step": 248 + }, + { + "epoch": 1.0966325036603222, + "grad_norm": 0.2745752678931537, + "learning_rate": 8.742553740855507e-06, + "loss": 1.6723, + "step": 249 + }, + { + "epoch": 1.1010248901903368, + "grad_norm": 0.3863733971251328, + "learning_rate": 8.730388462973209e-06, + "loss": 1.6268, + "step": 250 + }, + { + "epoch": 1.1054172767203514, + "grad_norm": 0.3196278576328997, + "learning_rate": 8.718173172570254e-06, + "loss": 1.6349, + "step": 251 + }, + { + "epoch": 1.109809663250366, + "grad_norm": 0.27017487934331, + "learning_rate": 8.705908033414426e-06, + "loss": 1.606, + "step": 252 + }, + { + "epoch": 1.1142020497803806, + "grad_norm": 0.6576768607622604, + "learning_rate": 8.693593209941826e-06, + "loss": 1.5936, + "step": 253 + }, + { + "epoch": 1.1185944363103952, + "grad_norm": 0.30260640047372234, + "learning_rate": 8.681228867254657e-06, + "loss": 1.5948, + "step": 254 + }, + { + "epoch": 1.12298682284041, + "grad_norm": 0.3102239482584757, + "learning_rate": 8.66881517111902e-06, + "loss": 1.6928, + "step": 255 + }, + { + "epoch": 1.1273792093704247, + "grad_norm": 0.2818931187575453, + "learning_rate": 8.656352287962687e-06, + "loss": 1.687, + "step": 256 + }, + { + "epoch": 1.1317715959004393, + "grad_norm": 0.30007928577375903, + "learning_rate": 8.643840384872865e-06, + "loss": 1.6628, + "step": 257 + }, + { + "epoch": 1.136163982430454, + "grad_norm": 0.3249258384227434, + "learning_rate": 8.631279629593968e-06, + "loss": 1.6123, + "step": 258 + }, + { + "epoch": 1.1405563689604685, + "grad_norm": 0.2871446068255736, + "learning_rate": 8.61867019052535e-06, + "loss": 1.6608, + "step": 259 + }, + { + "epoch": 1.1449487554904831, + "grad_norm": 0.3001183609451796, + "learning_rate": 8.606012236719073e-06, + "loss": 1.6405, + "step": 260 + }, + { + "epoch": 1.1493411420204978, + "grad_norm": 0.271155672303218, + "learning_rate": 8.593305937877614e-06, + "loss": 1.5819, + "step": 261 + }, + { + "epoch": 1.1537335285505124, + "grad_norm": 0.30999460151747915, + "learning_rate": 8.580551464351603e-06, + "loss": 1.6378, + "step": 262 + }, + { + "epoch": 1.158125915080527, + "grad_norm": 0.2979330701414515, + "learning_rate": 8.567748987137544e-06, + "loss": 1.7342, + "step": 263 + }, + { + "epoch": 1.1625183016105418, + "grad_norm": 0.29037637226651547, + "learning_rate": 8.554898677875509e-06, + "loss": 1.663, + "step": 264 + }, + { + "epoch": 1.1669106881405564, + "grad_norm": 0.29574080692093535, + "learning_rate": 8.542000708846851e-06, + "loss": 1.6937, + "step": 265 + }, + { + "epoch": 1.171303074670571, + "grad_norm": 0.2867463319690563, + "learning_rate": 8.52905525297188e-06, + "loss": 1.5812, + "step": 266 + }, + { + "epoch": 1.1756954612005857, + "grad_norm": 0.3075848284440638, + "learning_rate": 8.516062483807556e-06, + "loss": 1.6421, + "step": 267 + }, + { + "epoch": 1.1800878477306003, + "grad_norm": 0.2745442718449274, + "learning_rate": 8.503022575545159e-06, + "loss": 1.6592, + "step": 268 + }, + { + "epoch": 1.1844802342606149, + "grad_norm": 0.4008583979349149, + "learning_rate": 8.48993570300795e-06, + "loss": 1.6052, + "step": 269 + }, + { + "epoch": 1.1888726207906295, + "grad_norm": 0.5183284694523707, + "learning_rate": 8.476802041648832e-06, + "loss": 1.7352, + "step": 270 + }, + { + "epoch": 1.1932650073206443, + "grad_norm": 0.30560643793660064, + "learning_rate": 8.463621767547998e-06, + "loss": 1.5592, + "step": 271 + }, + { + "epoch": 1.197657393850659, + "grad_norm": 0.5272799941494408, + "learning_rate": 8.450395057410561e-06, + "loss": 1.5471, + "step": 272 + }, + { + "epoch": 1.2020497803806736, + "grad_norm": 0.4281815470371588, + "learning_rate": 8.437122088564197e-06, + "loss": 1.6445, + "step": 273 + }, + { + "epoch": 1.2064421669106882, + "grad_norm": 0.27224728594527714, + "learning_rate": 8.423803038956763e-06, + "loss": 1.6629, + "step": 274 + }, + { + "epoch": 1.2108345534407028, + "grad_norm": 0.3269535080116159, + "learning_rate": 8.41043808715391e-06, + "loss": 1.5722, + "step": 275 + }, + { + "epoch": 1.2152269399707174, + "grad_norm": 0.3054371244574999, + "learning_rate": 8.39702741233669e-06, + "loss": 1.6173, + "step": 276 + }, + { + "epoch": 1.219619326500732, + "grad_norm": 0.36010851057708126, + "learning_rate": 8.383571194299154e-06, + "loss": 1.596, + "step": 277 + }, + { + "epoch": 1.2240117130307466, + "grad_norm": 0.3803533141127172, + "learning_rate": 8.37006961344594e-06, + "loss": 1.6229, + "step": 278 + }, + { + "epoch": 1.2284040995607612, + "grad_norm": 0.28612951080269505, + "learning_rate": 8.356522850789852e-06, + "loss": 1.5549, + "step": 279 + }, + { + "epoch": 1.232796486090776, + "grad_norm": 0.2855964904759227, + "learning_rate": 8.342931087949446e-06, + "loss": 1.6512, + "step": 280 + }, + { + "epoch": 1.2371888726207907, + "grad_norm": 0.3054377930088084, + "learning_rate": 8.32929450714658e-06, + "loss": 1.6274, + "step": 281 + }, + { + "epoch": 1.2415812591508053, + "grad_norm": 0.29343763848165016, + "learning_rate": 8.315613291203977e-06, + "loss": 1.6372, + "step": 282 + }, + { + "epoch": 1.24597364568082, + "grad_norm": 0.40870431747557984, + "learning_rate": 8.301887623542774e-06, + "loss": 1.6667, + "step": 283 + }, + { + "epoch": 1.2503660322108345, + "grad_norm": 0.30881819618442474, + "learning_rate": 8.288117688180064e-06, + "loss": 1.5547, + "step": 284 + }, + { + "epoch": 1.2547584187408491, + "grad_norm": 0.28765815508847925, + "learning_rate": 8.274303669726427e-06, + "loss": 1.6097, + "step": 285 + }, + { + "epoch": 1.2591508052708638, + "grad_norm": 0.33007103881402516, + "learning_rate": 8.260445753383453e-06, + "loss": 1.7016, + "step": 286 + }, + { + "epoch": 1.2635431918008786, + "grad_norm": 0.3628853649167158, + "learning_rate": 8.246544124941266e-06, + "loss": 1.6086, + "step": 287 + }, + { + "epoch": 1.267935578330893, + "grad_norm": 0.40134140387051037, + "learning_rate": 8.232598970776027e-06, + "loss": 1.6789, + "step": 288 + }, + { + "epoch": 1.2723279648609078, + "grad_norm": 0.2920563442488122, + "learning_rate": 8.218610477847435e-06, + "loss": 1.5915, + "step": 289 + }, + { + "epoch": 1.2767203513909224, + "grad_norm": 0.28530665366941144, + "learning_rate": 8.204578833696228e-06, + "loss": 1.5949, + "step": 290 + }, + { + "epoch": 1.281112737920937, + "grad_norm": 0.3155789127902478, + "learning_rate": 8.190504226441654e-06, + "loss": 1.639, + "step": 291 + }, + { + "epoch": 1.2855051244509517, + "grad_norm": 0.3199350667177758, + "learning_rate": 8.176386844778969e-06, + "loss": 1.6818, + "step": 292 + }, + { + "epoch": 1.2898975109809663, + "grad_norm": 0.34596043922632713, + "learning_rate": 8.162226877976886e-06, + "loss": 1.6604, + "step": 293 + }, + { + "epoch": 1.294289897510981, + "grad_norm": 0.32781624288061406, + "learning_rate": 8.148024515875057e-06, + "loss": 1.5971, + "step": 294 + }, + { + "epoch": 1.2986822840409955, + "grad_norm": 0.5927243317332561, + "learning_rate": 8.133779948881514e-06, + "loss": 1.6574, + "step": 295 + }, + { + "epoch": 1.3030746705710103, + "grad_norm": 0.296769298222996, + "learning_rate": 8.11949336797012e-06, + "loss": 1.5505, + "step": 296 + }, + { + "epoch": 1.307467057101025, + "grad_norm": 0.28849112804215943, + "learning_rate": 8.105164964678009e-06, + "loss": 1.6729, + "step": 297 + }, + { + "epoch": 1.3118594436310396, + "grad_norm": 0.3286168384006033, + "learning_rate": 8.090794931103026e-06, + "loss": 1.5949, + "step": 298 + }, + { + "epoch": 1.3162518301610542, + "grad_norm": 0.31185844876531665, + "learning_rate": 8.076383459901138e-06, + "loss": 1.6918, + "step": 299 + }, + { + "epoch": 1.3206442166910688, + "grad_norm": 0.2789921585419944, + "learning_rate": 8.061930744283855e-06, + "loss": 1.6437, + "step": 300 + }, + { + "epoch": 1.3250366032210834, + "grad_norm": 0.30165508316706874, + "learning_rate": 8.04743697801565e-06, + "loss": 1.6555, + "step": 301 + }, + { + "epoch": 1.329428989751098, + "grad_norm": 0.2989059936488396, + "learning_rate": 8.032902355411345e-06, + "loss": 1.5141, + "step": 302 + }, + { + "epoch": 1.3338213762811129, + "grad_norm": 0.29928394703560823, + "learning_rate": 8.018327071333521e-06, + "loss": 1.5859, + "step": 303 + }, + { + "epoch": 1.3382137628111272, + "grad_norm": 0.29795453570714703, + "learning_rate": 8.003711321189895e-06, + "loss": 1.5864, + "step": 304 + }, + { + "epoch": 1.342606149341142, + "grad_norm": 0.30765817351768504, + "learning_rate": 7.989055300930704e-06, + "loss": 1.5743, + "step": 305 + }, + { + "epoch": 1.3469985358711567, + "grad_norm": 0.43955554329120444, + "learning_rate": 7.974359207046081e-06, + "loss": 1.6423, + "step": 306 + }, + { + "epoch": 1.3513909224011713, + "grad_norm": 0.37865307469865994, + "learning_rate": 7.959623236563412e-06, + "loss": 1.6102, + "step": 307 + }, + { + "epoch": 1.355783308931186, + "grad_norm": 0.3074090057367007, + "learning_rate": 7.944847587044704e-06, + "loss": 1.6602, + "step": 308 + }, + { + "epoch": 1.3601756954612005, + "grad_norm": 0.3361817362879421, + "learning_rate": 7.930032456583931e-06, + "loss": 1.6704, + "step": 309 + }, + { + "epoch": 1.3645680819912152, + "grad_norm": 0.2960444587212274, + "learning_rate": 7.915178043804382e-06, + "loss": 1.5536, + "step": 310 + }, + { + "epoch": 1.3689604685212298, + "grad_norm": 0.31080078465435335, + "learning_rate": 7.900284547855992e-06, + "loss": 1.659, + "step": 311 + }, + { + "epoch": 1.3733528550512446, + "grad_norm": 0.29259208311732576, + "learning_rate": 7.885352168412677e-06, + "loss": 1.6324, + "step": 312 + }, + { + "epoch": 1.3777452415812592, + "grad_norm": 0.40704907971444493, + "learning_rate": 7.870381105669657e-06, + "loss": 1.6954, + "step": 313 + }, + { + "epoch": 1.3821376281112738, + "grad_norm": 0.2884678272243352, + "learning_rate": 7.85537156034077e-06, + "loss": 1.6911, + "step": 314 + }, + { + "epoch": 1.3865300146412884, + "grad_norm": 0.3002277679918279, + "learning_rate": 7.84032373365578e-06, + "loss": 1.6384, + "step": 315 + }, + { + "epoch": 1.390922401171303, + "grad_norm": 0.29747656531257255, + "learning_rate": 7.825237827357684e-06, + "loss": 1.5728, + "step": 316 + }, + { + "epoch": 1.3953147877013177, + "grad_norm": 0.2922330023166882, + "learning_rate": 7.810114043700002e-06, + "loss": 1.6876, + "step": 317 + }, + { + "epoch": 1.3997071742313323, + "grad_norm": 0.29216993729442, + "learning_rate": 7.794952585444068e-06, + "loss": 1.7112, + "step": 318 + }, + { + "epoch": 1.4040995607613471, + "grad_norm": 0.2841430060967501, + "learning_rate": 7.779753655856313e-06, + "loss": 1.6352, + "step": 319 + }, + { + "epoch": 1.4084919472913615, + "grad_norm": 0.29535174268674796, + "learning_rate": 7.764517458705536e-06, + "loss": 1.638, + "step": 320 + }, + { + "epoch": 1.4128843338213763, + "grad_norm": 0.3553267662730052, + "learning_rate": 7.749244198260175e-06, + "loss": 1.6262, + "step": 321 + }, + { + "epoch": 1.417276720351391, + "grad_norm": 0.2977673782348279, + "learning_rate": 7.733934079285569e-06, + "loss": 1.6115, + "step": 322 + }, + { + "epoch": 1.4216691068814056, + "grad_norm": 0.2822196746941408, + "learning_rate": 7.718587307041209e-06, + "loss": 1.6468, + "step": 323 + }, + { + "epoch": 1.4260614934114202, + "grad_norm": 0.2837671497162217, + "learning_rate": 7.703204087277989e-06, + "loss": 1.6474, + "step": 324 + }, + { + "epoch": 1.4304538799414348, + "grad_norm": 0.31664451334685184, + "learning_rate": 7.687784626235448e-06, + "loss": 1.5919, + "step": 325 + }, + { + "epoch": 1.4348462664714494, + "grad_norm": 0.33195017711571867, + "learning_rate": 7.672329130639007e-06, + "loss": 1.6941, + "step": 326 + }, + { + "epoch": 1.439238653001464, + "grad_norm": 0.2995108911659512, + "learning_rate": 7.656837807697187e-06, + "loss": 1.6393, + "step": 327 + }, + { + "epoch": 1.4436310395314789, + "grad_norm": 0.2949122695416032, + "learning_rate": 7.641310865098845e-06, + "loss": 1.5135, + "step": 328 + }, + { + "epoch": 1.4480234260614935, + "grad_norm": 0.31946384200509315, + "learning_rate": 7.625748511010382e-06, + "loss": 1.5784, + "step": 329 + }, + { + "epoch": 1.452415812591508, + "grad_norm": 0.29208814812765094, + "learning_rate": 7.610150954072953e-06, + "loss": 1.6794, + "step": 330 + }, + { + "epoch": 1.4568081991215227, + "grad_norm": 0.34475174012170284, + "learning_rate": 7.594518403399666e-06, + "loss": 1.5773, + "step": 331 + }, + { + "epoch": 1.4612005856515373, + "grad_norm": 0.2876304119955936, + "learning_rate": 7.578851068572788e-06, + "loss": 1.5613, + "step": 332 + }, + { + "epoch": 1.465592972181552, + "grad_norm": 0.31542092184200016, + "learning_rate": 7.563149159640929e-06, + "loss": 1.6086, + "step": 333 + }, + { + "epoch": 1.4699853587115665, + "grad_norm": 0.2964161628058562, + "learning_rate": 7.547412887116224e-06, + "loss": 1.5963, + "step": 334 + }, + { + "epoch": 1.4743777452415814, + "grad_norm": 0.32832262869958717, + "learning_rate": 7.531642461971515e-06, + "loss": 1.6054, + "step": 335 + }, + { + "epoch": 1.4787701317715958, + "grad_norm": 0.3209241382730098, + "learning_rate": 7.51583809563752e-06, + "loss": 1.6748, + "step": 336 + }, + { + "epoch": 1.4831625183016106, + "grad_norm": 0.30060262616835987, + "learning_rate": 7.500000000000001e-06, + "loss": 1.6081, + "step": 337 + }, + { + "epoch": 1.4875549048316252, + "grad_norm": 0.3500388762321464, + "learning_rate": 7.4841283873969194e-06, + "loss": 1.5906, + "step": 338 + }, + { + "epoch": 1.4919472913616398, + "grad_norm": 0.28308587203259633, + "learning_rate": 7.468223470615593e-06, + "loss": 1.5879, + "step": 339 + }, + { + "epoch": 1.4963396778916545, + "grad_norm": 0.2901927981331445, + "learning_rate": 7.452285462889841e-06, + "loss": 1.6242, + "step": 340 + }, + { + "epoch": 1.500732064421669, + "grad_norm": 0.3314307283598877, + "learning_rate": 7.436314577897126e-06, + "loss": 1.646, + "step": 341 + }, + { + "epoch": 1.505124450951684, + "grad_norm": 0.3264131109049365, + "learning_rate": 7.420311029755688e-06, + "loss": 1.5855, + "step": 342 + }, + { + "epoch": 1.5095168374816983, + "grad_norm": 0.3519561244527488, + "learning_rate": 7.404275033021676e-06, + "loss": 1.6415, + "step": 343 + }, + { + "epoch": 1.5139092240117131, + "grad_norm": 0.3231864119230437, + "learning_rate": 7.388206802686272e-06, + "loss": 1.6038, + "step": 344 + }, + { + "epoch": 1.5183016105417275, + "grad_norm": 0.4648117736859699, + "learning_rate": 7.372106554172802e-06, + "loss": 1.6327, + "step": 345 + }, + { + "epoch": 1.5226939970717424, + "grad_norm": 0.30819599956019383, + "learning_rate": 7.355974503333859e-06, + "loss": 1.7179, + "step": 346 + }, + { + "epoch": 1.527086383601757, + "grad_norm": 0.3425241683440226, + "learning_rate": 7.339810866448398e-06, + "loss": 1.6486, + "step": 347 + }, + { + "epoch": 1.5314787701317716, + "grad_norm": 0.3229436387251198, + "learning_rate": 7.323615860218844e-06, + "loss": 1.5028, + "step": 348 + }, + { + "epoch": 1.5358711566617862, + "grad_norm": 0.32420322558759, + "learning_rate": 7.307389701768183e-06, + "loss": 1.517, + "step": 349 + }, + { + "epoch": 1.5402635431918008, + "grad_norm": 0.33308925785501975, + "learning_rate": 7.291132608637053e-06, + "loss": 1.6278, + "step": 350 + }, + { + "epoch": 1.5446559297218156, + "grad_norm": 0.33961176568859514, + "learning_rate": 7.274844798780826e-06, + "loss": 1.6651, + "step": 351 + }, + { + "epoch": 1.54904831625183, + "grad_norm": 0.32173930273995155, + "learning_rate": 7.258526490566687e-06, + "loss": 1.491, + "step": 352 + }, + { + "epoch": 1.5534407027818449, + "grad_norm": 0.3093139646127846, + "learning_rate": 7.242177902770706e-06, + "loss": 1.5796, + "step": 353 + }, + { + "epoch": 1.5578330893118595, + "grad_norm": 0.32919345439390313, + "learning_rate": 7.2257992545749045e-06, + "loss": 1.5865, + "step": 354 + }, + { + "epoch": 1.562225475841874, + "grad_norm": 0.2835010953118983, + "learning_rate": 7.209390765564318e-06, + "loss": 1.6417, + "step": 355 + }, + { + "epoch": 1.5666178623718887, + "grad_norm": 1.7445414238062138, + "learning_rate": 7.192952655724049e-06, + "loss": 1.6597, + "step": 356 + }, + { + "epoch": 1.5710102489019033, + "grad_norm": 0.5860196285629298, + "learning_rate": 7.176485145436325e-06, + "loss": 1.5387, + "step": 357 + }, + { + "epoch": 1.5754026354319182, + "grad_norm": 0.3537384385833807, + "learning_rate": 7.159988455477534e-06, + "loss": 1.6013, + "step": 358 + }, + { + "epoch": 1.5797950219619326, + "grad_norm": 0.3286829503374976, + "learning_rate": 7.143462807015271e-06, + "loss": 1.6264, + "step": 359 + }, + { + "epoch": 1.5841874084919474, + "grad_norm": 0.3197218822335203, + "learning_rate": 7.1269084216053756e-06, + "loss": 1.5552, + "step": 360 + }, + { + "epoch": 1.5885797950219618, + "grad_norm": 0.31209150272027425, + "learning_rate": 7.11032552118895e-06, + "loss": 1.6108, + "step": 361 + }, + { + "epoch": 1.5929721815519766, + "grad_norm": 0.3227458575298131, + "learning_rate": 7.093714328089398e-06, + "loss": 1.649, + "step": 362 + }, + { + "epoch": 1.5973645680819912, + "grad_norm": 0.3637374362635078, + "learning_rate": 7.0770750650094335e-06, + "loss": 1.6739, + "step": 363 + }, + { + "epoch": 1.6017569546120058, + "grad_norm": 0.31236210751966154, + "learning_rate": 7.060407955028098e-06, + "loss": 1.5946, + "step": 364 + }, + { + "epoch": 1.6061493411420205, + "grad_norm": 0.35651724986710415, + "learning_rate": 7.0437132215977744e-06, + "loss": 1.6337, + "step": 365 + }, + { + "epoch": 1.610541727672035, + "grad_norm": 0.31143483852268733, + "learning_rate": 7.026991088541184e-06, + "loss": 1.67, + "step": 366 + }, + { + "epoch": 1.61493411420205, + "grad_norm": 0.33050222076540253, + "learning_rate": 7.01024178004839e-06, + "loss": 1.6204, + "step": 367 + }, + { + "epoch": 1.6193265007320643, + "grad_norm": 0.3284547022539121, + "learning_rate": 6.99346552067379e-06, + "loss": 1.573, + "step": 368 + }, + { + "epoch": 1.6237188872620791, + "grad_norm": 0.3603027916184282, + "learning_rate": 6.976662535333107e-06, + "loss": 1.6553, + "step": 369 + }, + { + "epoch": 1.6281112737920937, + "grad_norm": 0.8725979374632243, + "learning_rate": 6.959833049300376e-06, + "loss": 1.6279, + "step": 370 + }, + { + "epoch": 1.6325036603221084, + "grad_norm": 0.28597465971107255, + "learning_rate": 6.942977288204915e-06, + "loss": 1.5074, + "step": 371 + }, + { + "epoch": 1.636896046852123, + "grad_norm": 0.31388416897723714, + "learning_rate": 6.926095478028312e-06, + "loss": 1.6896, + "step": 372 + }, + { + "epoch": 1.6412884333821376, + "grad_norm": 0.3008107505835411, + "learning_rate": 6.909187845101387e-06, + "loss": 1.5688, + "step": 373 + }, + { + "epoch": 1.6456808199121524, + "grad_norm": 0.32252769956105753, + "learning_rate": 6.892254616101158e-06, + "loss": 1.7391, + "step": 374 + }, + { + "epoch": 1.6500732064421668, + "grad_norm": 0.3253332219188239, + "learning_rate": 6.87529601804781e-06, + "loss": 1.5479, + "step": 375 + }, + { + "epoch": 1.6544655929721817, + "grad_norm": 0.3354516401894849, + "learning_rate": 6.858312278301638e-06, + "loss": 1.5944, + "step": 376 + }, + { + "epoch": 1.658857979502196, + "grad_norm": 0.3101157670728667, + "learning_rate": 6.841303624560012e-06, + "loss": 1.5979, + "step": 377 + }, + { + "epoch": 1.6632503660322109, + "grad_norm": 0.3263157294243148, + "learning_rate": 6.8242702848543185e-06, + "loss": 1.6651, + "step": 378 + }, + { + "epoch": 1.6676427525622255, + "grad_norm": 0.2963170265612591, + "learning_rate": 6.807212487546897e-06, + "loss": 1.6414, + "step": 379 + }, + { + "epoch": 1.67203513909224, + "grad_norm": 0.3436329546570268, + "learning_rate": 6.790130461327993e-06, + "loss": 1.5237, + "step": 380 + }, + { + "epoch": 1.6764275256222547, + "grad_norm": 0.30951635226199037, + "learning_rate": 6.773024435212678e-06, + "loss": 1.5608, + "step": 381 + }, + { + "epoch": 1.6808199121522693, + "grad_norm": 0.3024875350743093, + "learning_rate": 6.755894638537791e-06, + "loss": 1.5595, + "step": 382 + }, + { + "epoch": 1.6852122986822842, + "grad_norm": 0.30964409063235376, + "learning_rate": 6.73874130095885e-06, + "loss": 1.6166, + "step": 383 + }, + { + "epoch": 1.6896046852122986, + "grad_norm": 0.31495863587091266, + "learning_rate": 6.721564652446987e-06, + "loss": 1.5279, + "step": 384 + }, + { + "epoch": 1.6939970717423134, + "grad_norm": 0.3115937718660612, + "learning_rate": 6.704364923285858e-06, + "loss": 1.5694, + "step": 385 + }, + { + "epoch": 1.698389458272328, + "grad_norm": 0.29768001900234425, + "learning_rate": 6.687142344068552e-06, + "loss": 1.5578, + "step": 386 + }, + { + "epoch": 1.7027818448023426, + "grad_norm": 0.32146567067323123, + "learning_rate": 6.669897145694507e-06, + "loss": 1.6368, + "step": 387 + }, + { + "epoch": 1.7071742313323572, + "grad_norm": 0.29522637791127904, + "learning_rate": 6.6526295593664136e-06, + "loss": 1.6121, + "step": 388 + }, + { + "epoch": 1.7115666178623719, + "grad_norm": 0.30160303491140916, + "learning_rate": 6.635339816587109e-06, + "loss": 1.6238, + "step": 389 + }, + { + "epoch": 1.7159590043923867, + "grad_norm": 0.4208009773656739, + "learning_rate": 6.618028149156479e-06, + "loss": 1.62, + "step": 390 + }, + { + "epoch": 1.720351390922401, + "grad_norm": 0.2881943555017053, + "learning_rate": 6.600694789168345e-06, + "loss": 1.6364, + "step": 391 + }, + { + "epoch": 1.724743777452416, + "grad_norm": 0.3077307710744344, + "learning_rate": 6.583339969007364e-06, + "loss": 1.5677, + "step": 392 + }, + { + "epoch": 1.7291361639824303, + "grad_norm": 0.35235096673751176, + "learning_rate": 6.565963921345896e-06, + "loss": 1.6163, + "step": 393 + }, + { + "epoch": 1.7335285505124451, + "grad_norm": 0.3186476335255873, + "learning_rate": 6.548566879140897e-06, + "loss": 1.6289, + "step": 394 + }, + { + "epoch": 1.7379209370424598, + "grad_norm": 0.31476614898776, + "learning_rate": 6.531149075630796e-06, + "loss": 1.5874, + "step": 395 + }, + { + "epoch": 1.7423133235724744, + "grad_norm": 0.3469536206500234, + "learning_rate": 6.513710744332361e-06, + "loss": 1.6621, + "step": 396 + }, + { + "epoch": 1.746705710102489, + "grad_norm": 0.30680438680917854, + "learning_rate": 6.49625211903757e-06, + "loss": 1.595, + "step": 397 + }, + { + "epoch": 1.7510980966325036, + "grad_norm": 0.34458603376189034, + "learning_rate": 6.478773433810478e-06, + "loss": 1.5886, + "step": 398 + }, + { + "epoch": 1.7554904831625184, + "grad_norm": 0.34083953962031827, + "learning_rate": 6.461274922984087e-06, + "loss": 1.5705, + "step": 399 + }, + { + "epoch": 1.7598828696925328, + "grad_norm": 0.3345685083562738, + "learning_rate": 6.4437568211571864e-06, + "loss": 1.6181, + "step": 400 + }, + { + "epoch": 1.7642752562225477, + "grad_norm": 0.28767837558253834, + "learning_rate": 6.426219363191224e-06, + "loss": 1.6273, + "step": 401 + }, + { + "epoch": 1.7686676427525623, + "grad_norm": 0.3309056934700509, + "learning_rate": 6.408662784207149e-06, + "loss": 1.5575, + "step": 402 + }, + { + "epoch": 1.7730600292825769, + "grad_norm": 0.33936552783693097, + "learning_rate": 6.391087319582264e-06, + "loss": 1.5653, + "step": 403 + }, + { + "epoch": 1.7774524158125915, + "grad_norm": 0.38728001848935395, + "learning_rate": 6.373493204947065e-06, + "loss": 1.5962, + "step": 404 + }, + { + "epoch": 1.7818448023426061, + "grad_norm": 0.33731817557741206, + "learning_rate": 6.355880676182086e-06, + "loss": 1.5267, + "step": 405 + }, + { + "epoch": 1.786237188872621, + "grad_norm": 0.3073610397325029, + "learning_rate": 6.3382499694147345e-06, + "loss": 1.5735, + "step": 406 + }, + { + "epoch": 1.7906295754026353, + "grad_norm": 0.5666929994713482, + "learning_rate": 6.3206013210161285e-06, + "loss": 1.6164, + "step": 407 + }, + { + "epoch": 1.7950219619326502, + "grad_norm": 0.3423706928777578, + "learning_rate": 6.302934967597922e-06, + "loss": 1.6694, + "step": 408 + }, + { + "epoch": 1.7994143484626646, + "grad_norm": 0.3100272828530065, + "learning_rate": 6.2852511460091406e-06, + "loss": 1.6322, + "step": 409 + }, + { + "epoch": 1.8038067349926794, + "grad_norm": 0.3710603928040935, + "learning_rate": 6.267550093333e-06, + "loss": 1.5405, + "step": 410 + }, + { + "epoch": 1.808199121522694, + "grad_norm": 0.3024234235421662, + "learning_rate": 6.249832046883729e-06, + "loss": 1.6389, + "step": 411 + }, + { + "epoch": 1.8125915080527086, + "grad_norm": 0.2923050985231585, + "learning_rate": 6.232097244203388e-06, + "loss": 1.563, + "step": 412 + }, + { + "epoch": 1.8169838945827232, + "grad_norm": 0.37742465142395615, + "learning_rate": 6.214345923058686e-06, + "loss": 1.5605, + "step": 413 + }, + { + "epoch": 1.8213762811127379, + "grad_norm": 0.2763099605085667, + "learning_rate": 6.1965783214377895e-06, + "loss": 1.5757, + "step": 414 + }, + { + "epoch": 1.8257686676427527, + "grad_norm": 0.29226253974509314, + "learning_rate": 6.178794677547138e-06, + "loss": 1.6213, + "step": 415 + }, + { + "epoch": 1.830161054172767, + "grad_norm": 0.3892056822790433, + "learning_rate": 6.16099522980824e-06, + "loss": 1.6471, + "step": 416 + }, + { + "epoch": 1.834553440702782, + "grad_norm": 0.3547953314773269, + "learning_rate": 6.143180216854488e-06, + "loss": 1.5933, + "step": 417 + }, + { + "epoch": 1.8389458272327965, + "grad_norm": 0.2876417607221798, + "learning_rate": 6.125349877527952e-06, + "loss": 1.6056, + "step": 418 + }, + { + "epoch": 1.8433382137628112, + "grad_norm": 0.29632892867662236, + "learning_rate": 6.1075044508761804e-06, + "loss": 1.5798, + "step": 419 + }, + { + "epoch": 1.8477306002928258, + "grad_norm": 0.3351398469787539, + "learning_rate": 6.0896441761489925e-06, + "loss": 1.5775, + "step": 420 + }, + { + "epoch": 1.8521229868228404, + "grad_norm": 0.32572738949136165, + "learning_rate": 6.071769292795274e-06, + "loss": 1.5065, + "step": 421 + }, + { + "epoch": 1.8565153733528552, + "grad_norm": 0.32922431311553646, + "learning_rate": 6.053880040459765e-06, + "loss": 1.5768, + "step": 422 + }, + { + "epoch": 1.8609077598828696, + "grad_norm": 0.3229086221064645, + "learning_rate": 6.035976658979846e-06, + "loss": 1.6242, + "step": 423 + }, + { + "epoch": 1.8653001464128844, + "grad_norm": 0.3114352291982662, + "learning_rate": 6.018059388382327e-06, + "loss": 1.6473, + "step": 424 + }, + { + "epoch": 1.8696925329428988, + "grad_norm": 0.31045445243058734, + "learning_rate": 6.000128468880223e-06, + "loss": 1.535, + "step": 425 + }, + { + "epoch": 1.8740849194729137, + "grad_norm": 0.33523008129662585, + "learning_rate": 5.982184140869539e-06, + "loss": 1.6344, + "step": 426 + }, + { + "epoch": 1.8784773060029283, + "grad_norm": 0.3081522900734568, + "learning_rate": 5.964226644926045e-06, + "loss": 1.6386, + "step": 427 + }, + { + "epoch": 1.882869692532943, + "grad_norm": 0.2900227624443104, + "learning_rate": 5.946256221802052e-06, + "loss": 1.5843, + "step": 428 + }, + { + "epoch": 1.8872620790629575, + "grad_norm": 0.354099020882711, + "learning_rate": 5.928273112423177e-06, + "loss": 1.5758, + "step": 429 + }, + { + "epoch": 1.8916544655929721, + "grad_norm": 0.34160211290628684, + "learning_rate": 5.910277557885128e-06, + "loss": 1.5551, + "step": 430 + }, + { + "epoch": 1.896046852122987, + "grad_norm": 0.3162945077147358, + "learning_rate": 5.892269799450453e-06, + "loss": 1.6204, + "step": 431 + }, + { + "epoch": 1.9004392386530014, + "grad_norm": 0.3509407679369851, + "learning_rate": 5.8742500785453226e-06, + "loss": 1.6224, + "step": 432 + }, + { + "epoch": 1.9048316251830162, + "grad_norm": 0.2848983911128527, + "learning_rate": 5.856218636756281e-06, + "loss": 1.6059, + "step": 433 + }, + { + "epoch": 1.9092240117130308, + "grad_norm": 0.38193324539008777, + "learning_rate": 5.838175715827015e-06, + "loss": 1.5511, + "step": 434 + }, + { + "epoch": 1.9136163982430454, + "grad_norm": 0.31934508340416906, + "learning_rate": 5.820121557655109e-06, + "loss": 1.7028, + "step": 435 + }, + { + "epoch": 1.91800878477306, + "grad_norm": 0.33670681289920085, + "learning_rate": 5.8020564042888015e-06, + "loss": 1.5687, + "step": 436 + }, + { + "epoch": 1.9224011713030746, + "grad_norm": 0.318844847843566, + "learning_rate": 5.783980497923743e-06, + "loss": 1.567, + "step": 437 + }, + { + "epoch": 1.9267935578330895, + "grad_norm": 0.29610949062035014, + "learning_rate": 5.76589408089974e-06, + "loss": 1.5166, + "step": 438 + }, + { + "epoch": 1.9311859443631039, + "grad_norm": 0.28728684262143106, + "learning_rate": 5.747797395697525e-06, + "loss": 1.5813, + "step": 439 + }, + { + "epoch": 1.9355783308931187, + "grad_norm": 0.32309431631814955, + "learning_rate": 5.729690684935487e-06, + "loss": 1.641, + "step": 440 + }, + { + "epoch": 1.939970717423133, + "grad_norm": 0.2969130322032886, + "learning_rate": 5.711574191366427e-06, + "loss": 1.5608, + "step": 441 + }, + { + "epoch": 1.944363103953148, + "grad_norm": 0.42872939564364393, + "learning_rate": 5.693448157874299e-06, + "loss": 1.5983, + "step": 442 + }, + { + "epoch": 1.9487554904831625, + "grad_norm": 0.3798270075015977, + "learning_rate": 5.675312827470959e-06, + "loss": 1.5896, + "step": 443 + }, + { + "epoch": 1.9531478770131772, + "grad_norm": 0.30174438725527203, + "learning_rate": 5.657168443292909e-06, + "loss": 1.6043, + "step": 444 + }, + { + "epoch": 1.9575402635431918, + "grad_norm": 0.4119857865917871, + "learning_rate": 5.639015248598025e-06, + "loss": 1.5754, + "step": 445 + }, + { + "epoch": 1.9619326500732064, + "grad_norm": 0.32221313787015915, + "learning_rate": 5.620853486762307e-06, + "loss": 1.6569, + "step": 446 + }, + { + "epoch": 1.9663250366032212, + "grad_norm": 0.3547729959901477, + "learning_rate": 5.6026834012766155e-06, + "loss": 1.6228, + "step": 447 + }, + { + "epoch": 1.9707174231332356, + "grad_norm": 0.28279646337718173, + "learning_rate": 5.584505235743404e-06, + "loss": 1.5874, + "step": 448 + }, + { + "epoch": 1.9751098096632504, + "grad_norm": 0.3264218725678291, + "learning_rate": 5.566319233873446e-06, + "loss": 1.6313, + "step": 449 + }, + { + "epoch": 1.979502196193265, + "grad_norm": 0.28955653787826696, + "learning_rate": 5.548125639482587e-06, + "loss": 1.5989, + "step": 450 + }, + { + "epoch": 1.9838945827232797, + "grad_norm": 0.881097673855719, + "learning_rate": 5.529924696488456e-06, + "loss": 1.5796, + "step": 451 + }, + { + "epoch": 1.9882869692532943, + "grad_norm": 0.32048202284049887, + "learning_rate": 5.5117166489072014e-06, + "loss": 1.578, + "step": 452 + }, + { + "epoch": 1.992679355783309, + "grad_norm": 0.2896791795835667, + "learning_rate": 5.493501740850228e-06, + "loss": 1.6146, + "step": 453 + }, + { + "epoch": 1.9970717423133237, + "grad_norm": 0.35142442191521966, + "learning_rate": 5.475280216520913e-06, + "loss": 1.5522, + "step": 454 + }, + { + "epoch": 2.004392386530015, + "grad_norm": 0.6164551609296155, + "learning_rate": 5.45705232021134e-06, + "loss": 3.059, + "step": 455 + }, + { + "epoch": 2.0087847730600292, + "grad_norm": 0.33044889056247584, + "learning_rate": 5.438818296299015e-06, + "loss": 1.5982, + "step": 456 + }, + { + "epoch": 2.013177159590044, + "grad_norm": 0.31938902514769063, + "learning_rate": 5.4205783892435996e-06, + "loss": 1.6012, + "step": 457 + }, + { + "epoch": 2.0175695461200585, + "grad_norm": 1.1795222072761016, + "learning_rate": 5.402332843583631e-06, + "loss": 1.588, + "step": 458 + }, + { + "epoch": 2.0219619326500733, + "grad_norm": 0.31065801099482754, + "learning_rate": 5.384081903933235e-06, + "loss": 1.6023, + "step": 459 + }, + { + "epoch": 2.0263543191800877, + "grad_norm": 0.29281216005167415, + "learning_rate": 5.365825814978861e-06, + "loss": 1.5848, + "step": 460 + }, + { + "epoch": 2.0307467057101025, + "grad_norm": 0.2939045229396799, + "learning_rate": 5.34756482147599e-06, + "loss": 1.5619, + "step": 461 + }, + { + "epoch": 2.035139092240117, + "grad_norm": 0.2763356309694064, + "learning_rate": 5.3292991682458576e-06, + "loss": 1.6325, + "step": 462 + }, + { + "epoch": 2.0395314787701317, + "grad_norm": 0.32189639380967644, + "learning_rate": 5.311029100172173e-06, + "loss": 1.5714, + "step": 463 + }, + { + "epoch": 2.0439238653001466, + "grad_norm": 0.2991909339232657, + "learning_rate": 5.292754862197831e-06, + "loss": 1.5574, + "step": 464 + }, + { + "epoch": 2.048316251830161, + "grad_norm": 0.3129964860572516, + "learning_rate": 5.274476699321638e-06, + "loss": 1.6938, + "step": 465 + }, + { + "epoch": 2.052708638360176, + "grad_norm": 0.30394033701147166, + "learning_rate": 5.256194856595012e-06, + "loss": 1.5411, + "step": 466 + }, + { + "epoch": 2.05710102489019, + "grad_norm": 0.34065979728221907, + "learning_rate": 5.237909579118713e-06, + "loss": 1.6064, + "step": 467 + }, + { + "epoch": 2.061493411420205, + "grad_norm": 0.2826129934198123, + "learning_rate": 5.219621112039545e-06, + "loss": 1.6057, + "step": 468 + }, + { + "epoch": 2.0658857979502194, + "grad_norm": 0.3195263718137741, + "learning_rate": 5.201329700547077e-06, + "loss": 1.6617, + "step": 469 + }, + { + "epoch": 2.0702781844802343, + "grad_norm": 0.31086863726157415, + "learning_rate": 5.183035589870353e-06, + "loss": 1.5866, + "step": 470 + }, + { + "epoch": 2.074670571010249, + "grad_norm": 0.32171527645836867, + "learning_rate": 5.164739025274604e-06, + "loss": 1.5607, + "step": 471 + }, + { + "epoch": 2.0790629575402635, + "grad_norm": 0.30392336695969924, + "learning_rate": 5.146440252057962e-06, + "loss": 1.5615, + "step": 472 + }, + { + "epoch": 2.0834553440702783, + "grad_norm": 0.31809057906529703, + "learning_rate": 5.128139515548164e-06, + "loss": 1.5828, + "step": 473 + }, + { + "epoch": 2.0878477306002927, + "grad_norm": 0.2927979459326381, + "learning_rate": 5.109837061099274e-06, + "loss": 1.6074, + "step": 474 + }, + { + "epoch": 2.0922401171303076, + "grad_norm": 0.28785248814578307, + "learning_rate": 5.0915331340883875e-06, + "loss": 1.5747, + "step": 475 + }, + { + "epoch": 2.096632503660322, + "grad_norm": 0.32743369261911104, + "learning_rate": 5.07322797991234e-06, + "loss": 1.6416, + "step": 476 + }, + { + "epoch": 2.101024890190337, + "grad_norm": 0.3059835406726052, + "learning_rate": 5.0549218439844185e-06, + "loss": 1.5421, + "step": 477 + }, + { + "epoch": 2.1054172767203516, + "grad_norm": 0.3034377175450521, + "learning_rate": 5.036614971731076e-06, + "loss": 1.5441, + "step": 478 + }, + { + "epoch": 2.109809663250366, + "grad_norm": 0.4025542458464793, + "learning_rate": 5.018307608588637e-06, + "loss": 1.5459, + "step": 479 + }, + { + "epoch": 2.114202049780381, + "grad_norm": 0.29644549708934803, + "learning_rate": 5e-06, + "loss": 1.5845, + "step": 480 + }, + { + "epoch": 2.1185944363103952, + "grad_norm": 0.3821617049266223, + "learning_rate": 4.981692391411366e-06, + "loss": 1.5627, + "step": 481 + }, + { + "epoch": 2.12298682284041, + "grad_norm": 0.32702317134885744, + "learning_rate": 4.963385028268925e-06, + "loss": 1.6177, + "step": 482 + }, + { + "epoch": 2.1273792093704245, + "grad_norm": 0.5647033191984581, + "learning_rate": 4.945078156015582e-06, + "loss": 1.5839, + "step": 483 + }, + { + "epoch": 2.1317715959004393, + "grad_norm": 0.3030955667087971, + "learning_rate": 4.926772020087663e-06, + "loss": 1.6255, + "step": 484 + }, + { + "epoch": 2.1361639824304537, + "grad_norm": 0.33951362803589097, + "learning_rate": 4.908466865911615e-06, + "loss": 1.6158, + "step": 485 + }, + { + "epoch": 2.1405563689604685, + "grad_norm": 0.29445544201117774, + "learning_rate": 4.8901629389007274e-06, + "loss": 1.5388, + "step": 486 + }, + { + "epoch": 2.1449487554904834, + "grad_norm": 0.3590464064008913, + "learning_rate": 4.871860484451838e-06, + "loss": 1.5925, + "step": 487 + }, + { + "epoch": 2.1493411420204978, + "grad_norm": 0.29393299221073704, + "learning_rate": 4.853559747942041e-06, + "loss": 1.5905, + "step": 488 + }, + { + "epoch": 2.1537335285505126, + "grad_norm": 0.3045017300873621, + "learning_rate": 4.835260974725397e-06, + "loss": 1.6272, + "step": 489 + }, + { + "epoch": 2.158125915080527, + "grad_norm": 0.3005274825666973, + "learning_rate": 4.816964410129648e-06, + "loss": 1.5681, + "step": 490 + }, + { + "epoch": 2.162518301610542, + "grad_norm": 0.3689663925401503, + "learning_rate": 4.798670299452926e-06, + "loss": 1.5749, + "step": 491 + }, + { + "epoch": 2.166910688140556, + "grad_norm": 0.3613384924795449, + "learning_rate": 4.7803788879604585e-06, + "loss": 1.6029, + "step": 492 + }, + { + "epoch": 2.171303074670571, + "grad_norm": 0.3237181229811394, + "learning_rate": 4.762090420881289e-06, + "loss": 1.5673, + "step": 493 + }, + { + "epoch": 2.1756954612005854, + "grad_norm": 0.29104535885079846, + "learning_rate": 4.743805143404989e-06, + "loss": 1.5711, + "step": 494 + }, + { + "epoch": 2.1800878477306003, + "grad_norm": 0.310319811358895, + "learning_rate": 4.7255233006783626e-06, + "loss": 1.6182, + "step": 495 + }, + { + "epoch": 2.184480234260615, + "grad_norm": 0.31864865605925, + "learning_rate": 4.707245137802169e-06, + "loss": 1.636, + "step": 496 + }, + { + "epoch": 2.1888726207906295, + "grad_norm": 0.2889375392125057, + "learning_rate": 4.688970899827828e-06, + "loss": 1.6264, + "step": 497 + }, + { + "epoch": 2.1932650073206443, + "grad_norm": 0.337058227912909, + "learning_rate": 4.670700831754145e-06, + "loss": 1.5834, + "step": 498 + }, + { + "epoch": 2.1976573938506587, + "grad_norm": 0.3060498021030112, + "learning_rate": 4.652435178524013e-06, + "loss": 1.5825, + "step": 499 + }, + { + "epoch": 2.2020497803806736, + "grad_norm": 0.31142401889233506, + "learning_rate": 4.6341741850211406e-06, + "loss": 1.6192, + "step": 500 + }, + { + "epoch": 2.206442166910688, + "grad_norm": 0.3480563412111249, + "learning_rate": 4.615918096066766e-06, + "loss": 1.5718, + "step": 501 + }, + { + "epoch": 2.210834553440703, + "grad_norm": 0.2774237140655264, + "learning_rate": 4.597667156416371e-06, + "loss": 1.639, + "step": 502 + }, + { + "epoch": 2.2152269399707176, + "grad_norm": 0.2804945714593694, + "learning_rate": 4.5794216107564e-06, + "loss": 1.5795, + "step": 503 + }, + { + "epoch": 2.219619326500732, + "grad_norm": 0.4361446908640307, + "learning_rate": 4.561181703700986e-06, + "loss": 1.5406, + "step": 504 + }, + { + "epoch": 2.224011713030747, + "grad_norm": 0.3609641609090663, + "learning_rate": 4.542947679788662e-06, + "loss": 1.5926, + "step": 505 + }, + { + "epoch": 2.2284040995607612, + "grad_norm": 0.29968200375160275, + "learning_rate": 4.524719783479088e-06, + "loss": 1.6309, + "step": 506 + }, + { + "epoch": 2.232796486090776, + "grad_norm": 0.2902529148130268, + "learning_rate": 4.506498259149774e-06, + "loss": 1.6243, + "step": 507 + }, + { + "epoch": 2.2371888726207905, + "grad_norm": 0.35171986609259426, + "learning_rate": 4.488283351092799e-06, + "loss": 1.585, + "step": 508 + }, + { + "epoch": 2.2415812591508053, + "grad_norm": 0.3140743538598277, + "learning_rate": 4.470075303511546e-06, + "loss": 1.5835, + "step": 509 + }, + { + "epoch": 2.24597364568082, + "grad_norm": 0.743746136107164, + "learning_rate": 4.451874360517413e-06, + "loss": 1.497, + "step": 510 + }, + { + "epoch": 2.2503660322108345, + "grad_norm": 0.30150754058851265, + "learning_rate": 4.433680766126554e-06, + "loss": 1.6938, + "step": 511 + }, + { + "epoch": 2.2547584187408494, + "grad_norm": 0.3418824009574996, + "learning_rate": 4.4154947642566e-06, + "loss": 1.5996, + "step": 512 + }, + { + "epoch": 2.2591508052708638, + "grad_norm": 4.265008272718185, + "learning_rate": 4.397316598723385e-06, + "loss": 1.5282, + "step": 513 + }, + { + "epoch": 2.2635431918008786, + "grad_norm": 0.3174625288266337, + "learning_rate": 4.379146513237695e-06, + "loss": 1.646, + "step": 514 + }, + { + "epoch": 2.267935578330893, + "grad_norm": 0.3319690818479321, + "learning_rate": 4.360984751401977e-06, + "loss": 1.5278, + "step": 515 + }, + { + "epoch": 2.272327964860908, + "grad_norm": 0.30044205636049554, + "learning_rate": 4.342831556707093e-06, + "loss": 1.609, + "step": 516 + }, + { + "epoch": 2.276720351390922, + "grad_norm": 0.33317854776862893, + "learning_rate": 4.3246871725290414e-06, + "loss": 1.5675, + "step": 517 + }, + { + "epoch": 2.281112737920937, + "grad_norm": 0.31632328292339285, + "learning_rate": 4.306551842125702e-06, + "loss": 1.5537, + "step": 518 + }, + { + "epoch": 2.2855051244509514, + "grad_norm": 0.4655555090676632, + "learning_rate": 4.2884258086335755e-06, + "loss": 1.5998, + "step": 519 + }, + { + "epoch": 2.2898975109809663, + "grad_norm": 0.2769381188179044, + "learning_rate": 4.270309315064514e-06, + "loss": 1.6107, + "step": 520 + }, + { + "epoch": 2.294289897510981, + "grad_norm": 0.28065221196751095, + "learning_rate": 4.252202604302476e-06, + "loss": 1.6556, + "step": 521 + }, + { + "epoch": 2.2986822840409955, + "grad_norm": 0.42592024405532036, + "learning_rate": 4.234105919100261e-06, + "loss": 1.5993, + "step": 522 + }, + { + "epoch": 2.3030746705710103, + "grad_norm": 0.3058126067604197, + "learning_rate": 4.21601950207626e-06, + "loss": 1.6123, + "step": 523 + }, + { + "epoch": 2.3074670571010247, + "grad_norm": 0.3341616849592129, + "learning_rate": 4.1979435957111984e-06, + "loss": 1.617, + "step": 524 + }, + { + "epoch": 2.3118594436310396, + "grad_norm": 0.29572720448399287, + "learning_rate": 4.179878442344892e-06, + "loss": 1.6266, + "step": 525 + }, + { + "epoch": 2.316251830161054, + "grad_norm": 0.31535745712469815, + "learning_rate": 4.161824284172985e-06, + "loss": 1.5354, + "step": 526 + }, + { + "epoch": 2.320644216691069, + "grad_norm": 0.3933501995617675, + "learning_rate": 4.14378136324372e-06, + "loss": 1.5932, + "step": 527 + }, + { + "epoch": 2.3250366032210836, + "grad_norm": 0.2955075282035676, + "learning_rate": 4.125749921454679e-06, + "loss": 1.6059, + "step": 528 + }, + { + "epoch": 2.329428989751098, + "grad_norm": 0.3156864130791791, + "learning_rate": 4.107730200549549e-06, + "loss": 1.5252, + "step": 529 + }, + { + "epoch": 2.333821376281113, + "grad_norm": 0.31312483297202565, + "learning_rate": 4.089722442114873e-06, + "loss": 1.6255, + "step": 530 + }, + { + "epoch": 2.3382137628111272, + "grad_norm": 0.3984616100434099, + "learning_rate": 4.071726887576823e-06, + "loss": 1.6074, + "step": 531 + }, + { + "epoch": 2.342606149341142, + "grad_norm": 0.3245701785753724, + "learning_rate": 4.053743778197951e-06, + "loss": 1.6017, + "step": 532 + }, + { + "epoch": 2.3469985358711565, + "grad_norm": 0.3109891324799959, + "learning_rate": 4.035773355073956e-06, + "loss": 1.7142, + "step": 533 + }, + { + "epoch": 2.3513909224011713, + "grad_norm": 0.2812903996182882, + "learning_rate": 4.017815859130462e-06, + "loss": 1.6089, + "step": 534 + }, + { + "epoch": 2.355783308931186, + "grad_norm": 0.31403362482776737, + "learning_rate": 3.999871531119779e-06, + "loss": 1.5455, + "step": 535 + }, + { + "epoch": 2.3601756954612005, + "grad_norm": 0.3608937370940864, + "learning_rate": 3.981940611617675e-06, + "loss": 1.5522, + "step": 536 + }, + { + "epoch": 2.3645680819912154, + "grad_norm": 0.33749233448354743, + "learning_rate": 3.9640233410201555e-06, + "loss": 1.6777, + "step": 537 + }, + { + "epoch": 2.3689604685212298, + "grad_norm": 0.5153655618339453, + "learning_rate": 3.946119959540235e-06, + "loss": 1.5495, + "step": 538 + }, + { + "epoch": 2.3733528550512446, + "grad_norm": 0.31504839746477054, + "learning_rate": 3.928230707204729e-06, + "loss": 1.5521, + "step": 539 + }, + { + "epoch": 2.377745241581259, + "grad_norm": 0.27776484632999904, + "learning_rate": 3.910355823851008e-06, + "loss": 1.5042, + "step": 540 + }, + { + "epoch": 2.382137628111274, + "grad_norm": 0.2939400235215093, + "learning_rate": 3.892495549123821e-06, + "loss": 1.6054, + "step": 541 + }, + { + "epoch": 2.3865300146412887, + "grad_norm": 0.3176044597823968, + "learning_rate": 3.874650122472049e-06, + "loss": 1.5915, + "step": 542 + }, + { + "epoch": 2.390922401171303, + "grad_norm": 0.42146288925638176, + "learning_rate": 3.856819783145514e-06, + "loss": 1.5908, + "step": 543 + }, + { + "epoch": 2.395314787701318, + "grad_norm": 0.30849439803473244, + "learning_rate": 3.8390047701917625e-06, + "loss": 1.6203, + "step": 544 + }, + { + "epoch": 2.3997071742313323, + "grad_norm": 0.3104483621519379, + "learning_rate": 3.821205322452863e-06, + "loss": 1.5298, + "step": 545 + }, + { + "epoch": 2.404099560761347, + "grad_norm": 1.6029942546871143, + "learning_rate": 3.803421678562213e-06, + "loss": 1.4927, + "step": 546 + }, + { + "epoch": 2.4084919472913615, + "grad_norm": 0.332446513039922, + "learning_rate": 3.785654076941317e-06, + "loss": 1.5578, + "step": 547 + }, + { + "epoch": 2.4128843338213763, + "grad_norm": 0.35434480211680486, + "learning_rate": 3.7679027557966136e-06, + "loss": 1.5855, + "step": 548 + }, + { + "epoch": 2.4172767203513907, + "grad_norm": 0.3278075604228065, + "learning_rate": 3.750167953116272e-06, + "loss": 1.6277, + "step": 549 + }, + { + "epoch": 2.4216691068814056, + "grad_norm": 0.28809005974662427, + "learning_rate": 3.7324499066670008e-06, + "loss": 1.6628, + "step": 550 + }, + { + "epoch": 2.42606149341142, + "grad_norm": 0.315430224939953, + "learning_rate": 3.71474885399086e-06, + "loss": 1.5712, + "step": 551 + }, + { + "epoch": 2.430453879941435, + "grad_norm": 0.29541436843467994, + "learning_rate": 3.6970650324020784e-06, + "loss": 1.6554, + "step": 552 + }, + { + "epoch": 2.4348462664714496, + "grad_norm": 0.31836804528358076, + "learning_rate": 3.6793986789838745e-06, + "loss": 1.5242, + "step": 553 + }, + { + "epoch": 2.439238653001464, + "grad_norm": 0.3097916118349815, + "learning_rate": 3.6617500305852676e-06, + "loss": 1.5067, + "step": 554 + }, + { + "epoch": 2.443631039531479, + "grad_norm": 0.30552956835830547, + "learning_rate": 3.6441193238179152e-06, + "loss": 1.5544, + "step": 555 + }, + { + "epoch": 2.4480234260614933, + "grad_norm": 0.33196026415940266, + "learning_rate": 3.6265067950529363e-06, + "loss": 1.5967, + "step": 556 + }, + { + "epoch": 2.452415812591508, + "grad_norm": 0.45992827917949797, + "learning_rate": 3.6089126804177373e-06, + "loss": 1.5314, + "step": 557 + }, + { + "epoch": 2.4568081991215225, + "grad_norm": 0.3850494843555264, + "learning_rate": 3.5913372157928515e-06, + "loss": 1.584, + "step": 558 + }, + { + "epoch": 2.4612005856515373, + "grad_norm": 0.3130838633878226, + "learning_rate": 3.5737806368087776e-06, + "loss": 1.5472, + "step": 559 + }, + { + "epoch": 2.465592972181552, + "grad_norm": 0.33165390842885173, + "learning_rate": 3.556243178842816e-06, + "loss": 1.5396, + "step": 560 + }, + { + "epoch": 2.4699853587115665, + "grad_norm": 0.30410685779778446, + "learning_rate": 3.5387250770159152e-06, + "loss": 1.6436, + "step": 561 + }, + { + "epoch": 2.4743777452415814, + "grad_norm": 0.43575069016621865, + "learning_rate": 3.521226566189523e-06, + "loss": 1.6752, + "step": 562 + }, + { + "epoch": 2.4787701317715958, + "grad_norm": 0.3043077774937357, + "learning_rate": 3.5037478809624315e-06, + "loss": 1.5879, + "step": 563 + }, + { + "epoch": 2.4831625183016106, + "grad_norm": 0.29787633854732515, + "learning_rate": 3.48628925566764e-06, + "loss": 1.6488, + "step": 564 + }, + { + "epoch": 2.487554904831625, + "grad_norm": 0.33159507739731536, + "learning_rate": 3.4688509243692037e-06, + "loss": 1.5911, + "step": 565 + }, + { + "epoch": 2.49194729136164, + "grad_norm": 0.2790057356017803, + "learning_rate": 3.4514331208591027e-06, + "loss": 1.6126, + "step": 566 + }, + { + "epoch": 2.4963396778916547, + "grad_norm": 0.307396122565463, + "learning_rate": 3.4340360786541067e-06, + "loss": 1.6613, + "step": 567 + }, + { + "epoch": 2.500732064421669, + "grad_norm": 0.2896656371367208, + "learning_rate": 3.416660030992639e-06, + "loss": 1.6264, + "step": 568 + }, + { + "epoch": 2.505124450951684, + "grad_norm": 0.28661415223621256, + "learning_rate": 3.3993052108316566e-06, + "loss": 1.5758, + "step": 569 + }, + { + "epoch": 2.5095168374816983, + "grad_norm": 0.41303310304829133, + "learning_rate": 3.381971850843523e-06, + "loss": 1.5584, + "step": 570 + } + ], + "logging_steps": 1, + "max_steps": 908, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 114, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5070953314210284e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-570/training_args.bin b/checkpoint-570/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..282808fde3b579abe8666eba7a8b5732d5eaf3d4 --- /dev/null +++ b/checkpoint-570/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f598588716f36b5db7091989142a430a4fd275bce6f1e0d8e6c17f595272e1ec +size 9041 diff --git a/checkpoint-570/zero_to_fp32.py b/checkpoint-570/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint-570/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/checkpoint-684/README.md b/checkpoint-684/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb5cf17f38ede36f3fb46a89995daa3f44167898 --- /dev/null +++ b/checkpoint-684/README.md @@ -0,0 +1,202 @@ +--- +base_model: Gryphe/Codex-24B-Small-3.2 +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.15.2 \ No newline at end of file diff --git a/checkpoint-684/adapter_config.json b/checkpoint-684/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fa0535ca25a4eef78d9c51576f13234c64c95fb --- /dev/null +++ b/checkpoint-684/adapter_config.json @@ -0,0 +1,39 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "Gryphe/Codex-24B-Small-3.2", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_bias": false, + "lora_dropout": 0.5, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "o_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/checkpoint-684/adapter_model.safetensors b/checkpoint-684/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..44985e4a381c756724f0e7258609e9b7c5d34a36 --- /dev/null +++ b/checkpoint-684/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cce2c3cec65e9ba485c79ffbf17f93d2c5210182ec89d00c78c20fff345d9e01 +size 1478569432 diff --git a/checkpoint-684/global_step685/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-684/global_step685/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..627f6636a2df7d9cee579ef8f34edef7a7ca3437 --- /dev/null +++ b/checkpoint-684/global_step685/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c68d10147b838927053b18d410c20e8b45673dc297012f4289f1f5d1b9fef1 +size 4459517761 diff --git a/checkpoint-684/global_step685/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-684/global_step685/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5630076c79cf4a31461f7c19dbe9d0985614d9e0 --- /dev/null +++ b/checkpoint-684/global_step685/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434454d57799e270f6866e3c51ac9f1f8b0436d96eef3ff8f0e6d333855ad842 +size 557481 diff --git a/checkpoint-684/latest b/checkpoint-684/latest new file mode 100644 index 0000000000000000000000000000000000000000..df45eb9a7e55c08272a8fd526a06f6722b83a3f4 --- /dev/null +++ b/checkpoint-684/latest @@ -0,0 +1 @@ +global_step685 \ No newline at end of file diff --git a/checkpoint-684/rng_state.pth b/checkpoint-684/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..f9ebb072866f7656c61d096ea146fcbcbc10cbb6 --- /dev/null +++ b/checkpoint-684/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd2c60f6a0b48fdba81dac90a3305fb52c9a834d53107813f677ac203436fe5 +size 14709 diff --git a/checkpoint-684/scheduler.pt b/checkpoint-684/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecb3ff19a3d771c6387703d8387bed8986e95ac5 --- /dev/null +++ b/checkpoint-684/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97a2c15c77c83da3fdc303236b6cbc56fa0666beae56c64e73279b1a74f3a8c +size 1401 diff --git a/checkpoint-684/special_tokens_map.json b/checkpoint-684/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..4332a4c6d2a485806f55e02260db8a73698e11fb --- /dev/null +++ b/checkpoint-684/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-684/tokenizer.json b/checkpoint-684/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8fab5451b04e019d1612cc5088df62aee9a8208c --- /dev/null +++ b/checkpoint-684/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e5daf1e619597fd56ff036925ea189916417c665d5bdd61b80199cf8d4e6d2 +size 17078029 diff --git a/checkpoint-684/tokenizer_config.json b/checkpoint-684/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..814201fe59040b12e956fb649c749a42bdce1e2f --- /dev/null +++ b/checkpoint-684/tokenizer_config.json @@ -0,0 +1,8021 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "[IMG]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "[IMG_BREAK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "[IMG_END]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "[PREFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "[MIDDLE]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "[SUFFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "[SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "[/SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "[TOOL_CONTENT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "109": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "110": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "111": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "112": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "113": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "114": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "115": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "116": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "117": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "118": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "119": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "120": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "121": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "122": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "123": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "124": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "125": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "126": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "127": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "129": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "130": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "132": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "133": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "134": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "135": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "136": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "137": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "138": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "139": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "140": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "141": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "142": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "143": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "144": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "145": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "146": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "147": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "148": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "149": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "150": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "152": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "153": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "154": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "155": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "156": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "157": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "158": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "159": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "160": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "161": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "162": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "164": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "165": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "166": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "167": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "185": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "186": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "187": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "188": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "189": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "190": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "191": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "222": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "223": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "224": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "225": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "226": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "227": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "228": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "229": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "230": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "231": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "232": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "233": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "234": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "235": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "236": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "237": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "238": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "239": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "240": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "241": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "242": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "243": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "244": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "245": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "246": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "247": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "249": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "251": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "252": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "253": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "254": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "255": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "257": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "258": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "259": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "260": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "262": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "263": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "264": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "265": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "266": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "267": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "268": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "285": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "286": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "287": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "288": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "289": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "290": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "291": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "292": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "293": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "294": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "295": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "296": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "297": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "298": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "299": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "300": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "301": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "302": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "303": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "304": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "305": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "306": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "307": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "308": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "309": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "310": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "311": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "312": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "313": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "314": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "315": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "316": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "317": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "318": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "319": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "320": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "321": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "322": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "323": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "324": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "325": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "326": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "327": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "328": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "329": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "330": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "331": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "332": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "335": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "336": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "337": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "338": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "339": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "340": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "341": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "342": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "343": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "344": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "345": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "346": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "347": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "348": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "349": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "360": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "361": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "362": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "363": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "364": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "365": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "366": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "367": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "368": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "369": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "370": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "371": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "372": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "373": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "374": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "375": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "376": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "377": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "378": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "379": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "380": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "381": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "382": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "383": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "384": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "385": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "386": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "387": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "388": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "389": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "390": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "391": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "392": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "393": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "394": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "395": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "396": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "397": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "398": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "399": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "400": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "401": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "402": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "403": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "404": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "405": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "406": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "407": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "408": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "409": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "410": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "411": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "412": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "413": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "414": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "415": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "416": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "417": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "418": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "419": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "420": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "421": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "422": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "423": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "424": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "425": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "426": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "427": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "428": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "429": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "430": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "431": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "432": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "433": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "434": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "435": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "436": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "437": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "438": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "439": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "440": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "441": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "442": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "443": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "444": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "445": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "446": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "447": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "448": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "449": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "450": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "451": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "452": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "453": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "454": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "455": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "456": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "457": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "458": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "459": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "460": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "461": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "462": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "463": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "464": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "465": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "466": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "467": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "468": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "469": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "470": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "471": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "472": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "473": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "474": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "475": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "476": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "477": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "478": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "479": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "480": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "481": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "482": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "483": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "484": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "485": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "486": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "487": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "488": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "489": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "490": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "491": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "492": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "493": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "494": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "495": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "496": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "497": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "498": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "499": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "500": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "501": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "502": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "503": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "504": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "505": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "506": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "507": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "508": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "509": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "510": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "511": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "512": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "513": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "514": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "515": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "516": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "517": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "518": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "519": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "520": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "521": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "522": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "523": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "524": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "525": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "526": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "527": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "528": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "529": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "530": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "531": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "532": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "533": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "534": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "535": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "536": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "537": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "538": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "539": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "540": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "541": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "542": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "543": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "544": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "545": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "546": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "547": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "548": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "549": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "550": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "551": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "552": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "553": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "554": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "555": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "556": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "557": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "558": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "559": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "560": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "561": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "562": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "563": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "564": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "565": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "566": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "567": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "568": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "569": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "570": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "571": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "572": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "573": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "574": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "575": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "576": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "577": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "578": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "579": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "580": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "581": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "582": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "583": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "584": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "585": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "586": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "587": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "588": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "589": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "590": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "591": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "592": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "593": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "594": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "595": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "596": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "597": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "598": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "599": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "600": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "601": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "602": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "603": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "604": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "605": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "606": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "607": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "608": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "609": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "610": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "611": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "612": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "613": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "614": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "615": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "616": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "617": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "618": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "619": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "620": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "621": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "622": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "623": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "624": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "625": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "626": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "627": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "628": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "629": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "630": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "631": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "632": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "633": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "634": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "635": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "636": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "637": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "638": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "639": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "640": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "641": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "642": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "643": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "644": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "645": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "646": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "647": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "650": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "651": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "652": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "653": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "654": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "655": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "656": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "659": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "660": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "661": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "662": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "663": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "664": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "675": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "676": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "677": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "678": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "679": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "680": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "681": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "682": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "683": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "684": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "685": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "686": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "687": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "688": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "689": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "690": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "691": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "692": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "693": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "694": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "695": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "696": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "697": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "698": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "699": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "700": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "701": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "702": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "703": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "704": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "705": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "706": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "707": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "708": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "709": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "710": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "711": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "712": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "713": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "714": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "715": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "716": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "717": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "718": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "719": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "720": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "721": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "722": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "723": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "724": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "725": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "726": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "727": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "728": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "729": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "730": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "731": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "732": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "733": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "734": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "735": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "736": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "737": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "738": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "739": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "740": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "741": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "742": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "743": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "744": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "745": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "746": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "747": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "748": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "749": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "750": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "751": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "752": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "753": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "754": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "755": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "756": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "757": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "758": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "759": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "760": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "761": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "762": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "763": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "764": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "765": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "766": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "767": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "768": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "769": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "770": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "771": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "772": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "773": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "774": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "775": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "776": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "777": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "778": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "779": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "780": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "781": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "782": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "783": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "784": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "785": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "786": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "787": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "788": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "789": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "790": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "791": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "792": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "793": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "794": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "795": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "796": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "797": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "798": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "799": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "800": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "801": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "802": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "803": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "804": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "805": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "806": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "807": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "808": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "809": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "810": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "811": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "812": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "813": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "814": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "815": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "816": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "817": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "818": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "819": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "820": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "821": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "822": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "823": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "824": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "825": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "826": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "827": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "828": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "829": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "830": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "831": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "832": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "833": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "834": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "835": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "836": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "837": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "838": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "839": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "840": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "841": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "842": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "843": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "844": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "845": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "846": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "847": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "848": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "849": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "850": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "851": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "852": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "853": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "854": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "855": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "856": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "857": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "863": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "864": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "865": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "866": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "867": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "868": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "869": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "870": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "871": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "880": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "881": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "882": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "883": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "884": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "885": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "886": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "887": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "888": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "889": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "890": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "891": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "892": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "893": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "894": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "895": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "896": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "897": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "898": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "899": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "900": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "901": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "902": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "903": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "904": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "905": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "906": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "907": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "908": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "909": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "910": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "911": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "912": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "913": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "914": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "915": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "916": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "917": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "918": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "919": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "920": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "921": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "922": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "923": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "924": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "925": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "926": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "927": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "928": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "929": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "930": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "931": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "932": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "933": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "934": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "935": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "936": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "937": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "938": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "939": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "940": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "941": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "942": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "943": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "944": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "945": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "946": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "947": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "948": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "949": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "950": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "951": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "952": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "953": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "954": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "955": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "956": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "957": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "958": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "959": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "960": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "961": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "962": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "963": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "964": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "965": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "966": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "967": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "968": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "969": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "970": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "971": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "972": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "973": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "974": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "975": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "976": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "977": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "978": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "979": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "980": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "981": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "982": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "983": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "984": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "985": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "986": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "987": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "988": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "989": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "990": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "991": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "992": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "993": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "994": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "995": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "996": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "997": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "998": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "999": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 131072, + "pad_token": "", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-684/trainer_state.json b/checkpoint-684/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1125781636ab8b40e75cf7deefb2812c7ceb76b7 --- /dev/null +++ b/checkpoint-684/trainer_state.json @@ -0,0 +1,4822 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.013177159590044, + "eval_steps": 500, + "global_step": 684, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004392386530014641, + "grad_norm": 2.023870796411983, + "learning_rate": 0.0, + "loss": 1.8021, + "step": 1 + }, + { + "epoch": 0.008784773060029283, + "grad_norm": 2.0723948799697984, + "learning_rate": 2.0000000000000002e-07, + "loss": 1.8796, + "step": 2 + }, + { + "epoch": 0.013177159590043924, + "grad_norm": 2.0322569672979474, + "learning_rate": 4.0000000000000003e-07, + "loss": 1.855, + "step": 3 + }, + { + "epoch": 0.017569546120058566, + "grad_norm": 2.111472782868084, + "learning_rate": 6.000000000000001e-07, + "loss": 1.8545, + "step": 4 + }, + { + "epoch": 0.021961932650073207, + "grad_norm": 2.096607607233168, + "learning_rate": 8.000000000000001e-07, + "loss": 1.807, + "step": 5 + }, + { + "epoch": 0.02635431918008785, + "grad_norm": 2.0771855831079518, + "learning_rate": 1.0000000000000002e-06, + "loss": 1.8208, + "step": 6 + }, + { + "epoch": 0.03074670571010249, + "grad_norm": 2.0341840619969567, + "learning_rate": 1.2000000000000002e-06, + "loss": 1.8754, + "step": 7 + }, + { + "epoch": 0.03513909224011713, + "grad_norm": 2.0564329237094356, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.8882, + "step": 8 + }, + { + "epoch": 0.03953147877013177, + "grad_norm": 1.9900074702634307, + "learning_rate": 1.6000000000000001e-06, + "loss": 1.8733, + "step": 9 + }, + { + "epoch": 0.043923865300146414, + "grad_norm": 1.895599415585885, + "learning_rate": 1.8000000000000001e-06, + "loss": 1.8474, + "step": 10 + }, + { + "epoch": 0.048316251830161056, + "grad_norm": 1.9304209419460994, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.884, + "step": 11 + }, + { + "epoch": 0.0527086383601757, + "grad_norm": 1.6511439041472846, + "learning_rate": 2.2e-06, + "loss": 1.8452, + "step": 12 + }, + { + "epoch": 0.05710102489019034, + "grad_norm": 1.6727753891125912, + "learning_rate": 2.4000000000000003e-06, + "loss": 1.8632, + "step": 13 + }, + { + "epoch": 0.06149341142020498, + "grad_norm": 1.6571693399048197, + "learning_rate": 2.6e-06, + "loss": 1.8419, + "step": 14 + }, + { + "epoch": 0.06588579795021962, + "grad_norm": 1.5588712723347087, + "learning_rate": 2.8000000000000003e-06, + "loss": 1.8163, + "step": 15 + }, + { + "epoch": 0.07027818448023426, + "grad_norm": 1.57706405792705, + "learning_rate": 3e-06, + "loss": 1.7808, + "step": 16 + }, + { + "epoch": 0.0746705710102489, + "grad_norm": 1.4841447475095337, + "learning_rate": 3.2000000000000003e-06, + "loss": 1.7854, + "step": 17 + }, + { + "epoch": 0.07906295754026355, + "grad_norm": 1.2785291394155553, + "learning_rate": 3.4000000000000005e-06, + "loss": 1.7531, + "step": 18 + }, + { + "epoch": 0.08345534407027819, + "grad_norm": 1.1759630521885338, + "learning_rate": 3.6000000000000003e-06, + "loss": 1.8185, + "step": 19 + }, + { + "epoch": 0.08784773060029283, + "grad_norm": 1.1188613513306425, + "learning_rate": 3.8000000000000005e-06, + "loss": 1.849, + "step": 20 + }, + { + "epoch": 0.09224011713030747, + "grad_norm": 1.008741263333265, + "learning_rate": 4.000000000000001e-06, + "loss": 1.775, + "step": 21 + }, + { + "epoch": 0.09663250366032211, + "grad_norm": 0.9240991279618896, + "learning_rate": 4.2000000000000004e-06, + "loss": 1.8253, + "step": 22 + }, + { + "epoch": 0.10102489019033675, + "grad_norm": 0.9015845652170666, + "learning_rate": 4.4e-06, + "loss": 1.7967, + "step": 23 + }, + { + "epoch": 0.1054172767203514, + "grad_norm": 0.8007281301128234, + "learning_rate": 4.600000000000001e-06, + "loss": 1.7958, + "step": 24 + }, + { + "epoch": 0.10980966325036604, + "grad_norm": 0.7263390923205475, + "learning_rate": 4.800000000000001e-06, + "loss": 1.7744, + "step": 25 + }, + { + "epoch": 0.11420204978038068, + "grad_norm": 0.6527771074756341, + "learning_rate": 5e-06, + "loss": 1.7502, + "step": 26 + }, + { + "epoch": 0.11859443631039532, + "grad_norm": 0.5943189060332841, + "learning_rate": 5.2e-06, + "loss": 1.7378, + "step": 27 + }, + { + "epoch": 0.12298682284040996, + "grad_norm": 0.5426800655727773, + "learning_rate": 5.400000000000001e-06, + "loss": 1.7372, + "step": 28 + }, + { + "epoch": 0.1273792093704246, + "grad_norm": 0.46887045607953654, + "learning_rate": 5.600000000000001e-06, + "loss": 1.7141, + "step": 29 + }, + { + "epoch": 0.13177159590043924, + "grad_norm": 0.4408809706079413, + "learning_rate": 5.8e-06, + "loss": 1.6977, + "step": 30 + }, + { + "epoch": 0.13616398243045388, + "grad_norm": 0.4867065604810947, + "learning_rate": 6e-06, + "loss": 1.7379, + "step": 31 + }, + { + "epoch": 0.14055636896046853, + "grad_norm": 0.40882779662256746, + "learning_rate": 6.200000000000001e-06, + "loss": 1.8064, + "step": 32 + }, + { + "epoch": 0.14494875549048317, + "grad_norm": 0.4363561706655908, + "learning_rate": 6.4000000000000006e-06, + "loss": 1.6835, + "step": 33 + }, + { + "epoch": 0.1493411420204978, + "grad_norm": 0.4443411222267533, + "learning_rate": 6.600000000000001e-06, + "loss": 1.7464, + "step": 34 + }, + { + "epoch": 0.15373352855051245, + "grad_norm": 0.4519702758206936, + "learning_rate": 6.800000000000001e-06, + "loss": 1.6891, + "step": 35 + }, + { + "epoch": 0.1581259150805271, + "grad_norm": 0.4174999907173025, + "learning_rate": 7e-06, + "loss": 1.7262, + "step": 36 + }, + { + "epoch": 0.16251830161054173, + "grad_norm": 0.46742969885702906, + "learning_rate": 7.2000000000000005e-06, + "loss": 1.7608, + "step": 37 + }, + { + "epoch": 0.16691068814055637, + "grad_norm": 0.4155692294314683, + "learning_rate": 7.4e-06, + "loss": 1.7211, + "step": 38 + }, + { + "epoch": 0.17130307467057102, + "grad_norm": 0.4123169421407387, + "learning_rate": 7.600000000000001e-06, + "loss": 1.6956, + "step": 39 + }, + { + "epoch": 0.17569546120058566, + "grad_norm": 0.3887606381731466, + "learning_rate": 7.800000000000002e-06, + "loss": 1.7391, + "step": 40 + }, + { + "epoch": 0.1800878477306003, + "grad_norm": 0.37535703391318914, + "learning_rate": 8.000000000000001e-06, + "loss": 1.751, + "step": 41 + }, + { + "epoch": 0.18448023426061494, + "grad_norm": 0.48621902751310403, + "learning_rate": 8.2e-06, + "loss": 1.7091, + "step": 42 + }, + { + "epoch": 0.18887262079062958, + "grad_norm": 0.39145172998871297, + "learning_rate": 8.400000000000001e-06, + "loss": 1.7764, + "step": 43 + }, + { + "epoch": 0.19326500732064422, + "grad_norm": 0.3735488902443606, + "learning_rate": 8.6e-06, + "loss": 1.7058, + "step": 44 + }, + { + "epoch": 0.19765739385065886, + "grad_norm": 0.33988576479414434, + "learning_rate": 8.8e-06, + "loss": 1.7305, + "step": 45 + }, + { + "epoch": 0.2020497803806735, + "grad_norm": 0.9733163465798257, + "learning_rate": 9e-06, + "loss": 1.7567, + "step": 46 + }, + { + "epoch": 0.20644216691068815, + "grad_norm": 0.3344321176956884, + "learning_rate": 9.200000000000002e-06, + "loss": 1.7277, + "step": 47 + }, + { + "epoch": 0.2108345534407028, + "grad_norm": 0.3647805237752918, + "learning_rate": 9.4e-06, + "loss": 1.7197, + "step": 48 + }, + { + "epoch": 0.21522693997071743, + "grad_norm": 0.36967036325253694, + "learning_rate": 9.600000000000001e-06, + "loss": 1.6994, + "step": 49 + }, + { + "epoch": 0.21961932650073207, + "grad_norm": 0.32386928338357696, + "learning_rate": 9.800000000000001e-06, + "loss": 1.6975, + "step": 50 + }, + { + "epoch": 0.2240117130307467, + "grad_norm": 0.33940794357965215, + "learning_rate": 1e-05, + "loss": 1.7278, + "step": 51 + }, + { + "epoch": 0.22840409956076135, + "grad_norm": 0.3396058372359069, + "learning_rate": 9.999966483034437e-06, + "loss": 1.6342, + "step": 52 + }, + { + "epoch": 0.232796486090776, + "grad_norm": 0.3395876900960321, + "learning_rate": 9.999865932587107e-06, + "loss": 1.6846, + "step": 53 + }, + { + "epoch": 0.23718887262079064, + "grad_norm": 0.3320830942903407, + "learning_rate": 9.999698350006063e-06, + "loss": 1.6744, + "step": 54 + }, + { + "epoch": 0.24158125915080528, + "grad_norm": 0.318642304519145, + "learning_rate": 9.999463737538054e-06, + "loss": 1.6633, + "step": 55 + }, + { + "epoch": 0.24597364568081992, + "grad_norm": 0.3302002388842341, + "learning_rate": 9.999162098328475e-06, + "loss": 1.6994, + "step": 56 + }, + { + "epoch": 0.25036603221083453, + "grad_norm": 0.30428798634415904, + "learning_rate": 9.998793436421342e-06, + "loss": 1.6728, + "step": 57 + }, + { + "epoch": 0.2547584187408492, + "grad_norm": 0.30662673120989614, + "learning_rate": 9.998357756759223e-06, + "loss": 1.6283, + "step": 58 + }, + { + "epoch": 0.2591508052708638, + "grad_norm": 0.34305953470241257, + "learning_rate": 9.997855065183185e-06, + "loss": 1.7269, + "step": 59 + }, + { + "epoch": 0.2635431918008785, + "grad_norm": 0.29647698247102616, + "learning_rate": 9.997285368432704e-06, + "loss": 1.6108, + "step": 60 + }, + { + "epoch": 0.2679355783308931, + "grad_norm": 0.30904144706299363, + "learning_rate": 9.996648674145584e-06, + "loss": 1.5925, + "step": 61 + }, + { + "epoch": 0.27232796486090777, + "grad_norm": 0.28294691041282544, + "learning_rate": 9.995944990857848e-06, + "loss": 1.7004, + "step": 62 + }, + { + "epoch": 0.2767203513909224, + "grad_norm": 0.4537316624659888, + "learning_rate": 9.99517432800363e-06, + "loss": 1.673, + "step": 63 + }, + { + "epoch": 0.28111273792093705, + "grad_norm": 0.33334142649617554, + "learning_rate": 9.994336695915041e-06, + "loss": 1.7305, + "step": 64 + }, + { + "epoch": 0.28550512445095166, + "grad_norm": 0.2906595328556503, + "learning_rate": 9.993432105822034e-06, + "loss": 1.6511, + "step": 65 + }, + { + "epoch": 0.28989751098096633, + "grad_norm": 0.30937353104293036, + "learning_rate": 9.992460569852256e-06, + "loss": 1.7055, + "step": 66 + }, + { + "epoch": 0.29428989751098095, + "grad_norm": 0.29714918983678074, + "learning_rate": 9.99142210103088e-06, + "loss": 1.6022, + "step": 67 + }, + { + "epoch": 0.2986822840409956, + "grad_norm": 0.3077061364082642, + "learning_rate": 9.99031671328044e-06, + "loss": 1.6167, + "step": 68 + }, + { + "epoch": 0.30307467057101023, + "grad_norm": 0.31797999416909106, + "learning_rate": 9.98914442142063e-06, + "loss": 1.6063, + "step": 69 + }, + { + "epoch": 0.3074670571010249, + "grad_norm": 0.28544810434107143, + "learning_rate": 9.987905241168117e-06, + "loss": 1.673, + "step": 70 + }, + { + "epoch": 0.3118594436310395, + "grad_norm": 0.3113516293182134, + "learning_rate": 9.986599189136325e-06, + "loss": 1.6759, + "step": 71 + }, + { + "epoch": 0.3162518301610542, + "grad_norm": 0.32958924621680397, + "learning_rate": 9.985226282835216e-06, + "loss": 1.6894, + "step": 72 + }, + { + "epoch": 0.3206442166910688, + "grad_norm": 0.3189944736085633, + "learning_rate": 9.983786540671052e-06, + "loss": 1.7762, + "step": 73 + }, + { + "epoch": 0.32503660322108346, + "grad_norm": 0.30229970638681786, + "learning_rate": 9.982279981946143e-06, + "loss": 1.709, + "step": 74 + }, + { + "epoch": 0.3294289897510981, + "grad_norm": 0.308085411523194, + "learning_rate": 9.980706626858607e-06, + "loss": 1.6741, + "step": 75 + }, + { + "epoch": 0.33382137628111275, + "grad_norm": 0.2851270169881767, + "learning_rate": 9.979066496502075e-06, + "loss": 1.656, + "step": 76 + }, + { + "epoch": 0.33821376281112736, + "grad_norm": 0.3115977673015286, + "learning_rate": 9.977359612865424e-06, + "loss": 1.6633, + "step": 77 + }, + { + "epoch": 0.34260614934114203, + "grad_norm": 0.2960787226189042, + "learning_rate": 9.97558599883248e-06, + "loss": 1.673, + "step": 78 + }, + { + "epoch": 0.34699853587115664, + "grad_norm": 0.35701812130442995, + "learning_rate": 9.973745678181705e-06, + "loss": 1.6103, + "step": 79 + }, + { + "epoch": 0.3513909224011713, + "grad_norm": 0.367868955812032, + "learning_rate": 9.971838675585888e-06, + "loss": 1.6768, + "step": 80 + }, + { + "epoch": 0.3557833089311859, + "grad_norm": 0.3295428259298103, + "learning_rate": 9.9698650166118e-06, + "loss": 1.6668, + "step": 81 + }, + { + "epoch": 0.3601756954612006, + "grad_norm": 0.3242883305567894, + "learning_rate": 9.96782472771987e-06, + "loss": 1.7152, + "step": 82 + }, + { + "epoch": 0.3645680819912152, + "grad_norm": 0.3070622417895325, + "learning_rate": 9.965717836263813e-06, + "loss": 1.5697, + "step": 83 + }, + { + "epoch": 0.3689604685212299, + "grad_norm": 0.27753629978682975, + "learning_rate": 9.96354437049027e-06, + "loss": 1.6453, + "step": 84 + }, + { + "epoch": 0.3733528550512445, + "grad_norm": 0.2723902519890988, + "learning_rate": 9.961304359538437e-06, + "loss": 1.7165, + "step": 85 + }, + { + "epoch": 0.37774524158125916, + "grad_norm": 0.29061586618994084, + "learning_rate": 9.958997833439658e-06, + "loss": 1.7135, + "step": 86 + }, + { + "epoch": 0.3821376281112738, + "grad_norm": 0.33085661251847487, + "learning_rate": 9.956624823117036e-06, + "loss": 1.6109, + "step": 87 + }, + { + "epoch": 0.38653001464128844, + "grad_norm": 0.2931612046141165, + "learning_rate": 9.954185360385013e-06, + "loss": 1.7048, + "step": 88 + }, + { + "epoch": 0.39092240117130306, + "grad_norm": 0.2900600158100116, + "learning_rate": 9.951679477948946e-06, + "loss": 1.6861, + "step": 89 + }, + { + "epoch": 0.3953147877013177, + "grad_norm": 0.2775954202644971, + "learning_rate": 9.949107209404664e-06, + "loss": 1.5706, + "step": 90 + }, + { + "epoch": 0.39970717423133234, + "grad_norm": 0.2733537235090247, + "learning_rate": 9.946468589238022e-06, + "loss": 1.7146, + "step": 91 + }, + { + "epoch": 0.404099560761347, + "grad_norm": 0.2710139549067813, + "learning_rate": 9.943763652824437e-06, + "loss": 1.6488, + "step": 92 + }, + { + "epoch": 0.4084919472913616, + "grad_norm": 0.27892641103107996, + "learning_rate": 9.94099243642841e-06, + "loss": 1.6002, + "step": 93 + }, + { + "epoch": 0.4128843338213763, + "grad_norm": 0.4570340740318857, + "learning_rate": 9.93815497720305e-06, + "loss": 1.6686, + "step": 94 + }, + { + "epoch": 0.4172767203513909, + "grad_norm": 0.2816582371174757, + "learning_rate": 9.935251313189564e-06, + "loss": 1.7531, + "step": 95 + }, + { + "epoch": 0.4216691068814056, + "grad_norm": 0.2886345479986427, + "learning_rate": 9.932281483316759e-06, + "loss": 1.744, + "step": 96 + }, + { + "epoch": 0.4260614934114202, + "grad_norm": 0.3178449956639589, + "learning_rate": 9.929245527400504e-06, + "loss": 1.6404, + "step": 97 + }, + { + "epoch": 0.43045387994143486, + "grad_norm": 0.31466032997182714, + "learning_rate": 9.926143486143216e-06, + "loss": 1.6583, + "step": 98 + }, + { + "epoch": 0.43484626647144947, + "grad_norm": 0.3406321749653024, + "learning_rate": 9.922975401133292e-06, + "loss": 1.7313, + "step": 99 + }, + { + "epoch": 0.43923865300146414, + "grad_norm": 0.33050248521276526, + "learning_rate": 9.919741314844577e-06, + "loss": 1.6518, + "step": 100 + }, + { + "epoch": 0.44363103953147875, + "grad_norm": 0.2886134577231967, + "learning_rate": 9.916441270635772e-06, + "loss": 1.6957, + "step": 101 + }, + { + "epoch": 0.4480234260614934, + "grad_norm": 0.27520985458354463, + "learning_rate": 9.913075312749867e-06, + "loss": 1.6052, + "step": 102 + }, + { + "epoch": 0.45241581259150804, + "grad_norm": 0.2649316531653303, + "learning_rate": 9.909643486313533e-06, + "loss": 1.6739, + "step": 103 + }, + { + "epoch": 0.4568081991215227, + "grad_norm": 0.27557834576379425, + "learning_rate": 9.90614583733654e-06, + "loss": 1.6584, + "step": 104 + }, + { + "epoch": 0.4612005856515373, + "grad_norm": 0.3243046224133678, + "learning_rate": 9.90258241271112e-06, + "loss": 1.6491, + "step": 105 + }, + { + "epoch": 0.465592972181552, + "grad_norm": 0.2601090357711318, + "learning_rate": 9.89895326021134e-06, + "loss": 1.6662, + "step": 106 + }, + { + "epoch": 0.4699853587115666, + "grad_norm": 0.2842151241183655, + "learning_rate": 9.895258428492475e-06, + "loss": 1.6501, + "step": 107 + }, + { + "epoch": 0.4743777452415813, + "grad_norm": 0.2594313373745908, + "learning_rate": 9.891497967090344e-06, + "loss": 1.6475, + "step": 108 + }, + { + "epoch": 0.4787701317715959, + "grad_norm": 0.2844945420500575, + "learning_rate": 9.887671926420649e-06, + "loss": 1.7204, + "step": 109 + }, + { + "epoch": 0.48316251830161056, + "grad_norm": 0.27655451441999607, + "learning_rate": 9.8837803577783e-06, + "loss": 1.6838, + "step": 110 + }, + { + "epoch": 0.48755490483162517, + "grad_norm": 0.3017150137441088, + "learning_rate": 9.879823313336723e-06, + "loss": 1.6688, + "step": 111 + }, + { + "epoch": 0.49194729136163984, + "grad_norm": 0.26575250791745125, + "learning_rate": 9.87580084614717e-06, + "loss": 1.6753, + "step": 112 + }, + { + "epoch": 0.49633967789165445, + "grad_norm": 0.2581989783149807, + "learning_rate": 9.871713010137998e-06, + "loss": 1.6244, + "step": 113 + }, + { + "epoch": 0.5007320644216691, + "grad_norm": 0.2622294231157268, + "learning_rate": 9.86755986011395e-06, + "loss": 1.6261, + "step": 114 + }, + { + "epoch": 0.5051244509516838, + "grad_norm": 0.26209513370076304, + "learning_rate": 9.86334145175542e-06, + "loss": 1.6397, + "step": 115 + }, + { + "epoch": 0.5095168374816984, + "grad_norm": 0.2846184522412747, + "learning_rate": 9.859057841617709e-06, + "loss": 1.7091, + "step": 116 + }, + { + "epoch": 0.513909224011713, + "grad_norm": 0.2528850826310284, + "learning_rate": 9.854709087130261e-06, + "loss": 1.6831, + "step": 117 + }, + { + "epoch": 0.5183016105417276, + "grad_norm": 0.32698350420324157, + "learning_rate": 9.850295246595898e-06, + "loss": 1.6431, + "step": 118 + }, + { + "epoch": 0.5226939970717424, + "grad_norm": 0.35727071053972576, + "learning_rate": 9.845816379190037e-06, + "loss": 1.7088, + "step": 119 + }, + { + "epoch": 0.527086383601757, + "grad_norm": 0.28971742333278044, + "learning_rate": 9.841272544959892e-06, + "loss": 1.7195, + "step": 120 + }, + { + "epoch": 0.5314787701317716, + "grad_norm": 0.468869869211724, + "learning_rate": 9.836663804823683e-06, + "loss": 1.6896, + "step": 121 + }, + { + "epoch": 0.5358711566617862, + "grad_norm": 0.3908944943187778, + "learning_rate": 9.831990220569801e-06, + "loss": 1.7406, + "step": 122 + }, + { + "epoch": 0.5402635431918009, + "grad_norm": 0.2665924461613152, + "learning_rate": 9.827251854855992e-06, + "loss": 1.6609, + "step": 123 + }, + { + "epoch": 0.5446559297218155, + "grad_norm": 0.28211372742262864, + "learning_rate": 9.82244877120851e-06, + "loss": 1.6684, + "step": 124 + }, + { + "epoch": 0.5490483162518301, + "grad_norm": 0.2623738132657017, + "learning_rate": 9.817581034021273e-06, + "loss": 1.6923, + "step": 125 + }, + { + "epoch": 0.5534407027818448, + "grad_norm": 0.35357676308356323, + "learning_rate": 9.812648708554991e-06, + "loss": 1.6467, + "step": 126 + }, + { + "epoch": 0.5578330893118595, + "grad_norm": 0.2555639734647961, + "learning_rate": 9.807651860936299e-06, + "loss": 1.6893, + "step": 127 + }, + { + "epoch": 0.5622254758418741, + "grad_norm": 0.26746261958597517, + "learning_rate": 9.802590558156863e-06, + "loss": 1.6754, + "step": 128 + }, + { + "epoch": 0.5666178623718887, + "grad_norm": 0.26383031628885084, + "learning_rate": 9.797464868072489e-06, + "loss": 1.6363, + "step": 129 + }, + { + "epoch": 0.5710102489019033, + "grad_norm": 0.319619389966322, + "learning_rate": 9.792274859402205e-06, + "loss": 1.636, + "step": 130 + }, + { + "epoch": 0.575402635431918, + "grad_norm": 0.2639308579990406, + "learning_rate": 9.787020601727353e-06, + "loss": 1.6079, + "step": 131 + }, + { + "epoch": 0.5797950219619327, + "grad_norm": 0.28226380411742236, + "learning_rate": 9.781702165490638e-06, + "loss": 1.657, + "step": 132 + }, + { + "epoch": 0.5841874084919473, + "grad_norm": 0.27157003543418035, + "learning_rate": 9.7763196219952e-06, + "loss": 1.7009, + "step": 133 + }, + { + "epoch": 0.5885797950219619, + "grad_norm": 0.28487209604498265, + "learning_rate": 9.770873043403648e-06, + "loss": 1.6236, + "step": 134 + }, + { + "epoch": 0.5929721815519766, + "grad_norm": 0.3009729998770739, + "learning_rate": 9.765362502737098e-06, + "loss": 1.6812, + "step": 135 + }, + { + "epoch": 0.5973645680819912, + "grad_norm": 0.2576811873616972, + "learning_rate": 9.759788073874188e-06, + "loss": 1.6885, + "step": 136 + }, + { + "epoch": 0.6017569546120058, + "grad_norm": 0.3108368566509119, + "learning_rate": 9.754149831550097e-06, + "loss": 1.6086, + "step": 137 + }, + { + "epoch": 0.6061493411420205, + "grad_norm": 0.3336050806293322, + "learning_rate": 9.748447851355534e-06, + "loss": 1.6773, + "step": 138 + }, + { + "epoch": 0.6105417276720352, + "grad_norm": 0.28918467659013913, + "learning_rate": 9.742682209735727e-06, + "loss": 1.7141, + "step": 139 + }, + { + "epoch": 0.6149341142020498, + "grad_norm": 0.25507889874035206, + "learning_rate": 9.736852983989405e-06, + "loss": 1.6335, + "step": 140 + }, + { + "epoch": 0.6193265007320644, + "grad_norm": 0.2712795343500641, + "learning_rate": 9.730960252267744e-06, + "loss": 1.6186, + "step": 141 + }, + { + "epoch": 0.623718887262079, + "grad_norm": 0.29326296437100124, + "learning_rate": 9.725004093573343e-06, + "loss": 1.6039, + "step": 142 + }, + { + "epoch": 0.6281112737920937, + "grad_norm": 0.27412689936526563, + "learning_rate": 9.718984587759148e-06, + "loss": 1.594, + "step": 143 + }, + { + "epoch": 0.6325036603221084, + "grad_norm": 0.28908560688778157, + "learning_rate": 9.712901815527387e-06, + "loss": 1.6458, + "step": 144 + }, + { + "epoch": 0.636896046852123, + "grad_norm": 0.26538330998431864, + "learning_rate": 9.706755858428487e-06, + "loss": 1.6546, + "step": 145 + }, + { + "epoch": 0.6412884333821376, + "grad_norm": 0.26498403141947585, + "learning_rate": 9.70054679885998e-06, + "loss": 1.6224, + "step": 146 + }, + { + "epoch": 0.6456808199121523, + "grad_norm": 0.25028323667878993, + "learning_rate": 9.6942747200654e-06, + "loss": 1.6554, + "step": 147 + }, + { + "epoch": 0.6500732064421669, + "grad_norm": 0.3077620507501807, + "learning_rate": 9.687939706133168e-06, + "loss": 1.6751, + "step": 148 + }, + { + "epoch": 0.6544655929721815, + "grad_norm": 0.2738251492188122, + "learning_rate": 9.68154184199546e-06, + "loss": 1.6139, + "step": 149 + }, + { + "epoch": 0.6588579795021962, + "grad_norm": 0.300766059001253, + "learning_rate": 9.675081213427076e-06, + "loss": 1.6783, + "step": 150 + }, + { + "epoch": 0.6632503660322109, + "grad_norm": 0.2671963968809055, + "learning_rate": 9.668557907044278e-06, + "loss": 1.7028, + "step": 151 + }, + { + "epoch": 0.6676427525622255, + "grad_norm": 0.27596191143138304, + "learning_rate": 9.661972010303641e-06, + "loss": 1.6279, + "step": 152 + }, + { + "epoch": 0.6720351390922401, + "grad_norm": 0.2719766311854951, + "learning_rate": 9.655323611500876e-06, + "loss": 1.6634, + "step": 153 + }, + { + "epoch": 0.6764275256222547, + "grad_norm": 0.2697612116557389, + "learning_rate": 9.648612799769644e-06, + "loss": 1.617, + "step": 154 + }, + { + "epoch": 0.6808199121522694, + "grad_norm": 0.28137116472179047, + "learning_rate": 9.641839665080363e-06, + "loss": 1.6212, + "step": 155 + }, + { + "epoch": 0.6852122986822841, + "grad_norm": 0.2599526010890989, + "learning_rate": 9.635004298239004e-06, + "loss": 1.6542, + "step": 156 + }, + { + "epoch": 0.6896046852122987, + "grad_norm": 0.662388825119619, + "learning_rate": 9.628106790885866e-06, + "loss": 1.633, + "step": 157 + }, + { + "epoch": 0.6939970717423133, + "grad_norm": 0.2913416852335614, + "learning_rate": 9.621147235494357e-06, + "loss": 1.6904, + "step": 158 + }, + { + "epoch": 0.698389458272328, + "grad_norm": 0.26581120890113497, + "learning_rate": 9.614125725369748e-06, + "loss": 1.6583, + "step": 159 + }, + { + "epoch": 0.7027818448023426, + "grad_norm": 0.26696083510238133, + "learning_rate": 9.607042354647924e-06, + "loss": 1.6354, + "step": 160 + }, + { + "epoch": 0.7071742313323572, + "grad_norm": 0.2913753113559095, + "learning_rate": 9.599897218294122e-06, + "loss": 1.5914, + "step": 161 + }, + { + "epoch": 0.7115666178623719, + "grad_norm": 0.2794317973029827, + "learning_rate": 9.59269041210166e-06, + "loss": 1.5732, + "step": 162 + }, + { + "epoch": 0.7159590043923866, + "grad_norm": 0.27364398897711945, + "learning_rate": 9.585422032690644e-06, + "loss": 1.6796, + "step": 163 + }, + { + "epoch": 0.7203513909224012, + "grad_norm": 0.2924137561125512, + "learning_rate": 9.578092177506683e-06, + "loss": 1.6921, + "step": 164 + }, + { + "epoch": 0.7247437774524158, + "grad_norm": 0.27864643658701665, + "learning_rate": 9.570700944819584e-06, + "loss": 1.6442, + "step": 165 + }, + { + "epoch": 0.7291361639824304, + "grad_norm": 0.3172135018133218, + "learning_rate": 9.56324843372202e-06, + "loss": 1.7003, + "step": 166 + }, + { + "epoch": 0.7335285505124451, + "grad_norm": 0.33119379016577744, + "learning_rate": 9.55573474412821e-06, + "loss": 1.6277, + "step": 167 + }, + { + "epoch": 0.7379209370424598, + "grad_norm": 0.24315818847358012, + "learning_rate": 9.548159976772593e-06, + "loss": 1.621, + "step": 168 + }, + { + "epoch": 0.7423133235724744, + "grad_norm": 0.28066914391093295, + "learning_rate": 9.540524233208449e-06, + "loss": 1.6169, + "step": 169 + }, + { + "epoch": 0.746705710102489, + "grad_norm": 0.3096372829142638, + "learning_rate": 9.532827615806561e-06, + "loss": 1.6425, + "step": 170 + }, + { + "epoch": 0.7510980966325037, + "grad_norm": 0.3050362838238551, + "learning_rate": 9.525070227753835e-06, + "loss": 1.5835, + "step": 171 + }, + { + "epoch": 0.7554904831625183, + "grad_norm": 0.27558198897100084, + "learning_rate": 9.517252173051912e-06, + "loss": 1.6866, + "step": 172 + }, + { + "epoch": 0.7598828696925329, + "grad_norm": 0.3415215102331636, + "learning_rate": 9.509373556515781e-06, + "loss": 1.6954, + "step": 173 + }, + { + "epoch": 0.7642752562225475, + "grad_norm": 0.2833145266859935, + "learning_rate": 9.501434483772371e-06, + "loss": 1.5642, + "step": 174 + }, + { + "epoch": 0.7686676427525623, + "grad_norm": 0.261080979929468, + "learning_rate": 9.49343506125913e-06, + "loss": 1.6481, + "step": 175 + }, + { + "epoch": 0.7730600292825769, + "grad_norm": 0.27528584249968596, + "learning_rate": 9.48537539622261e-06, + "loss": 1.6356, + "step": 176 + }, + { + "epoch": 0.7774524158125915, + "grad_norm": 0.28493642680744213, + "learning_rate": 9.477255596717012e-06, + "loss": 1.617, + "step": 177 + }, + { + "epoch": 0.7818448023426061, + "grad_norm": 0.250792323656803, + "learning_rate": 9.469075771602756e-06, + "loss": 1.7373, + "step": 178 + }, + { + "epoch": 0.7862371888726208, + "grad_norm": 0.2815707976300464, + "learning_rate": 9.460836030545007e-06, + "loss": 1.7134, + "step": 179 + }, + { + "epoch": 0.7906295754026355, + "grad_norm": 0.33965220402585167, + "learning_rate": 9.452536484012212e-06, + "loss": 1.66, + "step": 180 + }, + { + "epoch": 0.7950219619326501, + "grad_norm": 0.337075115730111, + "learning_rate": 9.444177243274619e-06, + "loss": 1.7237, + "step": 181 + }, + { + "epoch": 0.7994143484626647, + "grad_norm": 0.2652773355794464, + "learning_rate": 9.43575842040278e-06, + "loss": 1.7238, + "step": 182 + }, + { + "epoch": 0.8038067349926794, + "grad_norm": 0.29569868187991866, + "learning_rate": 9.427280128266049e-06, + "loss": 1.6484, + "step": 183 + }, + { + "epoch": 0.808199121522694, + "grad_norm": 0.2647366968721245, + "learning_rate": 9.418742480531086e-06, + "loss": 1.6557, + "step": 184 + }, + { + "epoch": 0.8125915080527086, + "grad_norm": 0.2624692157687208, + "learning_rate": 9.410145591660301e-06, + "loss": 1.6501, + "step": 185 + }, + { + "epoch": 0.8169838945827232, + "grad_norm": 0.2604406630602592, + "learning_rate": 9.40148957691035e-06, + "loss": 1.6205, + "step": 186 + }, + { + "epoch": 0.821376281112738, + "grad_norm": 0.3809448031526879, + "learning_rate": 9.392774552330567e-06, + "loss": 1.6014, + "step": 187 + }, + { + "epoch": 0.8257686676427526, + "grad_norm": 0.285104899301775, + "learning_rate": 9.384000634761429e-06, + "loss": 1.7119, + "step": 188 + }, + { + "epoch": 0.8301610541727672, + "grad_norm": 0.3603217046093286, + "learning_rate": 9.375167941832974e-06, + "loss": 1.6129, + "step": 189 + }, + { + "epoch": 0.8345534407027818, + "grad_norm": 0.26192371812461096, + "learning_rate": 9.366276591963222e-06, + "loss": 1.6648, + "step": 190 + }, + { + "epoch": 0.8389458272327965, + "grad_norm": 0.27224376529649075, + "learning_rate": 9.357326704356602e-06, + "loss": 1.7368, + "step": 191 + }, + { + "epoch": 0.8433382137628112, + "grad_norm": 0.267560205737413, + "learning_rate": 9.348318399002347e-06, + "loss": 1.6456, + "step": 192 + }, + { + "epoch": 0.8477306002928258, + "grad_norm": 0.38772390686795116, + "learning_rate": 9.339251796672878e-06, + "loss": 1.6765, + "step": 193 + }, + { + "epoch": 0.8521229868228404, + "grad_norm": 0.28624234868841525, + "learning_rate": 9.330127018922195e-06, + "loss": 1.6397, + "step": 194 + }, + { + "epoch": 0.8565153733528551, + "grad_norm": 0.3517826456437289, + "learning_rate": 9.320944188084241e-06, + "loss": 1.6504, + "step": 195 + }, + { + "epoch": 0.8609077598828697, + "grad_norm": 0.3054896957644684, + "learning_rate": 9.31170342727127e-06, + "loss": 1.6581, + "step": 196 + }, + { + "epoch": 0.8653001464128843, + "grad_norm": 0.47217080493290625, + "learning_rate": 9.302404860372185e-06, + "loss": 1.6614, + "step": 197 + }, + { + "epoch": 0.8696925329428989, + "grad_norm": 0.3744290659075852, + "learning_rate": 9.293048612050883e-06, + "loss": 1.6181, + "step": 198 + }, + { + "epoch": 0.8740849194729137, + "grad_norm": 0.2928672407192581, + "learning_rate": 9.283634807744586e-06, + "loss": 1.5801, + "step": 199 + }, + { + "epoch": 0.8784773060029283, + "grad_norm": 0.5500989172351091, + "learning_rate": 9.274163573662158e-06, + "loss": 1.6213, + "step": 200 + }, + { + "epoch": 0.8828696925329429, + "grad_norm": 0.27844961793523915, + "learning_rate": 9.264635036782406e-06, + "loss": 1.6442, + "step": 201 + }, + { + "epoch": 0.8872620790629575, + "grad_norm": 0.2764381586339949, + "learning_rate": 9.255049324852388e-06, + "loss": 1.6403, + "step": 202 + }, + { + "epoch": 0.8916544655929722, + "grad_norm": 0.28692831842946026, + "learning_rate": 9.245406566385698e-06, + "loss": 1.6204, + "step": 203 + }, + { + "epoch": 0.8960468521229868, + "grad_norm": 0.2854958274623322, + "learning_rate": 9.235706890660735e-06, + "loss": 1.6777, + "step": 204 + }, + { + "epoch": 0.9004392386530015, + "grad_norm": 0.2884758073862357, + "learning_rate": 9.225950427718974e-06, + "loss": 1.6414, + "step": 205 + }, + { + "epoch": 0.9048316251830161, + "grad_norm": 0.2774591809427237, + "learning_rate": 9.216137308363235e-06, + "loss": 1.667, + "step": 206 + }, + { + "epoch": 0.9092240117130308, + "grad_norm": 0.26327408771476135, + "learning_rate": 9.206267664155906e-06, + "loss": 1.6472, + "step": 207 + }, + { + "epoch": 0.9136163982430454, + "grad_norm": 0.3163508666559799, + "learning_rate": 9.1963416274172e-06, + "loss": 1.6898, + "step": 208 + }, + { + "epoch": 0.91800878477306, + "grad_norm": 0.2691312569989342, + "learning_rate": 9.18635933122337e-06, + "loss": 1.6788, + "step": 209 + }, + { + "epoch": 0.9224011713030746, + "grad_norm": 0.329963898141559, + "learning_rate": 9.176320909404925e-06, + "loss": 1.7077, + "step": 210 + }, + { + "epoch": 0.9267935578330894, + "grad_norm": 0.3497378648580076, + "learning_rate": 9.16622649654484e-06, + "loss": 1.7203, + "step": 211 + }, + { + "epoch": 0.931185944363104, + "grad_norm": 0.2750435891659104, + "learning_rate": 9.156076227976752e-06, + "loss": 1.6372, + "step": 212 + }, + { + "epoch": 0.9355783308931186, + "grad_norm": 0.29344217135995226, + "learning_rate": 9.145870239783143e-06, + "loss": 1.6294, + "step": 213 + }, + { + "epoch": 0.9399707174231332, + "grad_norm": 0.2713649627402965, + "learning_rate": 9.135608668793511e-06, + "loss": 1.6391, + "step": 214 + }, + { + "epoch": 0.9443631039531479, + "grad_norm": 0.2995344678229402, + "learning_rate": 9.125291652582549e-06, + "loss": 1.6102, + "step": 215 + }, + { + "epoch": 0.9487554904831625, + "grad_norm": 0.32763546183738235, + "learning_rate": 9.114919329468283e-06, + "loss": 1.6823, + "step": 216 + }, + { + "epoch": 0.9531478770131772, + "grad_norm": 0.29620265372375704, + "learning_rate": 9.104491838510235e-06, + "loss": 1.6905, + "step": 217 + }, + { + "epoch": 0.9575402635431918, + "grad_norm": 0.43487598587006066, + "learning_rate": 9.094009319507547e-06, + "loss": 1.6611, + "step": 218 + }, + { + "epoch": 0.9619326500732065, + "grad_norm": 0.32989859860270077, + "learning_rate": 9.08347191299711e-06, + "loss": 1.634, + "step": 219 + }, + { + "epoch": 0.9663250366032211, + "grad_norm": 0.26622257061624277, + "learning_rate": 9.07287976025168e-06, + "loss": 1.6075, + "step": 220 + }, + { + "epoch": 0.9707174231332357, + "grad_norm": 0.28626358026286186, + "learning_rate": 9.062233003277983e-06, + "loss": 1.6045, + "step": 221 + }, + { + "epoch": 0.9751098096632503, + "grad_norm": 0.5424158794876999, + "learning_rate": 9.051531784814817e-06, + "loss": 1.5825, + "step": 222 + }, + { + "epoch": 0.9795021961932651, + "grad_norm": 0.2771987597060821, + "learning_rate": 9.040776248331128e-06, + "loss": 1.6083, + "step": 223 + }, + { + "epoch": 0.9838945827232797, + "grad_norm": 1.5311321947246777, + "learning_rate": 9.029966538024097e-06, + "loss": 1.5973, + "step": 224 + }, + { + "epoch": 0.9882869692532943, + "grad_norm": 0.3081246423484545, + "learning_rate": 9.019102798817196e-06, + "loss": 1.5915, + "step": 225 + }, + { + "epoch": 0.9926793557833089, + "grad_norm": 0.3066770476121554, + "learning_rate": 9.008185176358257e-06, + "loss": 1.5551, + "step": 226 + }, + { + "epoch": 0.9970717423133236, + "grad_norm": 0.29888292004137484, + "learning_rate": 8.997213817017508e-06, + "loss": 1.7461, + "step": 227 + }, + { + "epoch": 1.0043923865300146, + "grad_norm": 0.5938110979809961, + "learning_rate": 8.986188867885617e-06, + "loss": 3.2509, + "step": 228 + }, + { + "epoch": 1.0087847730600292, + "grad_norm": 0.2899326916339286, + "learning_rate": 8.975110476771724e-06, + "loss": 1.5874, + "step": 229 + }, + { + "epoch": 1.0131771595900438, + "grad_norm": 0.2777095345419943, + "learning_rate": 8.96397879220145e-06, + "loss": 1.6706, + "step": 230 + }, + { + "epoch": 1.0175695461200585, + "grad_norm": 0.26363195429841585, + "learning_rate": 8.952793963414908e-06, + "loss": 1.6317, + "step": 231 + }, + { + "epoch": 1.0219619326500733, + "grad_norm": 0.2688971491502818, + "learning_rate": 8.941556140364707e-06, + "loss": 1.654, + "step": 232 + }, + { + "epoch": 1.026354319180088, + "grad_norm": 0.28487309366468816, + "learning_rate": 8.930265473713939e-06, + "loss": 1.6133, + "step": 233 + }, + { + "epoch": 1.0307467057101025, + "grad_norm": 0.28141744313455036, + "learning_rate": 8.918922114834156e-06, + "loss": 1.589, + "step": 234 + }, + { + "epoch": 1.0351390922401171, + "grad_norm": 0.40308880749849646, + "learning_rate": 8.907526215803351e-06, + "loss": 1.6499, + "step": 235 + }, + { + "epoch": 1.0395314787701317, + "grad_norm": 0.2834055854316979, + "learning_rate": 8.896077929403901e-06, + "loss": 1.658, + "step": 236 + }, + { + "epoch": 1.0439238653001464, + "grad_norm": 0.2887035123274668, + "learning_rate": 8.884577409120535e-06, + "loss": 1.6196, + "step": 237 + }, + { + "epoch": 1.048316251830161, + "grad_norm": 0.2985213212589398, + "learning_rate": 8.873024809138272e-06, + "loss": 1.677, + "step": 238 + }, + { + "epoch": 1.0527086383601758, + "grad_norm": 0.2863532205939482, + "learning_rate": 8.861420284340352e-06, + "loss": 1.6682, + "step": 239 + }, + { + "epoch": 1.0571010248901904, + "grad_norm": 0.29235163398244823, + "learning_rate": 8.849763990306153e-06, + "loss": 1.6009, + "step": 240 + }, + { + "epoch": 1.061493411420205, + "grad_norm": 0.2899336107020558, + "learning_rate": 8.838056083309118e-06, + "loss": 1.6319, + "step": 241 + }, + { + "epoch": 1.0658857979502196, + "grad_norm": 0.3186837918366928, + "learning_rate": 8.826296720314658e-06, + "loss": 1.5875, + "step": 242 + }, + { + "epoch": 1.0702781844802343, + "grad_norm": 0.33938164798671433, + "learning_rate": 8.814486058978035e-06, + "loss": 1.6059, + "step": 243 + }, + { + "epoch": 1.0746705710102489, + "grad_norm": 0.2991115627129074, + "learning_rate": 8.802624257642262e-06, + "loss": 1.6599, + "step": 244 + }, + { + "epoch": 1.0790629575402635, + "grad_norm": 0.28608565865487656, + "learning_rate": 8.790711475335971e-06, + "loss": 1.6364, + "step": 245 + }, + { + "epoch": 1.083455344070278, + "grad_norm": 0.28359019806003394, + "learning_rate": 8.778747871771293e-06, + "loss": 1.6584, + "step": 246 + }, + { + "epoch": 1.0878477306002927, + "grad_norm": 0.34292617248780644, + "learning_rate": 8.766733607341697e-06, + "loss": 1.6613, + "step": 247 + }, + { + "epoch": 1.0922401171303076, + "grad_norm": 0.3469366526291968, + "learning_rate": 8.754668843119865e-06, + "loss": 1.6542, + "step": 248 + }, + { + "epoch": 1.0966325036603222, + "grad_norm": 0.2745752678931537, + "learning_rate": 8.742553740855507e-06, + "loss": 1.6723, + "step": 249 + }, + { + "epoch": 1.1010248901903368, + "grad_norm": 0.3863733971251328, + "learning_rate": 8.730388462973209e-06, + "loss": 1.6268, + "step": 250 + }, + { + "epoch": 1.1054172767203514, + "grad_norm": 0.3196278576328997, + "learning_rate": 8.718173172570254e-06, + "loss": 1.6349, + "step": 251 + }, + { + "epoch": 1.109809663250366, + "grad_norm": 0.27017487934331, + "learning_rate": 8.705908033414426e-06, + "loss": 1.606, + "step": 252 + }, + { + "epoch": 1.1142020497803806, + "grad_norm": 0.6576768607622604, + "learning_rate": 8.693593209941826e-06, + "loss": 1.5936, + "step": 253 + }, + { + "epoch": 1.1185944363103952, + "grad_norm": 0.30260640047372234, + "learning_rate": 8.681228867254657e-06, + "loss": 1.5948, + "step": 254 + }, + { + "epoch": 1.12298682284041, + "grad_norm": 0.3102239482584757, + "learning_rate": 8.66881517111902e-06, + "loss": 1.6928, + "step": 255 + }, + { + "epoch": 1.1273792093704247, + "grad_norm": 0.2818931187575453, + "learning_rate": 8.656352287962687e-06, + "loss": 1.687, + "step": 256 + }, + { + "epoch": 1.1317715959004393, + "grad_norm": 0.30007928577375903, + "learning_rate": 8.643840384872865e-06, + "loss": 1.6628, + "step": 257 + }, + { + "epoch": 1.136163982430454, + "grad_norm": 0.3249258384227434, + "learning_rate": 8.631279629593968e-06, + "loss": 1.6123, + "step": 258 + }, + { + "epoch": 1.1405563689604685, + "grad_norm": 0.2871446068255736, + "learning_rate": 8.61867019052535e-06, + "loss": 1.6608, + "step": 259 + }, + { + "epoch": 1.1449487554904831, + "grad_norm": 0.3001183609451796, + "learning_rate": 8.606012236719073e-06, + "loss": 1.6405, + "step": 260 + }, + { + "epoch": 1.1493411420204978, + "grad_norm": 0.271155672303218, + "learning_rate": 8.593305937877614e-06, + "loss": 1.5819, + "step": 261 + }, + { + "epoch": 1.1537335285505124, + "grad_norm": 0.30999460151747915, + "learning_rate": 8.580551464351603e-06, + "loss": 1.6378, + "step": 262 + }, + { + "epoch": 1.158125915080527, + "grad_norm": 0.2979330701414515, + "learning_rate": 8.567748987137544e-06, + "loss": 1.7342, + "step": 263 + }, + { + "epoch": 1.1625183016105418, + "grad_norm": 0.29037637226651547, + "learning_rate": 8.554898677875509e-06, + "loss": 1.663, + "step": 264 + }, + { + "epoch": 1.1669106881405564, + "grad_norm": 0.29574080692093535, + "learning_rate": 8.542000708846851e-06, + "loss": 1.6937, + "step": 265 + }, + { + "epoch": 1.171303074670571, + "grad_norm": 0.2867463319690563, + "learning_rate": 8.52905525297188e-06, + "loss": 1.5812, + "step": 266 + }, + { + "epoch": 1.1756954612005857, + "grad_norm": 0.3075848284440638, + "learning_rate": 8.516062483807556e-06, + "loss": 1.6421, + "step": 267 + }, + { + "epoch": 1.1800878477306003, + "grad_norm": 0.2745442718449274, + "learning_rate": 8.503022575545159e-06, + "loss": 1.6592, + "step": 268 + }, + { + "epoch": 1.1844802342606149, + "grad_norm": 0.4008583979349149, + "learning_rate": 8.48993570300795e-06, + "loss": 1.6052, + "step": 269 + }, + { + "epoch": 1.1888726207906295, + "grad_norm": 0.5183284694523707, + "learning_rate": 8.476802041648832e-06, + "loss": 1.7352, + "step": 270 + }, + { + "epoch": 1.1932650073206443, + "grad_norm": 0.30560643793660064, + "learning_rate": 8.463621767547998e-06, + "loss": 1.5592, + "step": 271 + }, + { + "epoch": 1.197657393850659, + "grad_norm": 0.5272799941494408, + "learning_rate": 8.450395057410561e-06, + "loss": 1.5471, + "step": 272 + }, + { + "epoch": 1.2020497803806736, + "grad_norm": 0.4281815470371588, + "learning_rate": 8.437122088564197e-06, + "loss": 1.6445, + "step": 273 + }, + { + "epoch": 1.2064421669106882, + "grad_norm": 0.27224728594527714, + "learning_rate": 8.423803038956763e-06, + "loss": 1.6629, + "step": 274 + }, + { + "epoch": 1.2108345534407028, + "grad_norm": 0.3269535080116159, + "learning_rate": 8.41043808715391e-06, + "loss": 1.5722, + "step": 275 + }, + { + "epoch": 1.2152269399707174, + "grad_norm": 0.3054371244574999, + "learning_rate": 8.39702741233669e-06, + "loss": 1.6173, + "step": 276 + }, + { + "epoch": 1.219619326500732, + "grad_norm": 0.36010851057708126, + "learning_rate": 8.383571194299154e-06, + "loss": 1.596, + "step": 277 + }, + { + "epoch": 1.2240117130307466, + "grad_norm": 0.3803533141127172, + "learning_rate": 8.37006961344594e-06, + "loss": 1.6229, + "step": 278 + }, + { + "epoch": 1.2284040995607612, + "grad_norm": 0.28612951080269505, + "learning_rate": 8.356522850789852e-06, + "loss": 1.5549, + "step": 279 + }, + { + "epoch": 1.232796486090776, + "grad_norm": 0.2855964904759227, + "learning_rate": 8.342931087949446e-06, + "loss": 1.6512, + "step": 280 + }, + { + "epoch": 1.2371888726207907, + "grad_norm": 0.3054377930088084, + "learning_rate": 8.32929450714658e-06, + "loss": 1.6274, + "step": 281 + }, + { + "epoch": 1.2415812591508053, + "grad_norm": 0.29343763848165016, + "learning_rate": 8.315613291203977e-06, + "loss": 1.6372, + "step": 282 + }, + { + "epoch": 1.24597364568082, + "grad_norm": 0.40870431747557984, + "learning_rate": 8.301887623542774e-06, + "loss": 1.6667, + "step": 283 + }, + { + "epoch": 1.2503660322108345, + "grad_norm": 0.30881819618442474, + "learning_rate": 8.288117688180064e-06, + "loss": 1.5547, + "step": 284 + }, + { + "epoch": 1.2547584187408491, + "grad_norm": 0.28765815508847925, + "learning_rate": 8.274303669726427e-06, + "loss": 1.6097, + "step": 285 + }, + { + "epoch": 1.2591508052708638, + "grad_norm": 0.33007103881402516, + "learning_rate": 8.260445753383453e-06, + "loss": 1.7016, + "step": 286 + }, + { + "epoch": 1.2635431918008786, + "grad_norm": 0.3628853649167158, + "learning_rate": 8.246544124941266e-06, + "loss": 1.6086, + "step": 287 + }, + { + "epoch": 1.267935578330893, + "grad_norm": 0.40134140387051037, + "learning_rate": 8.232598970776027e-06, + "loss": 1.6789, + "step": 288 + }, + { + "epoch": 1.2723279648609078, + "grad_norm": 0.2920563442488122, + "learning_rate": 8.218610477847435e-06, + "loss": 1.5915, + "step": 289 + }, + { + "epoch": 1.2767203513909224, + "grad_norm": 0.28530665366941144, + "learning_rate": 8.204578833696228e-06, + "loss": 1.5949, + "step": 290 + }, + { + "epoch": 1.281112737920937, + "grad_norm": 0.3155789127902478, + "learning_rate": 8.190504226441654e-06, + "loss": 1.639, + "step": 291 + }, + { + "epoch": 1.2855051244509517, + "grad_norm": 0.3199350667177758, + "learning_rate": 8.176386844778969e-06, + "loss": 1.6818, + "step": 292 + }, + { + "epoch": 1.2898975109809663, + "grad_norm": 0.34596043922632713, + "learning_rate": 8.162226877976886e-06, + "loss": 1.6604, + "step": 293 + }, + { + "epoch": 1.294289897510981, + "grad_norm": 0.32781624288061406, + "learning_rate": 8.148024515875057e-06, + "loss": 1.5971, + "step": 294 + }, + { + "epoch": 1.2986822840409955, + "grad_norm": 0.5927243317332561, + "learning_rate": 8.133779948881514e-06, + "loss": 1.6574, + "step": 295 + }, + { + "epoch": 1.3030746705710103, + "grad_norm": 0.296769298222996, + "learning_rate": 8.11949336797012e-06, + "loss": 1.5505, + "step": 296 + }, + { + "epoch": 1.307467057101025, + "grad_norm": 0.28849112804215943, + "learning_rate": 8.105164964678009e-06, + "loss": 1.6729, + "step": 297 + }, + { + "epoch": 1.3118594436310396, + "grad_norm": 0.3286168384006033, + "learning_rate": 8.090794931103026e-06, + "loss": 1.5949, + "step": 298 + }, + { + "epoch": 1.3162518301610542, + "grad_norm": 0.31185844876531665, + "learning_rate": 8.076383459901138e-06, + "loss": 1.6918, + "step": 299 + }, + { + "epoch": 1.3206442166910688, + "grad_norm": 0.2789921585419944, + "learning_rate": 8.061930744283855e-06, + "loss": 1.6437, + "step": 300 + }, + { + "epoch": 1.3250366032210834, + "grad_norm": 0.30165508316706874, + "learning_rate": 8.04743697801565e-06, + "loss": 1.6555, + "step": 301 + }, + { + "epoch": 1.329428989751098, + "grad_norm": 0.2989059936488396, + "learning_rate": 8.032902355411345e-06, + "loss": 1.5141, + "step": 302 + }, + { + "epoch": 1.3338213762811129, + "grad_norm": 0.29928394703560823, + "learning_rate": 8.018327071333521e-06, + "loss": 1.5859, + "step": 303 + }, + { + "epoch": 1.3382137628111272, + "grad_norm": 0.29795453570714703, + "learning_rate": 8.003711321189895e-06, + "loss": 1.5864, + "step": 304 + }, + { + "epoch": 1.342606149341142, + "grad_norm": 0.30765817351768504, + "learning_rate": 7.989055300930704e-06, + "loss": 1.5743, + "step": 305 + }, + { + "epoch": 1.3469985358711567, + "grad_norm": 0.43955554329120444, + "learning_rate": 7.974359207046081e-06, + "loss": 1.6423, + "step": 306 + }, + { + "epoch": 1.3513909224011713, + "grad_norm": 0.37865307469865994, + "learning_rate": 7.959623236563412e-06, + "loss": 1.6102, + "step": 307 + }, + { + "epoch": 1.355783308931186, + "grad_norm": 0.3074090057367007, + "learning_rate": 7.944847587044704e-06, + "loss": 1.6602, + "step": 308 + }, + { + "epoch": 1.3601756954612005, + "grad_norm": 0.3361817362879421, + "learning_rate": 7.930032456583931e-06, + "loss": 1.6704, + "step": 309 + }, + { + "epoch": 1.3645680819912152, + "grad_norm": 0.2960444587212274, + "learning_rate": 7.915178043804382e-06, + "loss": 1.5536, + "step": 310 + }, + { + "epoch": 1.3689604685212298, + "grad_norm": 0.31080078465435335, + "learning_rate": 7.900284547855992e-06, + "loss": 1.659, + "step": 311 + }, + { + "epoch": 1.3733528550512446, + "grad_norm": 0.29259208311732576, + "learning_rate": 7.885352168412677e-06, + "loss": 1.6324, + "step": 312 + }, + { + "epoch": 1.3777452415812592, + "grad_norm": 0.40704907971444493, + "learning_rate": 7.870381105669657e-06, + "loss": 1.6954, + "step": 313 + }, + { + "epoch": 1.3821376281112738, + "grad_norm": 0.2884678272243352, + "learning_rate": 7.85537156034077e-06, + "loss": 1.6911, + "step": 314 + }, + { + "epoch": 1.3865300146412884, + "grad_norm": 0.3002277679918279, + "learning_rate": 7.84032373365578e-06, + "loss": 1.6384, + "step": 315 + }, + { + "epoch": 1.390922401171303, + "grad_norm": 0.29747656531257255, + "learning_rate": 7.825237827357684e-06, + "loss": 1.5728, + "step": 316 + }, + { + "epoch": 1.3953147877013177, + "grad_norm": 0.2922330023166882, + "learning_rate": 7.810114043700002e-06, + "loss": 1.6876, + "step": 317 + }, + { + "epoch": 1.3997071742313323, + "grad_norm": 0.29216993729442, + "learning_rate": 7.794952585444068e-06, + "loss": 1.7112, + "step": 318 + }, + { + "epoch": 1.4040995607613471, + "grad_norm": 0.2841430060967501, + "learning_rate": 7.779753655856313e-06, + "loss": 1.6352, + "step": 319 + }, + { + "epoch": 1.4084919472913615, + "grad_norm": 0.29535174268674796, + "learning_rate": 7.764517458705536e-06, + "loss": 1.638, + "step": 320 + }, + { + "epoch": 1.4128843338213763, + "grad_norm": 0.3553267662730052, + "learning_rate": 7.749244198260175e-06, + "loss": 1.6262, + "step": 321 + }, + { + "epoch": 1.417276720351391, + "grad_norm": 0.2977673782348279, + "learning_rate": 7.733934079285569e-06, + "loss": 1.6115, + "step": 322 + }, + { + "epoch": 1.4216691068814056, + "grad_norm": 0.2822196746941408, + "learning_rate": 7.718587307041209e-06, + "loss": 1.6468, + "step": 323 + }, + { + "epoch": 1.4260614934114202, + "grad_norm": 0.2837671497162217, + "learning_rate": 7.703204087277989e-06, + "loss": 1.6474, + "step": 324 + }, + { + "epoch": 1.4304538799414348, + "grad_norm": 0.31664451334685184, + "learning_rate": 7.687784626235448e-06, + "loss": 1.5919, + "step": 325 + }, + { + "epoch": 1.4348462664714494, + "grad_norm": 0.33195017711571867, + "learning_rate": 7.672329130639007e-06, + "loss": 1.6941, + "step": 326 + }, + { + "epoch": 1.439238653001464, + "grad_norm": 0.2995108911659512, + "learning_rate": 7.656837807697187e-06, + "loss": 1.6393, + "step": 327 + }, + { + "epoch": 1.4436310395314789, + "grad_norm": 0.2949122695416032, + "learning_rate": 7.641310865098845e-06, + "loss": 1.5135, + "step": 328 + }, + { + "epoch": 1.4480234260614935, + "grad_norm": 0.31946384200509315, + "learning_rate": 7.625748511010382e-06, + "loss": 1.5784, + "step": 329 + }, + { + "epoch": 1.452415812591508, + "grad_norm": 0.29208814812765094, + "learning_rate": 7.610150954072953e-06, + "loss": 1.6794, + "step": 330 + }, + { + "epoch": 1.4568081991215227, + "grad_norm": 0.34475174012170284, + "learning_rate": 7.594518403399666e-06, + "loss": 1.5773, + "step": 331 + }, + { + "epoch": 1.4612005856515373, + "grad_norm": 0.2876304119955936, + "learning_rate": 7.578851068572788e-06, + "loss": 1.5613, + "step": 332 + }, + { + "epoch": 1.465592972181552, + "grad_norm": 0.31542092184200016, + "learning_rate": 7.563149159640929e-06, + "loss": 1.6086, + "step": 333 + }, + { + "epoch": 1.4699853587115665, + "grad_norm": 0.2964161628058562, + "learning_rate": 7.547412887116224e-06, + "loss": 1.5963, + "step": 334 + }, + { + "epoch": 1.4743777452415814, + "grad_norm": 0.32832262869958717, + "learning_rate": 7.531642461971515e-06, + "loss": 1.6054, + "step": 335 + }, + { + "epoch": 1.4787701317715958, + "grad_norm": 0.3209241382730098, + "learning_rate": 7.51583809563752e-06, + "loss": 1.6748, + "step": 336 + }, + { + "epoch": 1.4831625183016106, + "grad_norm": 0.30060262616835987, + "learning_rate": 7.500000000000001e-06, + "loss": 1.6081, + "step": 337 + }, + { + "epoch": 1.4875549048316252, + "grad_norm": 0.3500388762321464, + "learning_rate": 7.4841283873969194e-06, + "loss": 1.5906, + "step": 338 + }, + { + "epoch": 1.4919472913616398, + "grad_norm": 0.28308587203259633, + "learning_rate": 7.468223470615593e-06, + "loss": 1.5879, + "step": 339 + }, + { + "epoch": 1.4963396778916545, + "grad_norm": 0.2901927981331445, + "learning_rate": 7.452285462889841e-06, + "loss": 1.6242, + "step": 340 + }, + { + "epoch": 1.500732064421669, + "grad_norm": 0.3314307283598877, + "learning_rate": 7.436314577897126e-06, + "loss": 1.646, + "step": 341 + }, + { + "epoch": 1.505124450951684, + "grad_norm": 0.3264131109049365, + "learning_rate": 7.420311029755688e-06, + "loss": 1.5855, + "step": 342 + }, + { + "epoch": 1.5095168374816983, + "grad_norm": 0.3519561244527488, + "learning_rate": 7.404275033021676e-06, + "loss": 1.6415, + "step": 343 + }, + { + "epoch": 1.5139092240117131, + "grad_norm": 0.3231864119230437, + "learning_rate": 7.388206802686272e-06, + "loss": 1.6038, + "step": 344 + }, + { + "epoch": 1.5183016105417275, + "grad_norm": 0.4648117736859699, + "learning_rate": 7.372106554172802e-06, + "loss": 1.6327, + "step": 345 + }, + { + "epoch": 1.5226939970717424, + "grad_norm": 0.30819599956019383, + "learning_rate": 7.355974503333859e-06, + "loss": 1.7179, + "step": 346 + }, + { + "epoch": 1.527086383601757, + "grad_norm": 0.3425241683440226, + "learning_rate": 7.339810866448398e-06, + "loss": 1.6486, + "step": 347 + }, + { + "epoch": 1.5314787701317716, + "grad_norm": 0.3229436387251198, + "learning_rate": 7.323615860218844e-06, + "loss": 1.5028, + "step": 348 + }, + { + "epoch": 1.5358711566617862, + "grad_norm": 0.32420322558759, + "learning_rate": 7.307389701768183e-06, + "loss": 1.517, + "step": 349 + }, + { + "epoch": 1.5402635431918008, + "grad_norm": 0.33308925785501975, + "learning_rate": 7.291132608637053e-06, + "loss": 1.6278, + "step": 350 + }, + { + "epoch": 1.5446559297218156, + "grad_norm": 0.33961176568859514, + "learning_rate": 7.274844798780826e-06, + "loss": 1.6651, + "step": 351 + }, + { + "epoch": 1.54904831625183, + "grad_norm": 0.32173930273995155, + "learning_rate": 7.258526490566687e-06, + "loss": 1.491, + "step": 352 + }, + { + "epoch": 1.5534407027818449, + "grad_norm": 0.3093139646127846, + "learning_rate": 7.242177902770706e-06, + "loss": 1.5796, + "step": 353 + }, + { + "epoch": 1.5578330893118595, + "grad_norm": 0.32919345439390313, + "learning_rate": 7.2257992545749045e-06, + "loss": 1.5865, + "step": 354 + }, + { + "epoch": 1.562225475841874, + "grad_norm": 0.2835010953118983, + "learning_rate": 7.209390765564318e-06, + "loss": 1.6417, + "step": 355 + }, + { + "epoch": 1.5666178623718887, + "grad_norm": 1.7445414238062138, + "learning_rate": 7.192952655724049e-06, + "loss": 1.6597, + "step": 356 + }, + { + "epoch": 1.5710102489019033, + "grad_norm": 0.5860196285629298, + "learning_rate": 7.176485145436325e-06, + "loss": 1.5387, + "step": 357 + }, + { + "epoch": 1.5754026354319182, + "grad_norm": 0.3537384385833807, + "learning_rate": 7.159988455477534e-06, + "loss": 1.6013, + "step": 358 + }, + { + "epoch": 1.5797950219619326, + "grad_norm": 0.3286829503374976, + "learning_rate": 7.143462807015271e-06, + "loss": 1.6264, + "step": 359 + }, + { + "epoch": 1.5841874084919474, + "grad_norm": 0.3197218822335203, + "learning_rate": 7.1269084216053756e-06, + "loss": 1.5552, + "step": 360 + }, + { + "epoch": 1.5885797950219618, + "grad_norm": 0.31209150272027425, + "learning_rate": 7.11032552118895e-06, + "loss": 1.6108, + "step": 361 + }, + { + "epoch": 1.5929721815519766, + "grad_norm": 0.3227458575298131, + "learning_rate": 7.093714328089398e-06, + "loss": 1.649, + "step": 362 + }, + { + "epoch": 1.5973645680819912, + "grad_norm": 0.3637374362635078, + "learning_rate": 7.0770750650094335e-06, + "loss": 1.6739, + "step": 363 + }, + { + "epoch": 1.6017569546120058, + "grad_norm": 0.31236210751966154, + "learning_rate": 7.060407955028098e-06, + "loss": 1.5946, + "step": 364 + }, + { + "epoch": 1.6061493411420205, + "grad_norm": 0.35651724986710415, + "learning_rate": 7.0437132215977744e-06, + "loss": 1.6337, + "step": 365 + }, + { + "epoch": 1.610541727672035, + "grad_norm": 0.31143483852268733, + "learning_rate": 7.026991088541184e-06, + "loss": 1.67, + "step": 366 + }, + { + "epoch": 1.61493411420205, + "grad_norm": 0.33050222076540253, + "learning_rate": 7.01024178004839e-06, + "loss": 1.6204, + "step": 367 + }, + { + "epoch": 1.6193265007320643, + "grad_norm": 0.3284547022539121, + "learning_rate": 6.99346552067379e-06, + "loss": 1.573, + "step": 368 + }, + { + "epoch": 1.6237188872620791, + "grad_norm": 0.3603027916184282, + "learning_rate": 6.976662535333107e-06, + "loss": 1.6553, + "step": 369 + }, + { + "epoch": 1.6281112737920937, + "grad_norm": 0.8725979374632243, + "learning_rate": 6.959833049300376e-06, + "loss": 1.6279, + "step": 370 + }, + { + "epoch": 1.6325036603221084, + "grad_norm": 0.28597465971107255, + "learning_rate": 6.942977288204915e-06, + "loss": 1.5074, + "step": 371 + }, + { + "epoch": 1.636896046852123, + "grad_norm": 0.31388416897723714, + "learning_rate": 6.926095478028312e-06, + "loss": 1.6896, + "step": 372 + }, + { + "epoch": 1.6412884333821376, + "grad_norm": 0.3008107505835411, + "learning_rate": 6.909187845101387e-06, + "loss": 1.5688, + "step": 373 + }, + { + "epoch": 1.6456808199121524, + "grad_norm": 0.32252769956105753, + "learning_rate": 6.892254616101158e-06, + "loss": 1.7391, + "step": 374 + }, + { + "epoch": 1.6500732064421668, + "grad_norm": 0.3253332219188239, + "learning_rate": 6.87529601804781e-06, + "loss": 1.5479, + "step": 375 + }, + { + "epoch": 1.6544655929721817, + "grad_norm": 0.3354516401894849, + "learning_rate": 6.858312278301638e-06, + "loss": 1.5944, + "step": 376 + }, + { + "epoch": 1.658857979502196, + "grad_norm": 0.3101157670728667, + "learning_rate": 6.841303624560012e-06, + "loss": 1.5979, + "step": 377 + }, + { + "epoch": 1.6632503660322109, + "grad_norm": 0.3263157294243148, + "learning_rate": 6.8242702848543185e-06, + "loss": 1.6651, + "step": 378 + }, + { + "epoch": 1.6676427525622255, + "grad_norm": 0.2963170265612591, + "learning_rate": 6.807212487546897e-06, + "loss": 1.6414, + "step": 379 + }, + { + "epoch": 1.67203513909224, + "grad_norm": 0.3436329546570268, + "learning_rate": 6.790130461327993e-06, + "loss": 1.5237, + "step": 380 + }, + { + "epoch": 1.6764275256222547, + "grad_norm": 0.30951635226199037, + "learning_rate": 6.773024435212678e-06, + "loss": 1.5608, + "step": 381 + }, + { + "epoch": 1.6808199121522693, + "grad_norm": 0.3024875350743093, + "learning_rate": 6.755894638537791e-06, + "loss": 1.5595, + "step": 382 + }, + { + "epoch": 1.6852122986822842, + "grad_norm": 0.30964409063235376, + "learning_rate": 6.73874130095885e-06, + "loss": 1.6166, + "step": 383 + }, + { + "epoch": 1.6896046852122986, + "grad_norm": 0.31495863587091266, + "learning_rate": 6.721564652446987e-06, + "loss": 1.5279, + "step": 384 + }, + { + "epoch": 1.6939970717423134, + "grad_norm": 0.3115937718660612, + "learning_rate": 6.704364923285858e-06, + "loss": 1.5694, + "step": 385 + }, + { + "epoch": 1.698389458272328, + "grad_norm": 0.29768001900234425, + "learning_rate": 6.687142344068552e-06, + "loss": 1.5578, + "step": 386 + }, + { + "epoch": 1.7027818448023426, + "grad_norm": 0.32146567067323123, + "learning_rate": 6.669897145694507e-06, + "loss": 1.6368, + "step": 387 + }, + { + "epoch": 1.7071742313323572, + "grad_norm": 0.29522637791127904, + "learning_rate": 6.6526295593664136e-06, + "loss": 1.6121, + "step": 388 + }, + { + "epoch": 1.7115666178623719, + "grad_norm": 0.30160303491140916, + "learning_rate": 6.635339816587109e-06, + "loss": 1.6238, + "step": 389 + }, + { + "epoch": 1.7159590043923867, + "grad_norm": 0.4208009773656739, + "learning_rate": 6.618028149156479e-06, + "loss": 1.62, + "step": 390 + }, + { + "epoch": 1.720351390922401, + "grad_norm": 0.2881943555017053, + "learning_rate": 6.600694789168345e-06, + "loss": 1.6364, + "step": 391 + }, + { + "epoch": 1.724743777452416, + "grad_norm": 0.3077307710744344, + "learning_rate": 6.583339969007364e-06, + "loss": 1.5677, + "step": 392 + }, + { + "epoch": 1.7291361639824303, + "grad_norm": 0.35235096673751176, + "learning_rate": 6.565963921345896e-06, + "loss": 1.6163, + "step": 393 + }, + { + "epoch": 1.7335285505124451, + "grad_norm": 0.3186476335255873, + "learning_rate": 6.548566879140897e-06, + "loss": 1.6289, + "step": 394 + }, + { + "epoch": 1.7379209370424598, + "grad_norm": 0.31476614898776, + "learning_rate": 6.531149075630796e-06, + "loss": 1.5874, + "step": 395 + }, + { + "epoch": 1.7423133235724744, + "grad_norm": 0.3469536206500234, + "learning_rate": 6.513710744332361e-06, + "loss": 1.6621, + "step": 396 + }, + { + "epoch": 1.746705710102489, + "grad_norm": 0.30680438680917854, + "learning_rate": 6.49625211903757e-06, + "loss": 1.595, + "step": 397 + }, + { + "epoch": 1.7510980966325036, + "grad_norm": 0.34458603376189034, + "learning_rate": 6.478773433810478e-06, + "loss": 1.5886, + "step": 398 + }, + { + "epoch": 1.7554904831625184, + "grad_norm": 0.34083953962031827, + "learning_rate": 6.461274922984087e-06, + "loss": 1.5705, + "step": 399 + }, + { + "epoch": 1.7598828696925328, + "grad_norm": 0.3345685083562738, + "learning_rate": 6.4437568211571864e-06, + "loss": 1.6181, + "step": 400 + }, + { + "epoch": 1.7642752562225477, + "grad_norm": 0.28767837558253834, + "learning_rate": 6.426219363191224e-06, + "loss": 1.6273, + "step": 401 + }, + { + "epoch": 1.7686676427525623, + "grad_norm": 0.3309056934700509, + "learning_rate": 6.408662784207149e-06, + "loss": 1.5575, + "step": 402 + }, + { + "epoch": 1.7730600292825769, + "grad_norm": 0.33936552783693097, + "learning_rate": 6.391087319582264e-06, + "loss": 1.5653, + "step": 403 + }, + { + "epoch": 1.7774524158125915, + "grad_norm": 0.38728001848935395, + "learning_rate": 6.373493204947065e-06, + "loss": 1.5962, + "step": 404 + }, + { + "epoch": 1.7818448023426061, + "grad_norm": 0.33731817557741206, + "learning_rate": 6.355880676182086e-06, + "loss": 1.5267, + "step": 405 + }, + { + "epoch": 1.786237188872621, + "grad_norm": 0.3073610397325029, + "learning_rate": 6.3382499694147345e-06, + "loss": 1.5735, + "step": 406 + }, + { + "epoch": 1.7906295754026353, + "grad_norm": 0.5666929994713482, + "learning_rate": 6.3206013210161285e-06, + "loss": 1.6164, + "step": 407 + }, + { + "epoch": 1.7950219619326502, + "grad_norm": 0.3423706928777578, + "learning_rate": 6.302934967597922e-06, + "loss": 1.6694, + "step": 408 + }, + { + "epoch": 1.7994143484626646, + "grad_norm": 0.3100272828530065, + "learning_rate": 6.2852511460091406e-06, + "loss": 1.6322, + "step": 409 + }, + { + "epoch": 1.8038067349926794, + "grad_norm": 0.3710603928040935, + "learning_rate": 6.267550093333e-06, + "loss": 1.5405, + "step": 410 + }, + { + "epoch": 1.808199121522694, + "grad_norm": 0.3024234235421662, + "learning_rate": 6.249832046883729e-06, + "loss": 1.6389, + "step": 411 + }, + { + "epoch": 1.8125915080527086, + "grad_norm": 0.2923050985231585, + "learning_rate": 6.232097244203388e-06, + "loss": 1.563, + "step": 412 + }, + { + "epoch": 1.8169838945827232, + "grad_norm": 0.37742465142395615, + "learning_rate": 6.214345923058686e-06, + "loss": 1.5605, + "step": 413 + }, + { + "epoch": 1.8213762811127379, + "grad_norm": 0.2763099605085667, + "learning_rate": 6.1965783214377895e-06, + "loss": 1.5757, + "step": 414 + }, + { + "epoch": 1.8257686676427527, + "grad_norm": 0.29226253974509314, + "learning_rate": 6.178794677547138e-06, + "loss": 1.6213, + "step": 415 + }, + { + "epoch": 1.830161054172767, + "grad_norm": 0.3892056822790433, + "learning_rate": 6.16099522980824e-06, + "loss": 1.6471, + "step": 416 + }, + { + "epoch": 1.834553440702782, + "grad_norm": 0.3547953314773269, + "learning_rate": 6.143180216854488e-06, + "loss": 1.5933, + "step": 417 + }, + { + "epoch": 1.8389458272327965, + "grad_norm": 0.2876417607221798, + "learning_rate": 6.125349877527952e-06, + "loss": 1.6056, + "step": 418 + }, + { + "epoch": 1.8433382137628112, + "grad_norm": 0.29632892867662236, + "learning_rate": 6.1075044508761804e-06, + "loss": 1.5798, + "step": 419 + }, + { + "epoch": 1.8477306002928258, + "grad_norm": 0.3351398469787539, + "learning_rate": 6.0896441761489925e-06, + "loss": 1.5775, + "step": 420 + }, + { + "epoch": 1.8521229868228404, + "grad_norm": 0.32572738949136165, + "learning_rate": 6.071769292795274e-06, + "loss": 1.5065, + "step": 421 + }, + { + "epoch": 1.8565153733528552, + "grad_norm": 0.32922431311553646, + "learning_rate": 6.053880040459765e-06, + "loss": 1.5768, + "step": 422 + }, + { + "epoch": 1.8609077598828696, + "grad_norm": 0.3229086221064645, + "learning_rate": 6.035976658979846e-06, + "loss": 1.6242, + "step": 423 + }, + { + "epoch": 1.8653001464128844, + "grad_norm": 0.3114352291982662, + "learning_rate": 6.018059388382327e-06, + "loss": 1.6473, + "step": 424 + }, + { + "epoch": 1.8696925329428988, + "grad_norm": 0.31045445243058734, + "learning_rate": 6.000128468880223e-06, + "loss": 1.535, + "step": 425 + }, + { + "epoch": 1.8740849194729137, + "grad_norm": 0.33523008129662585, + "learning_rate": 5.982184140869539e-06, + "loss": 1.6344, + "step": 426 + }, + { + "epoch": 1.8784773060029283, + "grad_norm": 0.3081522900734568, + "learning_rate": 5.964226644926045e-06, + "loss": 1.6386, + "step": 427 + }, + { + "epoch": 1.882869692532943, + "grad_norm": 0.2900227624443104, + "learning_rate": 5.946256221802052e-06, + "loss": 1.5843, + "step": 428 + }, + { + "epoch": 1.8872620790629575, + "grad_norm": 0.354099020882711, + "learning_rate": 5.928273112423177e-06, + "loss": 1.5758, + "step": 429 + }, + { + "epoch": 1.8916544655929721, + "grad_norm": 0.34160211290628684, + "learning_rate": 5.910277557885128e-06, + "loss": 1.5551, + "step": 430 + }, + { + "epoch": 1.896046852122987, + "grad_norm": 0.3162945077147358, + "learning_rate": 5.892269799450453e-06, + "loss": 1.6204, + "step": 431 + }, + { + "epoch": 1.9004392386530014, + "grad_norm": 0.3509407679369851, + "learning_rate": 5.8742500785453226e-06, + "loss": 1.6224, + "step": 432 + }, + { + "epoch": 1.9048316251830162, + "grad_norm": 0.2848983911128527, + "learning_rate": 5.856218636756281e-06, + "loss": 1.6059, + "step": 433 + }, + { + "epoch": 1.9092240117130308, + "grad_norm": 0.38193324539008777, + "learning_rate": 5.838175715827015e-06, + "loss": 1.5511, + "step": 434 + }, + { + "epoch": 1.9136163982430454, + "grad_norm": 0.31934508340416906, + "learning_rate": 5.820121557655109e-06, + "loss": 1.7028, + "step": 435 + }, + { + "epoch": 1.91800878477306, + "grad_norm": 0.33670681289920085, + "learning_rate": 5.8020564042888015e-06, + "loss": 1.5687, + "step": 436 + }, + { + "epoch": 1.9224011713030746, + "grad_norm": 0.318844847843566, + "learning_rate": 5.783980497923743e-06, + "loss": 1.567, + "step": 437 + }, + { + "epoch": 1.9267935578330895, + "grad_norm": 0.29610949062035014, + "learning_rate": 5.76589408089974e-06, + "loss": 1.5166, + "step": 438 + }, + { + "epoch": 1.9311859443631039, + "grad_norm": 0.28728684262143106, + "learning_rate": 5.747797395697525e-06, + "loss": 1.5813, + "step": 439 + }, + { + "epoch": 1.9355783308931187, + "grad_norm": 0.32309431631814955, + "learning_rate": 5.729690684935487e-06, + "loss": 1.641, + "step": 440 + }, + { + "epoch": 1.939970717423133, + "grad_norm": 0.2969130322032886, + "learning_rate": 5.711574191366427e-06, + "loss": 1.5608, + "step": 441 + }, + { + "epoch": 1.944363103953148, + "grad_norm": 0.42872939564364393, + "learning_rate": 5.693448157874299e-06, + "loss": 1.5983, + "step": 442 + }, + { + "epoch": 1.9487554904831625, + "grad_norm": 0.3798270075015977, + "learning_rate": 5.675312827470959e-06, + "loss": 1.5896, + "step": 443 + }, + { + "epoch": 1.9531478770131772, + "grad_norm": 0.30174438725527203, + "learning_rate": 5.657168443292909e-06, + "loss": 1.6043, + "step": 444 + }, + { + "epoch": 1.9575402635431918, + "grad_norm": 0.4119857865917871, + "learning_rate": 5.639015248598025e-06, + "loss": 1.5754, + "step": 445 + }, + { + "epoch": 1.9619326500732064, + "grad_norm": 0.32221313787015915, + "learning_rate": 5.620853486762307e-06, + "loss": 1.6569, + "step": 446 + }, + { + "epoch": 1.9663250366032212, + "grad_norm": 0.3547729959901477, + "learning_rate": 5.6026834012766155e-06, + "loss": 1.6228, + "step": 447 + }, + { + "epoch": 1.9707174231332356, + "grad_norm": 0.28279646337718173, + "learning_rate": 5.584505235743404e-06, + "loss": 1.5874, + "step": 448 + }, + { + "epoch": 1.9751098096632504, + "grad_norm": 0.3264218725678291, + "learning_rate": 5.566319233873446e-06, + "loss": 1.6313, + "step": 449 + }, + { + "epoch": 1.979502196193265, + "grad_norm": 0.28955653787826696, + "learning_rate": 5.548125639482587e-06, + "loss": 1.5989, + "step": 450 + }, + { + "epoch": 1.9838945827232797, + "grad_norm": 0.881097673855719, + "learning_rate": 5.529924696488456e-06, + "loss": 1.5796, + "step": 451 + }, + { + "epoch": 1.9882869692532943, + "grad_norm": 0.32048202284049887, + "learning_rate": 5.5117166489072014e-06, + "loss": 1.578, + "step": 452 + }, + { + "epoch": 1.992679355783309, + "grad_norm": 0.2896791795835667, + "learning_rate": 5.493501740850228e-06, + "loss": 1.6146, + "step": 453 + }, + { + "epoch": 1.9970717423133237, + "grad_norm": 0.35142442191521966, + "learning_rate": 5.475280216520913e-06, + "loss": 1.5522, + "step": 454 + }, + { + "epoch": 2.004392386530015, + "grad_norm": 0.6164551609296155, + "learning_rate": 5.45705232021134e-06, + "loss": 3.059, + "step": 455 + }, + { + "epoch": 2.0087847730600292, + "grad_norm": 0.33044889056247584, + "learning_rate": 5.438818296299015e-06, + "loss": 1.5982, + "step": 456 + }, + { + "epoch": 2.013177159590044, + "grad_norm": 0.31938902514769063, + "learning_rate": 5.4205783892435996e-06, + "loss": 1.6012, + "step": 457 + }, + { + "epoch": 2.0175695461200585, + "grad_norm": 1.1795222072761016, + "learning_rate": 5.402332843583631e-06, + "loss": 1.588, + "step": 458 + }, + { + "epoch": 2.0219619326500733, + "grad_norm": 0.31065801099482754, + "learning_rate": 5.384081903933235e-06, + "loss": 1.6023, + "step": 459 + }, + { + "epoch": 2.0263543191800877, + "grad_norm": 0.29281216005167415, + "learning_rate": 5.365825814978861e-06, + "loss": 1.5848, + "step": 460 + }, + { + "epoch": 2.0307467057101025, + "grad_norm": 0.2939045229396799, + "learning_rate": 5.34756482147599e-06, + "loss": 1.5619, + "step": 461 + }, + { + "epoch": 2.035139092240117, + "grad_norm": 0.2763356309694064, + "learning_rate": 5.3292991682458576e-06, + "loss": 1.6325, + "step": 462 + }, + { + "epoch": 2.0395314787701317, + "grad_norm": 0.32189639380967644, + "learning_rate": 5.311029100172173e-06, + "loss": 1.5714, + "step": 463 + }, + { + "epoch": 2.0439238653001466, + "grad_norm": 0.2991909339232657, + "learning_rate": 5.292754862197831e-06, + "loss": 1.5574, + "step": 464 + }, + { + "epoch": 2.048316251830161, + "grad_norm": 0.3129964860572516, + "learning_rate": 5.274476699321638e-06, + "loss": 1.6938, + "step": 465 + }, + { + "epoch": 2.052708638360176, + "grad_norm": 0.30394033701147166, + "learning_rate": 5.256194856595012e-06, + "loss": 1.5411, + "step": 466 + }, + { + "epoch": 2.05710102489019, + "grad_norm": 0.34065979728221907, + "learning_rate": 5.237909579118713e-06, + "loss": 1.6064, + "step": 467 + }, + { + "epoch": 2.061493411420205, + "grad_norm": 0.2826129934198123, + "learning_rate": 5.219621112039545e-06, + "loss": 1.6057, + "step": 468 + }, + { + "epoch": 2.0658857979502194, + "grad_norm": 0.3195263718137741, + "learning_rate": 5.201329700547077e-06, + "loss": 1.6617, + "step": 469 + }, + { + "epoch": 2.0702781844802343, + "grad_norm": 0.31086863726157415, + "learning_rate": 5.183035589870353e-06, + "loss": 1.5866, + "step": 470 + }, + { + "epoch": 2.074670571010249, + "grad_norm": 0.32171527645836867, + "learning_rate": 5.164739025274604e-06, + "loss": 1.5607, + "step": 471 + }, + { + "epoch": 2.0790629575402635, + "grad_norm": 0.30392336695969924, + "learning_rate": 5.146440252057962e-06, + "loss": 1.5615, + "step": 472 + }, + { + "epoch": 2.0834553440702783, + "grad_norm": 0.31809057906529703, + "learning_rate": 5.128139515548164e-06, + "loss": 1.5828, + "step": 473 + }, + { + "epoch": 2.0878477306002927, + "grad_norm": 0.2927979459326381, + "learning_rate": 5.109837061099274e-06, + "loss": 1.6074, + "step": 474 + }, + { + "epoch": 2.0922401171303076, + "grad_norm": 0.28785248814578307, + "learning_rate": 5.0915331340883875e-06, + "loss": 1.5747, + "step": 475 + }, + { + "epoch": 2.096632503660322, + "grad_norm": 0.32743369261911104, + "learning_rate": 5.07322797991234e-06, + "loss": 1.6416, + "step": 476 + }, + { + "epoch": 2.101024890190337, + "grad_norm": 0.3059835406726052, + "learning_rate": 5.0549218439844185e-06, + "loss": 1.5421, + "step": 477 + }, + { + "epoch": 2.1054172767203516, + "grad_norm": 0.3034377175450521, + "learning_rate": 5.036614971731076e-06, + "loss": 1.5441, + "step": 478 + }, + { + "epoch": 2.109809663250366, + "grad_norm": 0.4025542458464793, + "learning_rate": 5.018307608588637e-06, + "loss": 1.5459, + "step": 479 + }, + { + "epoch": 2.114202049780381, + "grad_norm": 0.29644549708934803, + "learning_rate": 5e-06, + "loss": 1.5845, + "step": 480 + }, + { + "epoch": 2.1185944363103952, + "grad_norm": 0.3821617049266223, + "learning_rate": 4.981692391411366e-06, + "loss": 1.5627, + "step": 481 + }, + { + "epoch": 2.12298682284041, + "grad_norm": 0.32702317134885744, + "learning_rate": 4.963385028268925e-06, + "loss": 1.6177, + "step": 482 + }, + { + "epoch": 2.1273792093704245, + "grad_norm": 0.5647033191984581, + "learning_rate": 4.945078156015582e-06, + "loss": 1.5839, + "step": 483 + }, + { + "epoch": 2.1317715959004393, + "grad_norm": 0.3030955667087971, + "learning_rate": 4.926772020087663e-06, + "loss": 1.6255, + "step": 484 + }, + { + "epoch": 2.1361639824304537, + "grad_norm": 0.33951362803589097, + "learning_rate": 4.908466865911615e-06, + "loss": 1.6158, + "step": 485 + }, + { + "epoch": 2.1405563689604685, + "grad_norm": 0.29445544201117774, + "learning_rate": 4.8901629389007274e-06, + "loss": 1.5388, + "step": 486 + }, + { + "epoch": 2.1449487554904834, + "grad_norm": 0.3590464064008913, + "learning_rate": 4.871860484451838e-06, + "loss": 1.5925, + "step": 487 + }, + { + "epoch": 2.1493411420204978, + "grad_norm": 0.29393299221073704, + "learning_rate": 4.853559747942041e-06, + "loss": 1.5905, + "step": 488 + }, + { + "epoch": 2.1537335285505126, + "grad_norm": 0.3045017300873621, + "learning_rate": 4.835260974725397e-06, + "loss": 1.6272, + "step": 489 + }, + { + "epoch": 2.158125915080527, + "grad_norm": 0.3005274825666973, + "learning_rate": 4.816964410129648e-06, + "loss": 1.5681, + "step": 490 + }, + { + "epoch": 2.162518301610542, + "grad_norm": 0.3689663925401503, + "learning_rate": 4.798670299452926e-06, + "loss": 1.5749, + "step": 491 + }, + { + "epoch": 2.166910688140556, + "grad_norm": 0.3613384924795449, + "learning_rate": 4.7803788879604585e-06, + "loss": 1.6029, + "step": 492 + }, + { + "epoch": 2.171303074670571, + "grad_norm": 0.3237181229811394, + "learning_rate": 4.762090420881289e-06, + "loss": 1.5673, + "step": 493 + }, + { + "epoch": 2.1756954612005854, + "grad_norm": 0.29104535885079846, + "learning_rate": 4.743805143404989e-06, + "loss": 1.5711, + "step": 494 + }, + { + "epoch": 2.1800878477306003, + "grad_norm": 0.310319811358895, + "learning_rate": 4.7255233006783626e-06, + "loss": 1.6182, + "step": 495 + }, + { + "epoch": 2.184480234260615, + "grad_norm": 0.31864865605925, + "learning_rate": 4.707245137802169e-06, + "loss": 1.636, + "step": 496 + }, + { + "epoch": 2.1888726207906295, + "grad_norm": 0.2889375392125057, + "learning_rate": 4.688970899827828e-06, + "loss": 1.6264, + "step": 497 + }, + { + "epoch": 2.1932650073206443, + "grad_norm": 0.337058227912909, + "learning_rate": 4.670700831754145e-06, + "loss": 1.5834, + "step": 498 + }, + { + "epoch": 2.1976573938506587, + "grad_norm": 0.3060498021030112, + "learning_rate": 4.652435178524013e-06, + "loss": 1.5825, + "step": 499 + }, + { + "epoch": 2.2020497803806736, + "grad_norm": 0.31142401889233506, + "learning_rate": 4.6341741850211406e-06, + "loss": 1.6192, + "step": 500 + }, + { + "epoch": 2.206442166910688, + "grad_norm": 0.3480563412111249, + "learning_rate": 4.615918096066766e-06, + "loss": 1.5718, + "step": 501 + }, + { + "epoch": 2.210834553440703, + "grad_norm": 0.2774237140655264, + "learning_rate": 4.597667156416371e-06, + "loss": 1.639, + "step": 502 + }, + { + "epoch": 2.2152269399707176, + "grad_norm": 0.2804945714593694, + "learning_rate": 4.5794216107564e-06, + "loss": 1.5795, + "step": 503 + }, + { + "epoch": 2.219619326500732, + "grad_norm": 0.4361446908640307, + "learning_rate": 4.561181703700986e-06, + "loss": 1.5406, + "step": 504 + }, + { + "epoch": 2.224011713030747, + "grad_norm": 0.3609641609090663, + "learning_rate": 4.542947679788662e-06, + "loss": 1.5926, + "step": 505 + }, + { + "epoch": 2.2284040995607612, + "grad_norm": 0.29968200375160275, + "learning_rate": 4.524719783479088e-06, + "loss": 1.6309, + "step": 506 + }, + { + "epoch": 2.232796486090776, + "grad_norm": 0.2902529148130268, + "learning_rate": 4.506498259149774e-06, + "loss": 1.6243, + "step": 507 + }, + { + "epoch": 2.2371888726207905, + "grad_norm": 0.35171986609259426, + "learning_rate": 4.488283351092799e-06, + "loss": 1.585, + "step": 508 + }, + { + "epoch": 2.2415812591508053, + "grad_norm": 0.3140743538598277, + "learning_rate": 4.470075303511546e-06, + "loss": 1.5835, + "step": 509 + }, + { + "epoch": 2.24597364568082, + "grad_norm": 0.743746136107164, + "learning_rate": 4.451874360517413e-06, + "loss": 1.497, + "step": 510 + }, + { + "epoch": 2.2503660322108345, + "grad_norm": 0.30150754058851265, + "learning_rate": 4.433680766126554e-06, + "loss": 1.6938, + "step": 511 + }, + { + "epoch": 2.2547584187408494, + "grad_norm": 0.3418824009574996, + "learning_rate": 4.4154947642566e-06, + "loss": 1.5996, + "step": 512 + }, + { + "epoch": 2.2591508052708638, + "grad_norm": 4.265008272718185, + "learning_rate": 4.397316598723385e-06, + "loss": 1.5282, + "step": 513 + }, + { + "epoch": 2.2635431918008786, + "grad_norm": 0.3174625288266337, + "learning_rate": 4.379146513237695e-06, + "loss": 1.646, + "step": 514 + }, + { + "epoch": 2.267935578330893, + "grad_norm": 0.3319690818479321, + "learning_rate": 4.360984751401977e-06, + "loss": 1.5278, + "step": 515 + }, + { + "epoch": 2.272327964860908, + "grad_norm": 0.30044205636049554, + "learning_rate": 4.342831556707093e-06, + "loss": 1.609, + "step": 516 + }, + { + "epoch": 2.276720351390922, + "grad_norm": 0.33317854776862893, + "learning_rate": 4.3246871725290414e-06, + "loss": 1.5675, + "step": 517 + }, + { + "epoch": 2.281112737920937, + "grad_norm": 0.31632328292339285, + "learning_rate": 4.306551842125702e-06, + "loss": 1.5537, + "step": 518 + }, + { + "epoch": 2.2855051244509514, + "grad_norm": 0.4655555090676632, + "learning_rate": 4.2884258086335755e-06, + "loss": 1.5998, + "step": 519 + }, + { + "epoch": 2.2898975109809663, + "grad_norm": 0.2769381188179044, + "learning_rate": 4.270309315064514e-06, + "loss": 1.6107, + "step": 520 + }, + { + "epoch": 2.294289897510981, + "grad_norm": 0.28065221196751095, + "learning_rate": 4.252202604302476e-06, + "loss": 1.6556, + "step": 521 + }, + { + "epoch": 2.2986822840409955, + "grad_norm": 0.42592024405532036, + "learning_rate": 4.234105919100261e-06, + "loss": 1.5993, + "step": 522 + }, + { + "epoch": 2.3030746705710103, + "grad_norm": 0.3058126067604197, + "learning_rate": 4.21601950207626e-06, + "loss": 1.6123, + "step": 523 + }, + { + "epoch": 2.3074670571010247, + "grad_norm": 0.3341616849592129, + "learning_rate": 4.1979435957111984e-06, + "loss": 1.617, + "step": 524 + }, + { + "epoch": 2.3118594436310396, + "grad_norm": 0.29572720448399287, + "learning_rate": 4.179878442344892e-06, + "loss": 1.6266, + "step": 525 + }, + { + "epoch": 2.316251830161054, + "grad_norm": 0.31535745712469815, + "learning_rate": 4.161824284172985e-06, + "loss": 1.5354, + "step": 526 + }, + { + "epoch": 2.320644216691069, + "grad_norm": 0.3933501995617675, + "learning_rate": 4.14378136324372e-06, + "loss": 1.5932, + "step": 527 + }, + { + "epoch": 2.3250366032210836, + "grad_norm": 0.2955075282035676, + "learning_rate": 4.125749921454679e-06, + "loss": 1.6059, + "step": 528 + }, + { + "epoch": 2.329428989751098, + "grad_norm": 0.3156864130791791, + "learning_rate": 4.107730200549549e-06, + "loss": 1.5252, + "step": 529 + }, + { + "epoch": 2.333821376281113, + "grad_norm": 0.31312483297202565, + "learning_rate": 4.089722442114873e-06, + "loss": 1.6255, + "step": 530 + }, + { + "epoch": 2.3382137628111272, + "grad_norm": 0.3984616100434099, + "learning_rate": 4.071726887576823e-06, + "loss": 1.6074, + "step": 531 + }, + { + "epoch": 2.342606149341142, + "grad_norm": 0.3245701785753724, + "learning_rate": 4.053743778197951e-06, + "loss": 1.6017, + "step": 532 + }, + { + "epoch": 2.3469985358711565, + "grad_norm": 0.3109891324799959, + "learning_rate": 4.035773355073956e-06, + "loss": 1.7142, + "step": 533 + }, + { + "epoch": 2.3513909224011713, + "grad_norm": 0.2812903996182882, + "learning_rate": 4.017815859130462e-06, + "loss": 1.6089, + "step": 534 + }, + { + "epoch": 2.355783308931186, + "grad_norm": 0.31403362482776737, + "learning_rate": 3.999871531119779e-06, + "loss": 1.5455, + "step": 535 + }, + { + "epoch": 2.3601756954612005, + "grad_norm": 0.3608937370940864, + "learning_rate": 3.981940611617675e-06, + "loss": 1.5522, + "step": 536 + }, + { + "epoch": 2.3645680819912154, + "grad_norm": 0.33749233448354743, + "learning_rate": 3.9640233410201555e-06, + "loss": 1.6777, + "step": 537 + }, + { + "epoch": 2.3689604685212298, + "grad_norm": 0.5153655618339453, + "learning_rate": 3.946119959540235e-06, + "loss": 1.5495, + "step": 538 + }, + { + "epoch": 2.3733528550512446, + "grad_norm": 0.31504839746477054, + "learning_rate": 3.928230707204729e-06, + "loss": 1.5521, + "step": 539 + }, + { + "epoch": 2.377745241581259, + "grad_norm": 0.27776484632999904, + "learning_rate": 3.910355823851008e-06, + "loss": 1.5042, + "step": 540 + }, + { + "epoch": 2.382137628111274, + "grad_norm": 0.2939400235215093, + "learning_rate": 3.892495549123821e-06, + "loss": 1.6054, + "step": 541 + }, + { + "epoch": 2.3865300146412887, + "grad_norm": 0.3176044597823968, + "learning_rate": 3.874650122472049e-06, + "loss": 1.5915, + "step": 542 + }, + { + "epoch": 2.390922401171303, + "grad_norm": 0.42146288925638176, + "learning_rate": 3.856819783145514e-06, + "loss": 1.5908, + "step": 543 + }, + { + "epoch": 2.395314787701318, + "grad_norm": 0.30849439803473244, + "learning_rate": 3.8390047701917625e-06, + "loss": 1.6203, + "step": 544 + }, + { + "epoch": 2.3997071742313323, + "grad_norm": 0.3104483621519379, + "learning_rate": 3.821205322452863e-06, + "loss": 1.5298, + "step": 545 + }, + { + "epoch": 2.404099560761347, + "grad_norm": 1.6029942546871143, + "learning_rate": 3.803421678562213e-06, + "loss": 1.4927, + "step": 546 + }, + { + "epoch": 2.4084919472913615, + "grad_norm": 0.332446513039922, + "learning_rate": 3.785654076941317e-06, + "loss": 1.5578, + "step": 547 + }, + { + "epoch": 2.4128843338213763, + "grad_norm": 0.35434480211680486, + "learning_rate": 3.7679027557966136e-06, + "loss": 1.5855, + "step": 548 + }, + { + "epoch": 2.4172767203513907, + "grad_norm": 0.3278075604228065, + "learning_rate": 3.750167953116272e-06, + "loss": 1.6277, + "step": 549 + }, + { + "epoch": 2.4216691068814056, + "grad_norm": 0.28809005974662427, + "learning_rate": 3.7324499066670008e-06, + "loss": 1.6628, + "step": 550 + }, + { + "epoch": 2.42606149341142, + "grad_norm": 0.315430224939953, + "learning_rate": 3.71474885399086e-06, + "loss": 1.5712, + "step": 551 + }, + { + "epoch": 2.430453879941435, + "grad_norm": 0.29541436843467994, + "learning_rate": 3.6970650324020784e-06, + "loss": 1.6554, + "step": 552 + }, + { + "epoch": 2.4348462664714496, + "grad_norm": 0.31836804528358076, + "learning_rate": 3.6793986789838745e-06, + "loss": 1.5242, + "step": 553 + }, + { + "epoch": 2.439238653001464, + "grad_norm": 0.3097916118349815, + "learning_rate": 3.6617500305852676e-06, + "loss": 1.5067, + "step": 554 + }, + { + "epoch": 2.443631039531479, + "grad_norm": 0.30552956835830547, + "learning_rate": 3.6441193238179152e-06, + "loss": 1.5544, + "step": 555 + }, + { + "epoch": 2.4480234260614933, + "grad_norm": 0.33196026415940266, + "learning_rate": 3.6265067950529363e-06, + "loss": 1.5967, + "step": 556 + }, + { + "epoch": 2.452415812591508, + "grad_norm": 0.45992827917949797, + "learning_rate": 3.6089126804177373e-06, + "loss": 1.5314, + "step": 557 + }, + { + "epoch": 2.4568081991215225, + "grad_norm": 0.3850494843555264, + "learning_rate": 3.5913372157928515e-06, + "loss": 1.584, + "step": 558 + }, + { + "epoch": 2.4612005856515373, + "grad_norm": 0.3130838633878226, + "learning_rate": 3.5737806368087776e-06, + "loss": 1.5472, + "step": 559 + }, + { + "epoch": 2.465592972181552, + "grad_norm": 0.33165390842885173, + "learning_rate": 3.556243178842816e-06, + "loss": 1.5396, + "step": 560 + }, + { + "epoch": 2.4699853587115665, + "grad_norm": 0.30410685779778446, + "learning_rate": 3.5387250770159152e-06, + "loss": 1.6436, + "step": 561 + }, + { + "epoch": 2.4743777452415814, + "grad_norm": 0.43575069016621865, + "learning_rate": 3.521226566189523e-06, + "loss": 1.6752, + "step": 562 + }, + { + "epoch": 2.4787701317715958, + "grad_norm": 0.3043077774937357, + "learning_rate": 3.5037478809624315e-06, + "loss": 1.5879, + "step": 563 + }, + { + "epoch": 2.4831625183016106, + "grad_norm": 0.29787633854732515, + "learning_rate": 3.48628925566764e-06, + "loss": 1.6488, + "step": 564 + }, + { + "epoch": 2.487554904831625, + "grad_norm": 0.33159507739731536, + "learning_rate": 3.4688509243692037e-06, + "loss": 1.5911, + "step": 565 + }, + { + "epoch": 2.49194729136164, + "grad_norm": 0.2790057356017803, + "learning_rate": 3.4514331208591027e-06, + "loss": 1.6126, + "step": 566 + }, + { + "epoch": 2.4963396778916547, + "grad_norm": 0.307396122565463, + "learning_rate": 3.4340360786541067e-06, + "loss": 1.6613, + "step": 567 + }, + { + "epoch": 2.500732064421669, + "grad_norm": 0.2896656371367208, + "learning_rate": 3.416660030992639e-06, + "loss": 1.6264, + "step": 568 + }, + { + "epoch": 2.505124450951684, + "grad_norm": 0.28661415223621256, + "learning_rate": 3.3993052108316566e-06, + "loss": 1.5758, + "step": 569 + }, + { + "epoch": 2.5095168374816983, + "grad_norm": 0.41303310304829133, + "learning_rate": 3.381971850843523e-06, + "loss": 1.5584, + "step": 570 + }, + { + "epoch": 2.513909224011713, + "grad_norm": 0.28374903595837936, + "learning_rate": 3.3646601834128924e-06, + "loss": 1.6081, + "step": 571 + }, + { + "epoch": 2.5183016105417275, + "grad_norm": 0.3292975740580804, + "learning_rate": 3.3473704406335873e-06, + "loss": 1.5221, + "step": 572 + }, + { + "epoch": 2.5226939970717424, + "grad_norm": 0.3452782616763211, + "learning_rate": 3.3301028543054935e-06, + "loss": 1.5544, + "step": 573 + }, + { + "epoch": 2.527086383601757, + "grad_norm": 0.3294649058226655, + "learning_rate": 3.3128576559314507e-06, + "loss": 1.5776, + "step": 574 + }, + { + "epoch": 2.5314787701317716, + "grad_norm": 0.3605542745485364, + "learning_rate": 3.295635076714144e-06, + "loss": 1.5856, + "step": 575 + }, + { + "epoch": 2.535871156661786, + "grad_norm": 0.2941353695509803, + "learning_rate": 3.278435347553014e-06, + "loss": 1.576, + "step": 576 + }, + { + "epoch": 2.540263543191801, + "grad_norm": 0.3182229392679776, + "learning_rate": 3.2612586990411516e-06, + "loss": 1.5215, + "step": 577 + }, + { + "epoch": 2.5446559297218156, + "grad_norm": 0.3160193285643318, + "learning_rate": 3.2441053614622096e-06, + "loss": 1.6507, + "step": 578 + }, + { + "epoch": 2.54904831625183, + "grad_norm": 0.3308377767667811, + "learning_rate": 3.226975564787322e-06, + "loss": 1.5665, + "step": 579 + }, + { + "epoch": 2.553440702781845, + "grad_norm": 0.2976887584358953, + "learning_rate": 3.209869538672008e-06, + "loss": 1.5684, + "step": 580 + }, + { + "epoch": 2.5578330893118597, + "grad_norm": 0.3562637257648593, + "learning_rate": 3.1927875124531048e-06, + "loss": 1.5817, + "step": 581 + }, + { + "epoch": 2.562225475841874, + "grad_norm": 0.3864243441965815, + "learning_rate": 3.1757297151456844e-06, + "loss": 1.4897, + "step": 582 + }, + { + "epoch": 2.5666178623718885, + "grad_norm": 0.3054191855541809, + "learning_rate": 3.158696375439989e-06, + "loss": 1.5817, + "step": 583 + }, + { + "epoch": 2.5710102489019033, + "grad_norm": 0.40491573034866035, + "learning_rate": 3.141687721698363e-06, + "loss": 1.638, + "step": 584 + }, + { + "epoch": 2.575402635431918, + "grad_norm": 0.3120082453985608, + "learning_rate": 3.1247039819521907e-06, + "loss": 1.6502, + "step": 585 + }, + { + "epoch": 2.5797950219619326, + "grad_norm": 0.29842186795454706, + "learning_rate": 3.107745383898841e-06, + "loss": 1.5946, + "step": 586 + }, + { + "epoch": 2.5841874084919474, + "grad_norm": 0.3199137178402513, + "learning_rate": 3.090812154898614e-06, + "loss": 1.5514, + "step": 587 + }, + { + "epoch": 2.588579795021962, + "grad_norm": 0.32241669340221557, + "learning_rate": 3.0739045219716888e-06, + "loss": 1.6556, + "step": 588 + }, + { + "epoch": 2.5929721815519766, + "grad_norm": 0.32944149352991675, + "learning_rate": 3.0570227117950857e-06, + "loss": 1.5659, + "step": 589 + }, + { + "epoch": 2.597364568081991, + "grad_norm": 0.28766693574428776, + "learning_rate": 3.040166950699626e-06, + "loss": 1.5592, + "step": 590 + }, + { + "epoch": 2.601756954612006, + "grad_norm": 0.33175991862359117, + "learning_rate": 3.0233374646668935e-06, + "loss": 1.6019, + "step": 591 + }, + { + "epoch": 2.6061493411420207, + "grad_norm": 0.30897588965031886, + "learning_rate": 3.006534479326211e-06, + "loss": 1.5768, + "step": 592 + }, + { + "epoch": 2.610541727672035, + "grad_norm": 0.5063934549092376, + "learning_rate": 2.9897582199516105e-06, + "loss": 1.5424, + "step": 593 + }, + { + "epoch": 2.61493411420205, + "grad_norm": 0.40175797042520484, + "learning_rate": 2.973008911458816e-06, + "loss": 1.6206, + "step": 594 + }, + { + "epoch": 2.6193265007320643, + "grad_norm": 0.3006477529439128, + "learning_rate": 2.9562867784022264e-06, + "loss": 1.6468, + "step": 595 + }, + { + "epoch": 2.623718887262079, + "grad_norm": 0.32028780520359634, + "learning_rate": 2.9395920449719027e-06, + "loss": 1.6246, + "step": 596 + }, + { + "epoch": 2.6281112737920935, + "grad_norm": 0.2894216705891592, + "learning_rate": 2.9229249349905686e-06, + "loss": 1.5008, + "step": 597 + }, + { + "epoch": 2.6325036603221084, + "grad_norm": 0.30155534431390957, + "learning_rate": 2.9062856719106034e-06, + "loss": 1.5452, + "step": 598 + }, + { + "epoch": 2.636896046852123, + "grad_norm": 0.3405268172268061, + "learning_rate": 2.8896744788110497e-06, + "loss": 1.5117, + "step": 599 + }, + { + "epoch": 2.6412884333821376, + "grad_norm": 0.2963355726805553, + "learning_rate": 2.873091578394626e-06, + "loss": 1.5697, + "step": 600 + }, + { + "epoch": 2.6456808199121524, + "grad_norm": 0.3931854175541597, + "learning_rate": 2.8565371929847286e-06, + "loss": 1.5837, + "step": 601 + }, + { + "epoch": 2.650073206442167, + "grad_norm": 0.3081629305989419, + "learning_rate": 2.8400115445224676e-06, + "loss": 1.5532, + "step": 602 + }, + { + "epoch": 2.6544655929721817, + "grad_norm": 0.321923571672651, + "learning_rate": 2.8235148545636776e-06, + "loss": 1.5905, + "step": 603 + }, + { + "epoch": 2.658857979502196, + "grad_norm": 0.30381368428150873, + "learning_rate": 2.8070473442759517e-06, + "loss": 1.6202, + "step": 604 + }, + { + "epoch": 2.663250366032211, + "grad_norm": 0.34379437733116675, + "learning_rate": 2.7906092344356827e-06, + "loss": 1.5631, + "step": 605 + }, + { + "epoch": 2.6676427525622257, + "grad_norm": 0.3251066020024119, + "learning_rate": 2.7742007454250963e-06, + "loss": 1.6055, + "step": 606 + }, + { + "epoch": 2.67203513909224, + "grad_norm": 0.31178463454292143, + "learning_rate": 2.757822097229294e-06, + "loss": 1.6396, + "step": 607 + }, + { + "epoch": 2.6764275256222545, + "grad_norm": 0.6034144070140214, + "learning_rate": 2.741473509433314e-06, + "loss": 1.5724, + "step": 608 + }, + { + "epoch": 2.6808199121522693, + "grad_norm": 0.2879175094559386, + "learning_rate": 2.7251552012191763e-06, + "loss": 1.5744, + "step": 609 + }, + { + "epoch": 2.685212298682284, + "grad_norm": 0.3519868147832161, + "learning_rate": 2.708867391362948e-06, + "loss": 1.5935, + "step": 610 + }, + { + "epoch": 2.6896046852122986, + "grad_norm": 0.29308720247015513, + "learning_rate": 2.692610298231819e-06, + "loss": 1.595, + "step": 611 + }, + { + "epoch": 2.6939970717423134, + "grad_norm": 0.2759532499836008, + "learning_rate": 2.6763841397811576e-06, + "loss": 1.5826, + "step": 612 + }, + { + "epoch": 2.6983894582723282, + "grad_norm": 0.29629200692084146, + "learning_rate": 2.660189133551603e-06, + "loss": 1.5978, + "step": 613 + }, + { + "epoch": 2.7027818448023426, + "grad_norm": 0.4877644396819389, + "learning_rate": 2.644025496666143e-06, + "loss": 1.5618, + "step": 614 + }, + { + "epoch": 2.707174231332357, + "grad_norm": 0.32075291148747, + "learning_rate": 2.6278934458271998e-06, + "loss": 1.6419, + "step": 615 + }, + { + "epoch": 2.711566617862372, + "grad_norm": 0.31507589414611065, + "learning_rate": 2.6117931973137295e-06, + "loss": 1.5699, + "step": 616 + }, + { + "epoch": 2.7159590043923867, + "grad_norm": 0.3588085551568672, + "learning_rate": 2.5957249669783257e-06, + "loss": 1.5814, + "step": 617 + }, + { + "epoch": 2.720351390922401, + "grad_norm": 0.36359707618102943, + "learning_rate": 2.5796889702443127e-06, + "loss": 1.6475, + "step": 618 + }, + { + "epoch": 2.724743777452416, + "grad_norm": 0.29390296240068264, + "learning_rate": 2.5636854221028764e-06, + "loss": 1.5175, + "step": 619 + }, + { + "epoch": 2.7291361639824303, + "grad_norm": 0.26592753999493085, + "learning_rate": 2.54771453711016e-06, + "loss": 1.5708, + "step": 620 + }, + { + "epoch": 2.733528550512445, + "grad_norm": 0.34603630736032, + "learning_rate": 2.531776529384407e-06, + "loss": 1.6302, + "step": 621 + }, + { + "epoch": 2.7379209370424595, + "grad_norm": 0.28041321457757834, + "learning_rate": 2.5158716126030835e-06, + "loss": 1.5336, + "step": 622 + }, + { + "epoch": 2.7423133235724744, + "grad_norm": 0.3212436991662462, + "learning_rate": 2.5000000000000015e-06, + "loss": 1.5925, + "step": 623 + }, + { + "epoch": 2.746705710102489, + "grad_norm": 0.3047611974850631, + "learning_rate": 2.484161904362481e-06, + "loss": 1.6048, + "step": 624 + }, + { + "epoch": 2.7510980966325036, + "grad_norm": 0.3048539362248767, + "learning_rate": 2.4683575380284873e-06, + "loss": 1.5498, + "step": 625 + }, + { + "epoch": 2.7554904831625184, + "grad_norm": 0.34208807818895065, + "learning_rate": 2.4525871128837775e-06, + "loss": 1.576, + "step": 626 + }, + { + "epoch": 2.759882869692533, + "grad_norm": 0.2949849660779719, + "learning_rate": 2.436850840359073e-06, + "loss": 1.6486, + "step": 627 + }, + { + "epoch": 2.7642752562225477, + "grad_norm": 0.2779672502557031, + "learning_rate": 2.421148931427213e-06, + "loss": 1.5677, + "step": 628 + }, + { + "epoch": 2.768667642752562, + "grad_norm": 0.2960325820768123, + "learning_rate": 2.4054815966003365e-06, + "loss": 1.5999, + "step": 629 + }, + { + "epoch": 2.773060029282577, + "grad_norm": 0.30799619640674625, + "learning_rate": 2.389849045927049e-06, + "loss": 1.5423, + "step": 630 + }, + { + "epoch": 2.7774524158125917, + "grad_norm": 0.689995508540048, + "learning_rate": 2.3742514889896196e-06, + "loss": 1.6127, + "step": 631 + }, + { + "epoch": 2.781844802342606, + "grad_norm": 0.3022950443793775, + "learning_rate": 2.3586891349011555e-06, + "loss": 1.6556, + "step": 632 + }, + { + "epoch": 2.786237188872621, + "grad_norm": 0.32506564068047783, + "learning_rate": 2.3431621923028146e-06, + "loss": 1.581, + "step": 633 + }, + { + "epoch": 2.7906295754026353, + "grad_norm": 0.3659374843681539, + "learning_rate": 2.3276708693609947e-06, + "loss": 1.5245, + "step": 634 + }, + { + "epoch": 2.79502196193265, + "grad_norm": 0.2983223667818601, + "learning_rate": 2.3122153737645512e-06, + "loss": 1.5934, + "step": 635 + }, + { + "epoch": 2.7994143484626646, + "grad_norm": 0.3863672711683231, + "learning_rate": 2.296795912722014e-06, + "loss": 1.6243, + "step": 636 + }, + { + "epoch": 2.8038067349926794, + "grad_norm": 0.30331251914397456, + "learning_rate": 2.281412692958793e-06, + "loss": 1.5746, + "step": 637 + }, + { + "epoch": 2.8081991215226942, + "grad_norm": 0.3196136845979999, + "learning_rate": 2.266065920714432e-06, + "loss": 1.6012, + "step": 638 + }, + { + "epoch": 2.8125915080527086, + "grad_norm": 0.27541958919800974, + "learning_rate": 2.2507558017398263e-06, + "loss": 1.6426, + "step": 639 + }, + { + "epoch": 2.816983894582723, + "grad_norm": 0.28786178032428017, + "learning_rate": 2.2354825412944643e-06, + "loss": 1.5385, + "step": 640 + }, + { + "epoch": 2.821376281112738, + "grad_norm": 0.32511038200012793, + "learning_rate": 2.2202463441436885e-06, + "loss": 1.7098, + "step": 641 + }, + { + "epoch": 2.8257686676427527, + "grad_norm": 0.3001315837884633, + "learning_rate": 2.2050474145559326e-06, + "loss": 1.612, + "step": 642 + }, + { + "epoch": 2.830161054172767, + "grad_norm": 0.47527905160043865, + "learning_rate": 2.1898859563000003e-06, + "loss": 1.6511, + "step": 643 + }, + { + "epoch": 2.834553440702782, + "grad_norm": 0.3168531896637398, + "learning_rate": 2.174762172642319e-06, + "loss": 1.5376, + "step": 644 + }, + { + "epoch": 2.8389458272327968, + "grad_norm": 0.4624637579943484, + "learning_rate": 2.159676266344222e-06, + "loss": 1.6121, + "step": 645 + }, + { + "epoch": 2.843338213762811, + "grad_norm": 0.291816559938584, + "learning_rate": 2.144628439659231e-06, + "loss": 1.4975, + "step": 646 + }, + { + "epoch": 2.8477306002928255, + "grad_norm": 0.29391820275780933, + "learning_rate": 2.1296188943303446e-06, + "loss": 1.6212, + "step": 647 + }, + { + "epoch": 2.8521229868228404, + "grad_norm": 0.31828111479061616, + "learning_rate": 2.1146478315873238e-06, + "loss": 1.6776, + "step": 648 + }, + { + "epoch": 2.856515373352855, + "grad_norm": 0.2976871167454781, + "learning_rate": 2.09971545214401e-06, + "loss": 1.5939, + "step": 649 + }, + { + "epoch": 2.8609077598828696, + "grad_norm": 0.2929358425075381, + "learning_rate": 2.0848219561956205e-06, + "loss": 1.6026, + "step": 650 + }, + { + "epoch": 2.8653001464128844, + "grad_norm": 0.28268849637868454, + "learning_rate": 2.06996754341607e-06, + "loss": 1.5525, + "step": 651 + }, + { + "epoch": 2.869692532942899, + "grad_norm": 0.3205109285976867, + "learning_rate": 2.0551524129552986e-06, + "loss": 1.6212, + "step": 652 + }, + { + "epoch": 2.8740849194729137, + "grad_norm": 0.3396332248604952, + "learning_rate": 2.04037676343659e-06, + "loss": 1.6332, + "step": 653 + }, + { + "epoch": 2.878477306002928, + "grad_norm": 0.2893628943703605, + "learning_rate": 2.0256407929539203e-06, + "loss": 1.5796, + "step": 654 + }, + { + "epoch": 2.882869692532943, + "grad_norm": 0.31138257479839093, + "learning_rate": 2.0109446990692967e-06, + "loss": 1.5832, + "step": 655 + }, + { + "epoch": 2.8872620790629577, + "grad_norm": 0.30316986773262733, + "learning_rate": 1.996288678810105e-06, + "loss": 1.6454, + "step": 656 + }, + { + "epoch": 2.891654465592972, + "grad_norm": 0.3616046943167089, + "learning_rate": 1.98167292866648e-06, + "loss": 1.4587, + "step": 657 + }, + { + "epoch": 2.896046852122987, + "grad_norm": 0.3182467743924217, + "learning_rate": 1.967097644588657e-06, + "loss": 1.6342, + "step": 658 + }, + { + "epoch": 2.9004392386530014, + "grad_norm": 0.291092108286567, + "learning_rate": 1.9525630219843524e-06, + "loss": 1.5981, + "step": 659 + }, + { + "epoch": 2.904831625183016, + "grad_norm": 0.3198399401699167, + "learning_rate": 1.9380692557161453e-06, + "loss": 1.5587, + "step": 660 + }, + { + "epoch": 2.9092240117130306, + "grad_norm": 0.284926327834223, + "learning_rate": 1.923616540098864e-06, + "loss": 1.5807, + "step": 661 + }, + { + "epoch": 2.9136163982430454, + "grad_norm": 0.34133283690090355, + "learning_rate": 1.9092050688969736e-06, + "loss": 1.6016, + "step": 662 + }, + { + "epoch": 2.9180087847730602, + "grad_norm": 0.3234782424139794, + "learning_rate": 1.8948350353219913e-06, + "loss": 1.5699, + "step": 663 + }, + { + "epoch": 2.9224011713030746, + "grad_norm": 0.32527383445723507, + "learning_rate": 1.880506632029883e-06, + "loss": 1.6162, + "step": 664 + }, + { + "epoch": 2.9267935578330895, + "grad_norm": 0.3194926131288373, + "learning_rate": 1.8662200511184874e-06, + "loss": 1.5822, + "step": 665 + }, + { + "epoch": 2.931185944363104, + "grad_norm": 0.3436218283010263, + "learning_rate": 1.8519754841249444e-06, + "loss": 1.5979, + "step": 666 + }, + { + "epoch": 2.9355783308931187, + "grad_norm": 0.3035445625939091, + "learning_rate": 1.8377731220231144e-06, + "loss": 1.547, + "step": 667 + }, + { + "epoch": 2.939970717423133, + "grad_norm": 0.33552700833979515, + "learning_rate": 1.8236131552210317e-06, + "loss": 1.5559, + "step": 668 + }, + { + "epoch": 2.944363103953148, + "grad_norm": 0.30372475017209144, + "learning_rate": 1.8094957735583463e-06, + "loss": 1.5642, + "step": 669 + }, + { + "epoch": 2.9487554904831628, + "grad_norm": 0.31181597688183876, + "learning_rate": 1.795421166303773e-06, + "loss": 1.5574, + "step": 670 + }, + { + "epoch": 2.953147877013177, + "grad_norm": 0.33729677529922286, + "learning_rate": 1.7813895221525651e-06, + "loss": 1.6204, + "step": 671 + }, + { + "epoch": 2.9575402635431915, + "grad_norm": 0.4308912143370284, + "learning_rate": 1.7674010292239746e-06, + "loss": 1.5801, + "step": 672 + }, + { + "epoch": 2.9619326500732064, + "grad_norm": 0.30463101988066804, + "learning_rate": 1.7534558750587345e-06, + "loss": 1.5911, + "step": 673 + }, + { + "epoch": 2.966325036603221, + "grad_norm": 0.3768568218829568, + "learning_rate": 1.7395542466165488e-06, + "loss": 1.5717, + "step": 674 + }, + { + "epoch": 2.9707174231332356, + "grad_norm": 0.2999853917153679, + "learning_rate": 1.7256963302735752e-06, + "loss": 1.5727, + "step": 675 + }, + { + "epoch": 2.9751098096632504, + "grad_norm": 0.3013746219343556, + "learning_rate": 1.7118823118199363e-06, + "loss": 1.6322, + "step": 676 + }, + { + "epoch": 2.9795021961932653, + "grad_norm": 0.3074935580559331, + "learning_rate": 1.6981123764572272e-06, + "loss": 1.5159, + "step": 677 + }, + { + "epoch": 2.9838945827232797, + "grad_norm": 0.3333567378869067, + "learning_rate": 1.6843867087960252e-06, + "loss": 1.6671, + "step": 678 + }, + { + "epoch": 2.988286969253294, + "grad_norm": 0.32409788340370355, + "learning_rate": 1.670705492853421e-06, + "loss": 1.7106, + "step": 679 + }, + { + "epoch": 2.992679355783309, + "grad_norm": 0.29584784991310104, + "learning_rate": 1.6570689120505557e-06, + "loss": 1.6037, + "step": 680 + }, + { + "epoch": 2.9970717423133237, + "grad_norm": 0.33480028358751196, + "learning_rate": 1.6434771492101487e-06, + "loss": 1.5156, + "step": 681 + }, + { + "epoch": 3.004392386530015, + "grad_norm": 0.31677037762304755, + "learning_rate": 1.6299303865540616e-06, + "loss": 3.3042, + "step": 682 + }, + { + "epoch": 3.0087847730600292, + "grad_norm": 0.3119634571083339, + "learning_rate": 1.6164288057008466e-06, + "loss": 1.6111, + "step": 683 + }, + { + "epoch": 3.013177159590044, + "grad_norm": 0.3176907934650192, + "learning_rate": 1.6029725876633085e-06, + "loss": 1.5259, + "step": 684 + } + ], + "logging_steps": 1, + "max_steps": 908, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 114, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.8086729312832061e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-684/training_args.bin b/checkpoint-684/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..282808fde3b579abe8666eba7a8b5732d5eaf3d4 --- /dev/null +++ b/checkpoint-684/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f598588716f36b5db7091989142a430a4fd275bce6f1e0d8e6c17f595272e1ec +size 9041 diff --git a/checkpoint-684/zero_to_fp32.py b/checkpoint-684/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..e69ecd9acb5a235ffbf927091051106d902b3d39 --- /dev/null +++ b/checkpoint-684/zero_to_fp32.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import json +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + state_dict_split = split_torch_state_dict_into_shards(state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors} + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard, output_path, metadata={"format": "pt"}) + else: + torch.save(shard, output_path) + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a2e3db68c34cc62b8efc2666aa4252b63d733ee7 --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "_attn_implementation_autoset": true, + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 999, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 32768, + "max_position_embeddings": 131072, + "model_type": "mistral", + "num_attention_heads": 32, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000000.0, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.3", + "use_cache": false, + "vocab_size": 131072 +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..4332a4c6d2a485806f55e02260db8a73698e11fb --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8fab5451b04e019d1612cc5088df62aee9a8208c --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e5daf1e619597fd56ff036925ea189916417c665d5bdd61b80199cf8d4e6d2 +size 17078029 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..814201fe59040b12e956fb649c749a42bdce1e2f --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,8021 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "[IMG]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "[IMG_BREAK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "[IMG_END]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "[PREFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "[MIDDLE]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "[SUFFIX]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "[SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "[/SYSTEM_PROMPT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "[TOOL_CONTENT]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "109": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "110": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "111": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "112": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "113": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "114": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "115": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "116": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "117": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "118": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "119": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "120": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "121": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "122": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "123": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "124": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "125": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "126": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "127": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "129": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "130": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "131": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "132": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "133": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "134": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "135": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "136": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "137": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "138": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "139": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "140": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "141": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "142": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "143": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "144": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "145": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "146": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "147": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "148": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "149": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "150": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "152": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "153": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "154": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "155": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "156": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "157": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "158": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "159": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "160": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "161": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "162": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "163": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "164": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "165": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "166": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "167": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "185": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "186": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "187": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "188": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "189": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "190": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "191": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "222": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "223": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "224": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "225": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "226": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "227": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "228": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "229": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "230": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "231": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "232": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "233": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "234": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "235": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "236": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "237": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "238": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "239": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "240": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "241": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "242": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "243": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "244": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "245": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "246": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "247": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "249": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "251": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "252": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "253": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "254": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "255": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "256": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "257": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "258": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "259": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "260": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "262": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "263": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "264": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "265": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "266": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "267": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "268": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "285": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "286": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "287": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "288": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "289": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "290": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "291": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "292": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "293": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "294": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "295": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "296": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "297": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "298": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "299": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "300": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "301": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "302": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "303": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "304": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "305": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "306": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "307": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "308": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "309": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "310": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "311": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "312": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "313": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "314": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "315": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "316": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "317": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "318": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "319": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "320": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "321": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "322": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "323": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "324": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "325": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "326": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "327": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "328": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "329": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "330": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "331": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "332": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "335": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "336": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "337": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "338": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "339": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "340": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "341": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "342": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "343": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "344": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "345": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "346": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "347": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "348": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "349": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "360": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "361": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "362": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "363": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "364": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "365": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "366": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "367": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "368": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "369": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "370": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "371": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "372": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "373": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "374": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "375": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "376": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "377": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "378": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "379": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "380": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "381": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "382": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "383": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "384": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "385": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "386": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "387": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "388": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "389": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "390": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "391": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "392": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "393": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "394": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "395": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "396": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "397": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "398": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "399": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "400": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "401": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "402": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "403": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "404": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "405": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "406": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "407": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "408": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "409": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "410": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "411": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "412": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "413": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "414": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "415": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "416": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "417": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "418": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "419": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "420": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "421": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "422": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "423": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "424": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "425": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "426": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "427": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "428": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "429": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "430": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "431": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "432": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "433": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "434": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "435": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "436": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "437": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "438": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "439": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "440": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "441": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "442": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "443": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "444": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "445": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "446": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "447": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "448": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "449": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "450": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "451": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "452": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "453": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "454": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "455": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "456": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "457": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "458": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "459": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "460": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "461": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "462": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "463": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "464": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "465": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "466": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "467": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "468": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "469": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "470": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "471": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "472": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "473": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "474": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "475": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "476": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "477": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "478": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "479": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "480": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "481": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "482": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "483": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "484": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "485": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "486": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "487": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "488": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "489": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "490": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "491": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "492": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "493": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "494": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "495": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "496": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "497": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "498": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "499": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "500": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "501": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "502": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "503": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "504": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "505": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "506": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "507": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "508": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "509": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "510": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "511": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "512": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "513": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "514": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "515": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "516": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "517": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "518": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "519": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "520": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "521": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "522": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "523": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "524": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "525": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "526": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "527": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "528": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "529": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "530": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "531": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "532": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "533": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "534": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "535": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "536": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "537": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "538": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "539": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "540": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "541": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "542": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "543": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "544": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "545": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "546": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "547": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "548": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "549": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "550": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "551": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "552": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "553": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "554": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "555": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "556": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "557": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "558": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "559": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "560": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "561": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "562": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "563": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "564": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "565": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "566": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "567": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "568": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "569": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "570": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "571": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "572": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "573": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "574": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "575": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "576": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "577": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "578": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "579": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "580": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "581": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "582": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "583": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "584": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "585": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "586": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "587": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "588": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "589": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "590": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "591": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "592": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "593": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "594": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "595": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "596": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "597": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "598": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "599": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "600": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "601": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "602": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "603": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "604": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "605": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "606": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "607": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "608": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "609": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "610": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "611": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "612": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "613": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "614": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "615": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "616": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "617": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "618": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "619": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "620": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "621": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "622": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "623": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "624": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "625": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "626": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "627": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "628": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "629": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "630": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "631": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "632": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "633": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "634": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "635": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "636": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "637": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "638": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "639": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "640": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "641": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "642": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "643": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "644": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "645": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "646": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "647": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "648": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "649": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "650": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "651": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "652": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "653": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "654": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "655": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "656": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "659": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "660": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "661": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "662": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "663": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "664": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "669": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "670": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "671": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "672": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "673": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "674": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "675": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "676": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "677": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "678": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "679": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "680": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "681": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "682": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "683": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "684": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "685": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "686": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "687": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "688": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "689": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "690": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "691": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "692": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "693": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "694": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "695": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "696": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "697": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "698": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "699": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "700": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "701": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "702": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "703": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "704": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "705": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "706": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "707": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "708": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "709": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "710": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "711": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "712": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "713": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "714": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "715": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "716": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "717": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "718": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "719": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "720": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "721": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "722": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "723": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "724": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "725": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "726": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "727": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "728": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "729": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "730": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "731": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "732": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "733": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "734": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "735": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "736": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "737": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "738": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "739": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "740": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "741": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "742": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "743": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "744": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "745": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "746": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "747": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "748": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "749": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "750": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "751": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "752": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "753": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "754": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "755": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "756": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "757": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "758": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "759": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "760": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "761": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "762": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "763": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "764": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "765": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "766": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "767": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "768": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "769": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "770": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "771": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "772": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "773": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "774": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "775": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "776": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "777": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "778": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "779": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "780": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "781": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "782": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "783": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "784": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "785": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "786": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "787": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "788": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "789": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "790": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "791": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "792": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "793": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "794": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "795": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "796": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "797": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "798": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "799": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "800": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "801": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "802": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "803": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "804": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "805": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "806": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "807": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "808": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "809": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "810": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "811": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "812": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "813": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "814": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "815": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "816": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "817": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "818": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "819": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "820": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "821": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "822": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "823": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "824": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "825": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "826": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "827": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "828": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "829": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "830": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "831": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "832": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "833": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "834": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "835": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "836": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "837": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "838": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "839": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "840": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "841": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "842": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "843": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "844": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "845": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "846": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "847": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "848": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "849": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "850": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "851": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "852": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "853": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "854": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "855": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "856": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "857": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "858": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "859": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "860": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "861": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "862": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "863": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "864": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "865": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "866": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "867": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "868": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "869": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "870": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "871": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "872": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "873": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "874": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "875": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "876": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "877": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "878": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "879": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "880": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "881": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "882": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "883": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "884": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "885": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "886": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "887": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "888": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "889": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "890": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "891": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "892": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "893": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "894": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "895": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "896": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "897": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "898": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "899": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "900": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "901": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "902": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "903": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "904": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "905": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "906": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "907": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "908": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "909": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "910": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "911": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "912": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "913": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "914": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "915": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "916": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "917": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "918": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "919": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "920": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "921": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "922": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "923": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "924": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "925": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "926": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "927": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "928": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "929": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "930": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "931": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "932": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "933": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "934": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "935": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "936": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "937": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "938": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "939": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "940": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "941": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "942": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "943": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "944": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "945": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "946": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "947": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "948": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "949": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "950": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "951": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "952": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "953": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "954": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "955": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "956": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "957": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "958": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "959": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "960": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "961": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "962": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "963": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "964": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "965": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "966": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "967": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "968": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "969": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "970": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "971": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "972": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "973": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "974": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "975": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "976": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "977": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "978": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "979": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "980": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "981": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "982": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "983": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "984": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "985": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "986": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "987": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "988": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "989": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "990": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "991": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "992": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "993": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "994": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "995": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "996": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "997": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "998": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "999": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>" + ], + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 131072, + "pad_token": "", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +}