Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

args.json

@@ -0,0 +1,374 @@
+{
+  "output_dir": "/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage2",
+  "per_device_train_batch_size": 2,
+  "num_train_epochs": 2.0,
+  "max_steps": -1,
+  "learning_rate": 1e-06,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "wandb"
+  ],
+  "run_name": "minicpmv46-2stage-2a800-z2-bs2-ga8-4096",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": null,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 2,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_on_each_node": false,
+  "save_total_limit": 2,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 4,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": false,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": "nccl",
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": {
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 2,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "allgather_partitions": true,
+      "allgather_bucket_size": 500000000.0,
+      "overlap_comm": true,
+      "reduce_scatter": true,
+      "reduce_bucket_size": 500000000.0,
+      "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.05,
+  "logging_dir": "/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage2/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage1/checkpoint-1928",
+  "model_type": "minicpmv4_6",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "sdpa",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "minicpmv4_6",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "is_binary_loss_scale": null,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "preserve_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/data/leizikun/msp/data/WorldVQA/worldvqa_swift_5x.jsonl",
+    "/data/leizikun/msp/script/occupy/data/train_neko_subset_15k.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "disable_auto_column_mapping": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "enable_npu_model_patch": true,
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": true,
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [
+    "model.vision_tower",
+    "model.merger"
+  ],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": false,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.3.0",
+  "ckpt_dir": "/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage1/checkpoint-1928",
+  "rank": 0,
+  "global_world_size": 2,
+  "local_world_size": 2,
+  "model_suffix": "checkpoint-1928",
+  "model_info": "ModelInfo(model_type='minicpmv4_6', model_dir='/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage1/checkpoint-1928', torch_dtype=torch.bfloat16, max_model_len=262144, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='minicpmv4_6', model_groups=[ModelGroup(models=[Model(ms_model_id='OpenBMB/MiniCPM-V-4.6', hf_model_id='openbmb/MiniCPM-V-4.6', model_path=None, ms_revision=None, hf_revision=None)], template=None, ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.models.minicpm.MiniCPMV4_6Loader'>, template='minicpmv4_6', model_arch=MultiModelKeys(arch_name='minicpmv4_6', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model'], aligner=['model.merger'], vision_tower=['model.vision_tower'], generator=[]), mcore_model_type=None, architectures=['MiniCPMV4_6ForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=5.7.0'], tags=['vision', 'video'])",
+  "model_dir": "/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage1/checkpoint-1928",
+  "template_meta": "ChatmlTemplateMeta(template_type='minicpmv4_6', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.templates.minicpm.MiniCPMV4_6Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system=None, auto_add_bos=True, stop_words=[], agent_template=None, is_thinking=True, thinking_prefix='<think>\\n', non_thinking_prefix='<think>\\n\\n</think>\\n\\n', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage2', per_device_train_batch_size=2, num_train_epochs=2.0, max_steps=-1, learning_rate=1e-06, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.05, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.01, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['wandb'], run_name='minicpmv46-2stage-2a800-z2-bs2-ga8-4096', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=None, eval_delay=0, per_device_eval_batch_size=2, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_on_each_node=False, save_total_limit=2, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=4, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend='nccl', ddp_timeout=18000000, fsdp=[], fsdp_config=None, deepspeed={'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 2, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'allgather_partitions': True, 'allgather_bucket_size': 500000000.0, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000.0, 'contiguous_gradients': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.05, logging_dir='/data/leizikun/msp/output/minicpmv46_neko_worldvqa_stage2/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

chat_template.jinja

@@ -0,0 +1,145 @@
+{%- if enable_thinking is not defined -%}
+    {%- set enable_thinking = true -%}
+{%- endif -%}
+{%- macro render_content(content, is_system_content=false) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- set ns = namespace(parts=[]) -%}
+        {%- for item in content -%}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain images.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}
+            {%- elif 'video' in item or item.type == 'video' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain videos.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}
+            {%- elif 'text' in item -%}
+                {%- set ns.parts = ns.parts + [item.text] -%}
+            {%- else -%}
+                {{- raise_exception('Unexpected item type in content.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- ns.parts | join('\n') -}}
+    {%- elif content is none or content is undefined -%}
+        {{- '' -}}
+    {%- else -%}
+        {{- raise_exception('Unexpected content type.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,110 @@
+{
+  "architectures": [
+    "MiniCPMV4_6ForConditionalGeneration"
+  ],
+  "bos_token_id": 248045,
+  "downsample_mode": "16x",
+  "drop_vision_last_layer": false,
+  "dtype": "bfloat16",
+  "eos_token_id": 248046,
+  "hidden_size": 1024,
+  "image_size": 1120,
+  "image_token_id": 248056,
+  "insert_layer_id": 6,
+  "merge_kernel_size": [
+    2,
+    2
+  ],
+  "merger_times": 1,
+  "model_type": "minicpmv4_6",
+  "pad_token_id": 248044,
+  "patch_size": 14,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": 248044,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "vocab_size": 248094
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "use_cache": false,
+  "video_token_id": 248057,
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 980,
+    "insert_layer_id": 6,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "minicpmv4_6_vision",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "window_kernel_size": [
+      2,
+      2
+    ]
+  }
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 248045,
+  "do_sample": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "repetition_penalty": 1.0,
+  "temperature": 0.7,
+  "top_k": 0,
+  "top_p": 1.0,
+  "transformers_version": "5.10.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3006aa273e5169a19c90258c9708ff9d4f8a38ffb584699ec675e930fdf50c4a
+size 3109054480

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "image_processor_type": "MiniCPMV4_6ImageProcessor",
+  "processor_class": "MiniCPMV4_6Processor",
+  "max_slice_nums": 9,
+  "scale_resolution": 448,
+  "patch_size": 14,
+  "use_image_id": true,
+  "slice_mode": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ]
+}

processor_config.json

@@ -0,0 +1,57 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "downsample_mode": "16x",
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "MiniCPMV4_6ImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_slice_nums": 9,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "scale_resolution": 448,
+    "slice_mode": true,
+    "use_image_id": true
+  },
+  "processor_class": "MiniCPMV4_6Processor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "downsample_mode": "16x",
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_num_frames": 128,
+    "max_slice_nums": 9,
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "scale_resolution": 448,
+    "slice_mode": true,
+    "stack_frames": 1,
+    "use_image_id": true,
+    "video_processor_type": "MiniCPMV4_6VideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33861e37bb955af1e3f3061182b820f347eba2b9c2c1011c82794bf0d6e77b54
+size 19992481

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_end_token": "</image>",
+  "image_id_end_token": "</image_id>",
+  "image_id_start_token": "<image_id>",
+  "image_start_token": "<image>",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_end_token": "</image>",
+    "image_id_end_token": "</image_id>",
+    "image_id_start_token": "<image_id>",
+    "image_start_token": "<image>",
+    "image_token": "<|image_pad|>",
+    "slice_end_token": "</slice>",
+    "slice_start_token": "<slice>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "MiniCPMV4_6Processor",
+  "slice_end_token": "</slice>",
+  "slice_start_token": "<slice>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": "<unk>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

trainer_state.json

@@ -0,0 +1,1538 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1876,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0010668089078543807,
+      "grad_norm": 7.006295680999756,
+      "learning_rate": 1.0638297872340425e-08,
+      "loss": 2.3478875160217285,
+      "step": 1,
+      "token_acc": 0.516892104435357
+    },
+    {
+      "epoch": 0.010668089078543806,
+      "grad_norm": 6.471454620361328,
+      "learning_rate": 1.0638297872340425e-07,
+      "loss": 2.4334648980034723,
+      "step": 10,
+      "token_acc": 0.5040707838310321
+    },
+    {
+      "epoch": 0.02133617815708761,
+      "grad_norm": 7.791952610015869,
+      "learning_rate": 2.127659574468085e-07,
+      "loss": 2.32786865234375,
+      "step": 20,
+      "token_acc": 0.5216088251677121
+    },
+    {
+      "epoch": 0.03200426723563142,
+      "grad_norm": 8.015645027160645,
+      "learning_rate": 3.1914893617021275e-07,
+      "loss": 2.419228363037109,
+      "step": 30,
+      "token_acc": 0.5090634923677535
+    },
+    {
+      "epoch": 0.04267235631417522,
+      "grad_norm": 5.430706024169922,
+      "learning_rate": 4.25531914893617e-07,
+      "loss": 2.4733400344848633,
+      "step": 40,
+      "token_acc": 0.4971268954509178
+    },
+    {
+      "epoch": 0.05334044539271903,
+      "grad_norm": 4.553371429443359,
+      "learning_rate": 5.319148936170212e-07,
+      "loss": 2.332263946533203,
+      "step": 50,
+      "token_acc": 0.513448871266867
+    },
+    {
+      "epoch": 0.06400853447126284,
+      "grad_norm": 4.459324359893799,
+      "learning_rate": 6.382978723404255e-07,
+      "loss": 2.2413455963134767,
+      "step": 60,
+      "token_acc": 0.5251403216723497
+    },
+    {
+      "epoch": 0.07467662354980664,
+      "grad_norm": 4.4227190017700195,
+      "learning_rate": 7.446808510638297e-07,
+      "loss": 2.2557914733886717,
+      "step": 70,
+      "token_acc": 0.5192819019893256
+    },
+    {
+      "epoch": 0.08534471262835044,
+      "grad_norm": 4.304351329803467,
+      "learning_rate": 8.51063829787234e-07,
+      "loss": 2.16949462890625,
+      "step": 80,
+      "token_acc": 0.5302786541144531
+    },
+    {
+      "epoch": 0.09601280170689425,
+      "grad_norm": 3.8063337802886963,
+      "learning_rate": 9.574468085106384e-07,
+      "loss": 2.1234577178955076,
+      "step": 90,
+      "token_acc": 0.5377233620119126
+    },
+    {
+      "epoch": 0.10668089078543806,
+      "grad_norm": 3.889915704727173,
+      "learning_rate": 9.999720280459574e-07,
+      "loss": 2.1470478057861326,
+      "step": 100,
+      "token_acc": 0.5297030687536047
+    },
+    {
+      "epoch": 0.11734897986398186,
+      "grad_norm": 4.103745460510254,
+      "learning_rate": 9.99801099660646e-07,
+      "loss": 2.091233825683594,
+      "step": 110,
+      "token_acc": 0.5380800430587059
+    },
+    {
+      "epoch": 0.12801706894252568,
+      "grad_norm": 4.0285964012146,
+      "learning_rate": 9.994748359236668e-07,
+      "loss": 2.1219104766845702,
+      "step": 120,
+      "token_acc": 0.5315024974893832
+    },
+    {
+      "epoch": 0.13868515802106948,
+      "grad_norm": 3.918626308441162,
+      "learning_rate": 9.989933382359422e-07,
+      "loss": 1.9738199234008789,
+      "step": 130,
+      "token_acc": 0.5553643663587514
+    },
+    {
+      "epoch": 0.14935324709961328,
+      "grad_norm": 3.5399599075317383,
+      "learning_rate": 9.983567562442268e-07,
+      "loss": 1.9851512908935547,
+      "step": 140,
+      "token_acc": 0.555815294562636
+    },
+    {
+      "epoch": 0.1600213361781571,
+      "grad_norm": 4.044088363647461,
+      "learning_rate": 9.97565287794599e-07,
+      "loss": 2.0619443893432616,
+      "step": 150,
+      "token_acc": 0.54069917888452
+    },
+    {
+      "epoch": 0.1706894252567009,
+      "grad_norm": 3.7363321781158447,
+      "learning_rate": 9.966191788709714e-07,
+      "loss": 2.010793685913086,
+      "step": 160,
+      "token_acc": 0.547274155538099
+    },
+    {
+      "epoch": 0.1813575143352447,
+      "grad_norm": 4.034898281097412,
+      "learning_rate": 9.955187235186403e-07,
+      "loss": 1.9935316085815429,
+      "step": 170,
+      "token_acc": 0.5503274190127947
+    },
+    {
+      "epoch": 0.1920256034137885,
+      "grad_norm": 4.587524890899658,
+      "learning_rate": 9.942642637528975e-07,
+      "loss": 2.047301483154297,
+      "step": 180,
+      "token_acc": 0.5417270064744885
+    },
+    {
+      "epoch": 0.20269369249233232,
+      "grad_norm": 3.7277448177337646,
+      "learning_rate": 9.928561894527351e-07,
+      "loss": 1.966663932800293,
+      "step": 190,
+      "token_acc": 0.5556000125035166
+    },
+    {
+      "epoch": 0.21336178157087612,
+      "grad_norm": 3.9594995975494385,
+      "learning_rate": 9.912949382396726e-07,
+      "loss": 1.9452442169189452,
+      "step": 200,
+      "token_acc": 0.5587059846947138
+    },
+    {
+      "epoch": 0.22402987064941993,
+      "grad_norm": 3.81302547454834,
+      "learning_rate": 9.895809953417462e-07,
+      "loss": 2.012272262573242,
+      "step": 210,
+      "token_acc": 0.5448511381289237
+    },
+    {
+      "epoch": 0.23469795972796373,
+      "grad_norm": 3.490966320037842,
+      "learning_rate": 9.877148934427035e-07,
+      "loss": 1.9165828704833985,
+      "step": 220,
+      "token_acc": 0.5619211368183942
+    },
+    {
+      "epoch": 0.24536604880650753,
+      "grad_norm": 3.570113182067871,
+      "learning_rate": 9.85697212516448e-07,
+      "loss": 1.9754508972167968,
+      "step": 230,
+      "token_acc": 0.5536200862349996
+    },
+    {
+      "epoch": 0.25603413788505136,
+      "grad_norm": 3.9454100131988525,
+      "learning_rate": 9.835285796467865e-07,
+      "loss": 1.9415111541748047,
+      "step": 240,
+      "token_acc": 0.5551580241745592
+    },
+    {
+      "epoch": 0.26670222696359513,
+      "grad_norm": 3.9396939277648926,
+      "learning_rate": 9.812096688325353e-07,
+      "loss": 1.8981182098388671,
+      "step": 250,
+      "token_acc": 0.5659307235382524
+    },
+    {
+      "epoch": 0.27737031604213896,
+      "grad_norm": 4.019906044006348,
+      "learning_rate": 9.787412007780444e-07,
+      "loss": 1.953832244873047,
+      "step": 260,
+      "token_acc": 0.5528575849796256
+    },
+    {
+      "epoch": 0.28803840512068274,
+      "grad_norm": 3.6888175010681152,
+      "learning_rate": 9.761239426692076e-07,
+      "loss": 1.9162483215332031,
+      "step": 270,
+      "token_acc": 0.564068832885431
+    },
+    {
+      "epoch": 0.29870649419922657,
+      "grad_norm": 3.746838092803955,
+      "learning_rate": 9.733587079350252e-07,
+      "loss": 1.910070037841797,
+      "step": 280,
+      "token_acc": 0.5619884448724122
+    },
+    {
+      "epoch": 0.30937458327777034,
+      "grad_norm": 3.9895079135894775,
+      "learning_rate": 9.704463559947942e-07,
+      "loss": 1.9012699127197266,
+      "step": 290,
+      "token_acc": 0.5634749160120394
+    },
+    {
+      "epoch": 0.3200426723563142,
+      "grad_norm": 4.03073263168335,
+      "learning_rate": 9.673877919910066e-07,
+      "loss": 1.9407228469848632,
+      "step": 300,
+      "token_acc": 0.556453737558073
+    },
+    {
+      "epoch": 0.330710761434858,
+      "grad_norm": 3.8840065002441406,
+      "learning_rate": 9.641839665080363e-07,
+      "loss": 1.8843194961547851,
+      "step": 310,
+      "token_acc": 0.5667126952619315
+    },
+    {
+      "epoch": 0.3413788505134018,
+      "grad_norm": 4.215559482574463,
+      "learning_rate": 9.60835875276701e-07,
+      "loss": 1.9429502487182617,
+      "step": 320,
+      "token_acc": 0.555164028621677
+    },
+    {
+      "epoch": 0.3520469395919456,
+      "grad_norm": 4.554783821105957,
+      "learning_rate": 9.573445588647978e-07,
+      "loss": 1.9644149780273437,
+      "step": 330,
+      "token_acc": 0.5522241856177013
+    },
+    {
+      "epoch": 0.3627150286704894,
+      "grad_norm": 4.137307643890381,
+      "learning_rate": 9.537111023536971e-07,
+      "loss": 1.9509822845458984,
+      "step": 340,
+      "token_acc": 0.5558621228192331
+    },
+    {
+      "epoch": 0.3733831177490332,
+      "grad_norm": 4.0477375984191895,
+      "learning_rate": 9.499366350011091e-07,
+      "loss": 1.871917724609375,
+      "step": 350,
+      "token_acc": 0.5673395880780275
+    },
+    {
+      "epoch": 0.384051206827577,
+      "grad_norm": 3.989922523498535,
+      "learning_rate": 9.460223298901138e-07,
+      "loss": 1.8953327178955077,
+      "step": 360,
+      "token_acc": 0.56321044546851
+    },
+    {
+      "epoch": 0.3947192959061208,
+      "grad_norm": 3.5498158931732178,
+      "learning_rate": 9.419694035645751e-07,
+      "loss": 1.8773033142089843,
+      "step": 370,
+      "token_acc": 0.5685798614498976
+    },
+    {
+      "epoch": 0.40538738498466464,
+      "grad_norm": 4.623654365539551,
+      "learning_rate": 9.377791156510454e-07,
+      "loss": 1.964752769470215,
+      "step": 380,
+      "token_acc": 0.552791747345048
+    },
+    {
+      "epoch": 0.4160554740632084,
+      "grad_norm": 3.953861951828003,
+      "learning_rate": 9.334527684672808e-07,
+      "loss": 1.9003120422363282,
+      "step": 390,
+      "token_acc": 0.5622615155074442
+    },
+    {
+      "epoch": 0.42672356314175225,
+      "grad_norm": 3.7582802772521973,
+      "learning_rate": 9.289917066174885e-07,
+      "loss": 1.859695053100586,
+      "step": 400,
+      "token_acc": 0.5688303235353716
+    },
+    {
+      "epoch": 0.437391652220296,
+      "grad_norm": 4.4737467765808105,
+      "learning_rate": 9.243973165744305e-07,
+      "loss": 1.9053627014160157,
+      "step": 410,
+      "token_acc": 0.5629376049605994
+    },
+    {
+      "epoch": 0.44805974129883985,
+      "grad_norm": 3.771303653717041,
+      "learning_rate": 9.196710262485168e-07,
+      "loss": 1.896707534790039,
+      "step": 420,
+      "token_acc": 0.5621554723462294
+    },
+    {
+      "epoch": 0.4587278303773836,
+      "grad_norm": 3.9137330055236816,
+      "learning_rate": 9.148143045440179e-07,
+      "loss": 1.852376365661621,
+      "step": 430,
+      "token_acc": 0.5686185246305886
+    },
+    {
+      "epoch": 0.46939591945592746,
+      "grad_norm": 3.798518180847168,
+      "learning_rate": 9.098286609025391e-07,
+      "loss": 1.865086555480957,
+      "step": 440,
+      "token_acc": 0.5662914941616878
+    },
+    {
+      "epoch": 0.4800640085344713,
+      "grad_norm": 4.035417556762695,
+      "learning_rate": 9.047156448338926e-07,
+      "loss": 1.8744647979736329,
+      "step": 450,
+      "token_acc": 0.5680482591436246
+    },
+    {
+      "epoch": 0.49073209761301506,
+      "grad_norm": 4.174037933349609,
+      "learning_rate": 8.994768454345205e-07,
+      "loss": 1.8902172088623046,
+      "step": 460,
+      "token_acc": 0.5636073504641532
+    },
+    {
+      "epoch": 0.5014001866915588,
+      "grad_norm": 4.3247575759887695,
+      "learning_rate": 8.941138908936116e-07,
+      "loss": 1.8696575164794922,
+      "step": 470,
+      "token_acc": 0.5677604970719198
+    },
+    {
+      "epoch": 0.5120682757701027,
+      "grad_norm": 3.8726143836975098,
+      "learning_rate": 8.886284479870698e-07,
+      "loss": 1.838299560546875,
+      "step": 480,
+      "token_acc": 0.5764252373856177
+    },
+    {
+      "epoch": 0.5227363648486465,
+      "grad_norm": 4.121581554412842,
+      "learning_rate": 8.83022221559489e-07,
+      "loss": 1.8638439178466797,
+      "step": 490,
+      "token_acc": 0.5698095928568703
+    },
+    {
+      "epoch": 0.5334044539271903,
+      "grad_norm": 4.610175132751465,
+      "learning_rate": 8.77296953994298e-07,
+      "loss": 1.858209991455078,
+      "step": 500,
+      "token_acc": 0.5696208987002059
+    },
+    {
+      "epoch": 0.544072543005734,
+      "grad_norm": 4.455285549163818,
+      "learning_rate": 8.714544246722369e-07,
+      "loss": 1.8948783874511719,
+      "step": 510,
+      "token_acc": 0.5608375235097686
+    },
+    {
+      "epoch": 0.5547406320842779,
+      "grad_norm": 4.052758693695068,
+      "learning_rate": 8.654964494183358e-07,
+      "loss": 1.856728744506836,
+      "step": 520,
+      "token_acc": 0.5684638641855052
+    },
+    {
+      "epoch": 0.5654087211628217,
+      "grad_norm": 4.3134660720825195,
+      "learning_rate": 8.594248799375669e-07,
+      "loss": 1.8250436782836914,
+      "step": 530,
+      "token_acc": 0.5724758520722962
+    },
+    {
+      "epoch": 0.5760768102413655,
+      "grad_norm": 3.7139341831207275,
+      "learning_rate": 8.532416032393447e-07,
+      "loss": 1.8022869110107422,
+      "step": 540,
+      "token_acc": 0.57848322291492
+    },
+    {
+      "epoch": 0.5867448993199094,
+      "grad_norm": 3.976836681365967,
+      "learning_rate": 8.469485410510544e-07,
+      "loss": 1.8598838806152345,
+      "step": 550,
+      "token_acc": 0.5676960075729471
+    },
+    {
+      "epoch": 0.5974129883984531,
+      "grad_norm": 4.145941257476807,
+      "learning_rate": 8.4054764922079e-07,
+      "loss": 1.8461755752563476,
+      "step": 560,
+      "token_acc": 0.5693920927753626
+    },
+    {
+      "epoch": 0.6080810774769969,
+      "grad_norm": 4.197259426116943,
+      "learning_rate": 8.340409171094873e-07,
+      "loss": 1.8348304748535156,
+      "step": 570,
+      "token_acc": 0.5728334165834166
+    },
+    {
+      "epoch": 0.6187491665555407,
+      "grad_norm": 4.145263671875,
+      "learning_rate": 8.274303669726426e-07,
+      "loss": 1.8206695556640624,
+      "step": 580,
+      "token_acc": 0.5718382929363102
+    },
+    {
+      "epoch": 0.6294172556340846,
+      "grad_norm": 3.7828235626220703,
+      "learning_rate": 8.207180533318061e-07,
+      "loss": 1.8166313171386719,
+      "step": 590,
+      "token_acc": 0.5745853534064812
+    },
+    {
+      "epoch": 0.6400853447126283,
+      "grad_norm": 4.152252197265625,
+      "learning_rate": 8.139060623360493e-07,
+      "loss": 1.8334476470947265,
+      "step": 600,
+      "token_acc": 0.5736091696952754
+    },
+    {
+      "epoch": 0.6507534337911721,
+      "grad_norm": 3.9253435134887695,
+      "learning_rate": 8.069965111136009e-07,
+      "loss": 1.8342958450317384,
+      "step": 610,
+      "token_acc": 0.572730870797999
+    },
+    {
+      "epoch": 0.661421522869716,
+      "grad_norm": 3.783778429031372,
+      "learning_rate": 7.999915471138561e-07,
+      "loss": 1.8457708358764648,
+      "step": 620,
+      "token_acc": 0.5711007991750451
+    },
+    {
+      "epoch": 0.6720896119482598,
+      "grad_norm": 4.500645160675049,
+      "learning_rate": 7.928933474399601e-07,
+      "loss": 1.8366958618164062,
+      "step": 630,
+      "token_acc": 0.5711011556940657
+    },
+    {
+      "epoch": 0.6827577010268036,
+      "grad_norm": 4.189162254333496,
+      "learning_rate": 7.857041181721787e-07,
+      "loss": 1.8609766006469726,
+      "step": 640,
+      "token_acc": 0.5636673978790514
+    },
+    {
+      "epoch": 0.6934257901053473,
+      "grad_norm": 4.374728679656982,
+      "learning_rate": 7.784260936822591e-07,
+      "loss": 1.863561248779297,
+      "step": 650,
+      "token_acc": 0.5696042614986878
+    },
+    {
+      "epoch": 0.7040938791838912,
+      "grad_norm": 4.001153945922852,
+      "learning_rate": 7.710615359390017e-07,
+      "loss": 1.8359176635742187,
+      "step": 660,
+      "token_acc": 0.5730398306910491
+    },
+    {
+      "epoch": 0.714761968262435,
+      "grad_norm": 3.9148805141448975,
+      "learning_rate": 7.636127338052511e-07,
+      "loss": 1.8227685928344726,
+      "step": 670,
+      "token_acc": 0.5748739871115932
+    },
+    {
+      "epoch": 0.7254300573409788,
+      "grad_norm": 4.053929805755615,
+      "learning_rate": 7.560820023265325e-07,
+      "loss": 1.856583023071289,
+      "step": 680,
+      "token_acc": 0.5661723369196472
+    },
+    {
+      "epoch": 0.7360981464195226,
+      "grad_norm": 4.32113790512085,
+      "learning_rate": 7.48471682011546e-07,
+      "loss": 1.8252262115478515,
+      "step": 690,
+      "token_acc": 0.5741169732068987
+    },
+    {
+      "epoch": 0.7467662354980664,
+      "grad_norm": 3.9224932193756104,
+      "learning_rate": 7.407841381047531e-07,
+      "loss": 1.8305496215820312,
+      "step": 700,
+      "token_acc": 0.5729056069568734
+    },
+    {
+      "epoch": 0.7574343245766102,
+      "grad_norm": 4.077065467834473,
+      "learning_rate": 7.330217598512694e-07,
+      "loss": 1.8242149353027344,
+      "step": 710,
+      "token_acc": 0.5721009316067469
+    },
+    {
+      "epoch": 0.768102413655154,
+      "grad_norm": 4.011868000030518,
+      "learning_rate": 7.251869597543018e-07,
+      "loss": 1.806910514831543,
+      "step": 720,
+      "token_acc": 0.575511092893375
+    },
+    {
+      "epoch": 0.7787705027336979,
+      "grad_norm": 4.1462273597717285,
+      "learning_rate": 7.172821728253562e-07,
+      "loss": 1.8008831024169922,
+      "step": 730,
+      "token_acc": 0.5784865805469288
+    },
+    {
+      "epoch": 0.7894385918122416,
+      "grad_norm": 4.203607559204102,
+      "learning_rate": 7.093098558274493e-07,
+      "loss": 1.8120304107666017,
+      "step": 740,
+      "token_acc": 0.5752062416064463
+    },
+    {
+      "epoch": 0.8001066808907854,
+      "grad_norm": 4.279056549072266,
+      "learning_rate": 7.012724865115613e-07,
+      "loss": 1.8188220977783203,
+      "step": 750,
+      "token_acc": 0.5747139588100687
+    },
+    {
+      "epoch": 0.8107747699693293,
+      "grad_norm": 3.8039934635162354,
+      "learning_rate": 6.931725628465642e-07,
+      "loss": 1.8142536163330079,
+      "step": 760,
+      "token_acc": 0.5760542312466185
+    },
+    {
+      "epoch": 0.8214428590478731,
+      "grad_norm": 3.7880845069885254,
+      "learning_rate": 6.850126022428677e-07,
+      "loss": 1.7894207000732423,
+      "step": 770,
+      "token_acc": 0.5769731452847082
+    },
+    {
+      "epoch": 0.8321109481264168,
+      "grad_norm": 4.270044326782227,
+      "learning_rate": 6.767951407700216e-07,
+      "loss": 1.8551364898681642,
+      "step": 780,
+      "token_acc": 0.5668973471741637
+    },
+    {
+      "epoch": 0.8427790372049606,
+      "grad_norm": 3.97334623336792,
+      "learning_rate": 6.685227323685209e-07,
+      "loss": 1.8029193878173828,
+      "step": 790,
+      "token_acc": 0.5781002423703444
+    },
+    {
+      "epoch": 0.8534471262835045,
+      "grad_norm": 4.129730224609375,
+      "learning_rate": 6.601979480560542e-07,
+      "loss": 1.8175867080688477,
+      "step": 800,
+      "token_acc": 0.5746865306479094
+    },
+    {
+      "epoch": 0.8641152153620483,
+      "grad_norm": 3.9668161869049072,
+      "learning_rate": 6.518233751284472e-07,
+      "loss": 1.7865169525146485,
+      "step": 810,
+      "token_acc": 0.5799188730546251
+    },
+    {
+      "epoch": 0.874783304440592,
+      "grad_norm": 4.027299880981445,
+      "learning_rate": 6.434016163555451e-07,
+      "loss": 1.7904687881469727,
+      "step": 820,
+      "token_acc": 0.5808848869923743
+    },
+    {
+      "epoch": 0.8854513935191359,
+      "grad_norm": 4.224595069885254,
+      "learning_rate": 6.349352891722866e-07,
+      "loss": 1.8047224044799806,
+      "step": 830,
+      "token_acc": 0.575895102159917
+    },
+    {
+      "epoch": 0.8961194825976797,
+      "grad_norm": 3.965972661972046,
+      "learning_rate": 6.264270248652198e-07,
+      "loss": 1.8042530059814452,
+      "step": 840,
+      "token_acc": 0.5774797143684499
+    },
+    {
+      "epoch": 0.9067875716762235,
+      "grad_norm": 3.985602855682373,
+      "learning_rate": 6.178794677547137e-07,
+      "loss": 1.847981834411621,
+      "step": 850,
+      "token_acc": 0.5686120808905869
+    },
+    {
+      "epoch": 0.9174556607547673,
+      "grad_norm": 3.7282555103302,
+      "learning_rate": 6.092952743731178e-07,
+      "loss": 1.7887779235839845,
+      "step": 860,
+      "token_acc": 0.5776786905238286
+    },
+    {
+      "epoch": 0.9281237498333111,
+      "grad_norm": 4.324705600738525,
+      "learning_rate": 6.006771126391277e-07,
+      "loss": 1.806238555908203,
+      "step": 870,
+      "token_acc": 0.575960881693935
+    },
+    {
+      "epoch": 0.9387918389118549,
+      "grad_norm": 4.746899604797363,
+      "learning_rate": 5.920276610286101e-07,
+      "loss": 1.797719955444336,
+      "step": 880,
+      "token_acc": 0.575878918644498
+    },
+    {
+      "epoch": 0.9494599279903987,
+      "grad_norm": 4.021754741668701,
+      "learning_rate": 5.833496077421484e-07,
+      "loss": 1.7537151336669923,
+      "step": 890,
+      "token_acc": 0.5848000196239559
+    },
+    {
+      "epoch": 0.9601280170689426,
+      "grad_norm": 3.837305784225464,
+      "learning_rate": 5.746456498695647e-07,
+      "loss": 1.7645259857177735,
+      "step": 900,
+      "token_acc": 0.5852038155997914
+    },
+    {
+      "epoch": 0.9707961061474863,
+      "grad_norm": 4.20936393737793,
+      "learning_rate": 5.659184925516801e-07,
+      "loss": 1.8052492141723633,
+      "step": 910,
+      "token_acc": 0.5749897032576932
+    },
+    {
+      "epoch": 0.9814641952260301,
+      "grad_norm": 3.9602720737457275,
+      "learning_rate": 5.571708481395718e-07,
+      "loss": 1.7849920272827149,
+      "step": 920,
+      "token_acc": 0.5777625975982686
+    },
+    {
+      "epoch": 0.9921322843045739,
+      "grad_norm": 4.546433925628662,
+      "learning_rate": 5.484054353515895e-07,
+      "loss": 1.7990896224975585,
+      "step": 930,
+      "token_acc": 0.5758030425592986
+    },
+    {
+      "epoch": 1.0021336178157088,
+      "grad_norm": 4.241868019104004,
+      "learning_rate": 5.396249784283942e-07,
+      "loss": 1.7917800903320313,
+      "step": 940,
+      "token_acc": 0.575703485206652
+    },
+    {
+      "epoch": 1.0128017068942525,
+      "grad_norm": 4.015085697174072,
+      "learning_rate": 5.308322062862786e-07,
+      "loss": 1.7706895828247071,
+      "step": 950,
+      "token_acc": 0.5819618553298342
+    },
+    {
+      "epoch": 1.0234697959727963,
+      "grad_norm": 3.8207788467407227,
+      "learning_rate": 5.220298516690352e-07,
+      "loss": 1.7699962615966798,
+      "step": 960,
+      "token_acc": 0.5831101297821492
+    },
+    {
+      "epoch": 1.03413788505134,
+      "grad_norm": 4.0016913414001465,
+      "learning_rate": 5.132206502986367e-07,
+      "loss": 1.7162120819091797,
+      "step": 970,
+      "token_acc": 0.5933027241351231
+    },
+    {
+      "epoch": 1.044805974129884,
+      "grad_norm": 4.224870204925537,
+      "learning_rate": 5.044073400249867e-07,
+      "loss": 1.7514739990234376,
+      "step": 980,
+      "token_acc": 0.5842004937345708
+    },
+    {
+      "epoch": 1.0554740632084278,
+      "grad_norm": 4.110172748565674,
+      "learning_rate": 4.955926599750133e-07,
+      "loss": 1.769965362548828,
+      "step": 990,
+      "token_acc": 0.5810418163437879
+    },
+    {
+      "epoch": 1.0661421522869716,
+      "grad_norm": 4.555731773376465,
+      "learning_rate": 4.867793497013633e-07,
+      "loss": 1.7587167739868164,
+      "step": 1000,
+      "token_acc": 0.5840167709557539
+    },
+    {
+      "epoch": 1.0768102413655154,
+      "grad_norm": 4.186336994171143,
+      "learning_rate": 4.779701483309647e-07,
+      "loss": 1.782842254638672,
+      "step": 1010,
+      "token_acc": 0.5791882975730762
+    },
+    {
+      "epoch": 1.0874783304440592,
+      "grad_norm": 4.142239093780518,
+      "learning_rate": 4.691677937137216e-07,
+      "loss": 1.7737754821777343,
+      "step": 1020,
+      "token_acc": 0.5780588477311638
+    },
+    {
+      "epoch": 1.098146419522603,
+      "grad_norm": 4.100138187408447,
+      "learning_rate": 4.6037502157160567e-07,
+      "loss": 1.7566829681396485,
+      "step": 1030,
+      "token_acc": 0.5844300829849783
+    },
+    {
+      "epoch": 1.1088145086011467,
+      "grad_norm": 4.242199420928955,
+      "learning_rate": 4.5159456464841047e-07,
+      "loss": 1.7504650115966798,
+      "step": 1040,
+      "token_acc": 0.5846727898966705
+    },
+    {
+      "epoch": 1.1194825976796907,
+      "grad_norm": 4.044170379638672,
+      "learning_rate": 4.4282915186042824e-07,
+      "loss": 1.7210964202880858,
+      "step": 1050,
+      "token_acc": 0.5922978224891682
+    },
+    {
+      "epoch": 1.1301506867582345,
+      "grad_norm": 4.421886920928955,
+      "learning_rate": 4.340815074483198e-07,
+      "loss": 1.7688575744628907,
+      "step": 1060,
+      "token_acc": 0.5833048064610196
+    },
+    {
+      "epoch": 1.1408187758367783,
+      "grad_norm": 3.845952272415161,
+      "learning_rate": 4.253543501304353e-07,
+      "loss": 1.7529470443725585,
+      "step": 1070,
+      "token_acc": 0.5855110605760571
+    },
+    {
+      "epoch": 1.151486864915322,
+      "grad_norm": 4.058477878570557,
+      "learning_rate": 4.1665039225785155e-07,
+      "loss": 1.7334306716918946,
+      "step": 1080,
+      "token_acc": 0.5878252170224082
+    },
+    {
+      "epoch": 1.1621549539938658,
+      "grad_norm": 4.219770908355713,
+      "learning_rate": 4.0797233897138985e-07,
+      "loss": 1.760676956176758,
+      "step": 1090,
+      "token_acc": 0.5842919090153719
+    },
+    {
+      "epoch": 1.1728230430724096,
+      "grad_norm": 4.5863938331604,
+      "learning_rate": 3.9932288736087235e-07,
+      "loss": 1.7337118148803712,
+      "step": 1100,
+      "token_acc": 0.5892282560269294
+    },
+    {
+      "epoch": 1.1834911321509534,
+      "grad_norm": 4.078279972076416,
+      "learning_rate": 3.9070472562688216e-07,
+      "loss": 1.7463794708251954,
+      "step": 1110,
+      "token_acc": 0.5860925705080576
+    },
+    {
+      "epoch": 1.1941592212294974,
+      "grad_norm": 4.5078229904174805,
+      "learning_rate": 3.821205322452863e-07,
+      "loss": 1.7371578216552734,
+      "step": 1120,
+      "token_acc": 0.5872009522946938
+    },
+    {
+      "epoch": 1.2048273103080411,
+      "grad_norm": 4.2302021980285645,
+      "learning_rate": 3.735729751347803e-07,
+      "loss": 1.7652847290039062,
+      "step": 1130,
+      "token_acc": 0.582140173048388
+    },
+    {
+      "epoch": 1.215495399386585,
+      "grad_norm": 4.255001068115234,
+      "learning_rate": 3.650647108277135e-07,
+      "loss": 1.7338531494140625,
+      "step": 1140,
+      "token_acc": 0.5879334364335834
+    },
+    {
+      "epoch": 1.2261634884651287,
+      "grad_norm": 3.864320755004883,
+      "learning_rate": 3.56598383644455e-07,
+      "loss": 1.7408796310424806,
+      "step": 1150,
+      "token_acc": 0.5880632023098545
+    },
+    {
+      "epoch": 1.2368315775436725,
+      "grad_norm": 4.506700038909912,
+      "learning_rate": 3.4817662487155277e-07,
+      "loss": 1.7883380889892577,
+      "step": 1160,
+      "token_acc": 0.5781155518213702
+    },
+    {
+      "epoch": 1.2474996666222162,
+      "grad_norm": 4.539206504821777,
+      "learning_rate": 3.3980205194394584e-07,
+      "loss": 1.8089128494262696,
+      "step": 1170,
+      "token_acc": 0.5740162863583134
+    },
+    {
+      "epoch": 1.25816775570076,
+      "grad_norm": 4.161843776702881,
+      "learning_rate": 3.314772676314791e-07,
+      "loss": 1.676759910583496,
+      "step": 1180,
+      "token_acc": 0.597660963460289
+    },
+    {
+      "epoch": 1.268835844779304,
+      "grad_norm": 4.23718786239624,
+      "learning_rate": 3.2320485922997837e-07,
+      "loss": 1.7459901809692382,
+      "step": 1190,
+      "token_acc": 0.5877557574372688
+    },
+    {
+      "epoch": 1.2795039338578478,
+      "grad_norm": 4.368854522705078,
+      "learning_rate": 3.149873977571324e-07,
+      "loss": 1.7589942932128906,
+      "step": 1200,
+      "token_acc": 0.5836031026646863
+    },
+    {
+      "epoch": 1.2901720229363915,
+      "grad_norm": 3.9527323246002197,
+      "learning_rate": 3.068274371534356e-07,
+      "loss": 1.7633914947509766,
+      "step": 1210,
+      "token_acc": 0.5808518610248496
+    },
+    {
+      "epoch": 1.3008401120149353,
+      "grad_norm": 4.670387268066406,
+      "learning_rate": 2.987275134884387e-07,
+      "loss": 1.7531061172485352,
+      "step": 1220,
+      "token_acc": 0.583694726370747
+    },
+    {
+      "epoch": 1.311508201093479,
+      "grad_norm": 4.837648391723633,
+      "learning_rate": 2.906901441725507e-07,
+      "loss": 1.7216800689697265,
+      "step": 1230,
+      "token_acc": 0.5906113881201974
+    },
+    {
+      "epoch": 1.3221762901720229,
+      "grad_norm": 4.389393329620361,
+      "learning_rate": 2.827178271746441e-07,
+      "loss": 1.762483787536621,
+      "step": 1240,
+      "token_acc": 0.5833013298270509
+    },
+    {
+      "epoch": 1.3328443792505666,
+      "grad_norm": 4.803798198699951,
+      "learning_rate": 2.748130402456982e-07,
+      "loss": 1.7307064056396484,
+      "step": 1250,
+      "token_acc": 0.589897529389577
+    },
+    {
+      "epoch": 1.3435124683291106,
+      "grad_norm": 4.382832050323486,
+      "learning_rate": 2.669782401487307e-07,
+      "loss": 1.7516584396362305,
+      "step": 1260,
+      "token_acc": 0.5837193656895723
+    },
+    {
+      "epoch": 1.3541805574076544,
+      "grad_norm": 4.203041076660156,
+      "learning_rate": 2.592158618952469e-07,
+      "loss": 1.7157066345214844,
+      "step": 1270,
+      "token_acc": 0.58897627643115
+    },
+    {
+      "epoch": 1.3648486464861982,
+      "grad_norm": 3.8712828159332275,
+      "learning_rate": 2.5152831798845395e-07,
+      "loss": 1.7508321762084962,
+      "step": 1280,
+      "token_acc": 0.5845150172328902
+    },
+    {
+      "epoch": 1.375516735564742,
+      "grad_norm": 4.0551862716674805,
+      "learning_rate": 2.4391799767346765e-07,
+      "loss": 1.7275760650634766,
+      "step": 1290,
+      "token_acc": 0.5885003490512153
+    },
+    {
+      "epoch": 1.3861848246432857,
+      "grad_norm": 4.279505252838135,
+      "learning_rate": 2.3638726619474875e-07,
+      "loss": 1.7430885314941407,
+      "step": 1300,
+      "token_acc": 0.5865010456936435
+    },
+    {
+      "epoch": 1.3968529137218295,
+      "grad_norm": 4.156595230102539,
+      "learning_rate": 2.2893846406099841e-07,
+      "loss": 1.722519302368164,
+      "step": 1310,
+      "token_acc": 0.5894925388356631
+    },
+    {
+      "epoch": 1.4075210028003733,
+      "grad_norm": 4.309932231903076,
+      "learning_rate": 2.2157390631774088e-07,
+      "loss": 1.7328716278076173,
+      "step": 1320,
+      "token_acc": 0.5868015297004017
+    },
+    {
+      "epoch": 1.4181890918789173,
+      "grad_norm": 4.3499345779418945,
+      "learning_rate": 2.1429588182782144e-07,
+      "loss": 1.7189838409423828,
+      "step": 1330,
+      "token_acc": 0.5921618458033138
+    },
+    {
+      "epoch": 1.428857180957461,
+      "grad_norm": 4.327865123748779,
+      "learning_rate": 2.071066525600399e-07,
+      "loss": 1.7395309448242187,
+      "step": 1340,
+      "token_acc": 0.5846225647558922
+    },
+    {
+      "epoch": 1.4395252700360048,
+      "grad_norm": 4.3813371658325195,
+      "learning_rate": 2.0000845288614392e-07,
+      "loss": 1.7395313262939454,
+      "step": 1350,
+      "token_acc": 0.5878291566355234
+    },
+    {
+      "epoch": 1.4501933591145486,
+      "grad_norm": 4.583224296569824,
+      "learning_rate": 1.9300348888639912e-07,
+      "loss": 1.7289115905761718,
+      "step": 1360,
+      "token_acc": 0.5864353964908652
+    },
+    {
+      "epoch": 1.4608614481930924,
+      "grad_norm": 4.292775630950928,
+      "learning_rate": 1.860939376639508e-07,
+      "loss": 1.6974550247192384,
+      "step": 1370,
+      "token_acc": 0.5946563716027821
+    },
+    {
+      "epoch": 1.4715295372716362,
+      "grad_norm": 4.551062107086182,
+      "learning_rate": 1.7928194666819395e-07,
+      "loss": 1.7184766769409179,
+      "step": 1380,
+      "token_acc": 0.5937895103314745
+    },
+    {
+      "epoch": 1.48219762635018,
+      "grad_norm": 3.984744071960449,
+      "learning_rate": 1.725696330273575e-07,
+      "loss": 1.712306594848633,
+      "step": 1390,
+      "token_acc": 0.5913718280618817
+    },
+    {
+      "epoch": 1.492865715428724,
+      "grad_norm": 4.165930271148682,
+      "learning_rate": 1.6595908289051265e-07,
+      "loss": 1.7679101943969726,
+      "step": 1400,
+      "token_acc": 0.5828852512501334
+    },
+    {
+      "epoch": 1.5035338045072675,
+      "grad_norm": 4.256524085998535,
+      "learning_rate": 1.594523507792101e-07,
+      "loss": 1.695474624633789,
+      "step": 1410,
+      "token_acc": 0.5945638762771305
+    },
+    {
+      "epoch": 1.5142018935858115,
+      "grad_norm": 3.5626895427703857,
+      "learning_rate": 1.5305145894894544e-07,
+      "loss": 1.742639923095703,
+      "step": 1420,
+      "token_acc": 0.5851085284997605
+    },
+    {
+      "epoch": 1.5248699826643553,
+      "grad_norm": 4.088000774383545,
+      "learning_rate": 1.467583967606553e-07,
+      "loss": 1.7252666473388671,
+      "step": 1430,
+      "token_acc": 0.5911103426556585
+    },
+    {
+      "epoch": 1.535538071742899,
+      "grad_norm": 4.023231029510498,
+      "learning_rate": 1.4057512006243312e-07,
+      "loss": 1.7037853240966796,
+      "step": 1440,
+      "token_acc": 0.5923428756589688
+    },
+    {
+      "epoch": 1.546206160821443,
+      "grad_norm": 4.324033737182617,
+      "learning_rate": 1.3450355058166418e-07,
+      "loss": 1.7532779693603515,
+      "step": 1450,
+      "token_acc": 0.5835899663927827
+    },
+    {
+      "epoch": 1.5568742498999866,
+      "grad_norm": 4.487975597381592,
+      "learning_rate": 1.2854557532776322e-07,
+      "loss": 1.7331933975219727,
+      "step": 1460,
+      "token_acc": 0.5876274808378866
+    },
+    {
+      "epoch": 1.5675423389785306,
+      "grad_norm": 4.267901420593262,
+      "learning_rate": 1.2270304600570193e-07,
+      "loss": 1.7457307815551757,
+      "step": 1470,
+      "token_acc": 0.5854581981146655
+    },
+    {
+      "epoch": 1.5782104280570741,
+      "grad_norm": 4.140320777893066,
+      "learning_rate": 1.1697777844051104e-07,
+      "loss": 1.7602937698364258,
+      "step": 1480,
+      "token_acc": 0.5820569483603851
+    },
+    {
+      "epoch": 1.5888785171356181,
+      "grad_norm": 4.402993202209473,
+      "learning_rate": 1.113715520129302e-07,
+      "loss": 1.768077278137207,
+      "step": 1490,
+      "token_acc": 0.5807759946527882
+    },
+    {
+      "epoch": 1.599546606214162,
+      "grad_norm": 4.2369842529296875,
+      "learning_rate": 1.0588610910638824e-07,
+      "loss": 1.7579662322998046,
+      "step": 1500,
+      "token_acc": 0.5823074645524229
+    },
+    {
+      "epoch": 1.6102146952927057,
+      "grad_norm": 4.478653430938721,
+      "learning_rate": 1.0052315456547932e-07,
+      "loss": 1.7446666717529298,
+      "step": 1510,
+      "token_acc": 0.5843921670005816
+    },
+    {
+      "epoch": 1.6208827843712497,
+      "grad_norm": 4.06418514251709,
+      "learning_rate": 9.528435516610728e-08,
+      "loss": 1.760988998413086,
+      "step": 1520,
+      "token_acc": 0.5854237477708114
+    },
+    {
+      "epoch": 1.6315508734497932,
+      "grad_norm": 4.865668296813965,
+      "learning_rate": 9.017133909746094e-08,
+      "loss": 1.7328973770141602,
+      "step": 1530,
+      "token_acc": 0.5875920698270547
+    },
+    {
+      "epoch": 1.6422189625283372,
+      "grad_norm": 4.391946315765381,
+      "learning_rate": 8.518569545598197e-08,
+      "loss": 1.7731132507324219,
+      "step": 1540,
+      "token_acc": 0.578755576304596
+    },
+    {
+      "epoch": 1.6528870516068808,
+      "grad_norm": 4.87280797958374,
+      "learning_rate": 8.032897375148324e-08,
+      "loss": 1.739859390258789,
+      "step": 1550,
+      "token_acc": 0.5865034392506495
+    },
+    {
+      "epoch": 1.6635551406854248,
+      "grad_norm": 3.9779059886932373,
+      "learning_rate": 7.560268342556946e-08,
+      "loss": 1.7412452697753906,
+      "step": 1560,
+      "token_acc": 0.5860936113436255
+    },
+    {
+      "epoch": 1.6742232297639685,
+      "grad_norm": 4.201171398162842,
+      "learning_rate": 7.100829338251146e-08,
+      "loss": 1.7185377120971679,
+      "step": 1570,
+      "token_acc": 0.5900551731842189
+    },
+    {
+      "epoch": 1.6848913188425123,
+      "grad_norm": 4.249425888061523,
+      "learning_rate": 6.654723153271912e-08,
+      "loss": 1.7351497650146483,
+      "step": 1580,
+      "token_acc": 0.5861808195187964
+    },
+    {
+      "epoch": 1.6955594079210563,
+      "grad_norm": 4.077160835266113,
+      "learning_rate": 6.222088434895461e-08,
+      "loss": 1.7461994171142579,
+      "step": 1590,
+      "token_acc": 0.5874196325536241
+    },
+    {
+      "epoch": 1.7062274969995999,
+      "grad_norm": 4.179897308349609,
+      "learning_rate": 5.8030596435424905e-08,
+      "loss": 1.7850582122802734,
+      "step": 1600,
+      "token_acc": 0.5798739362151004
+    },
+    {
+      "epoch": 1.7168955860781439,
+      "grad_norm": 4.09893798828125,
+      "learning_rate": 5.397767010988613e-08,
+      "loss": 1.7305503845214845,
+      "step": 1610,
+      "token_acc": 0.5885713556195303
+    },
+    {
+      "epoch": 1.7275636751566874,
+      "grad_norm": 4.29001522064209,
+      "learning_rate": 5.006336499889075e-08,
+      "loss": 1.737938117980957,
+      "step": 1620,
+      "token_acc": 0.5866698573029887
+    },
+    {
+      "epoch": 1.7382317642352314,
+      "grad_norm": 4.288162708282471,
+      "learning_rate": 4.6288897646302784e-08,
+      "loss": 1.701319122314453,
+      "step": 1630,
+      "token_acc": 0.5959787699032157
+    },
+    {
+      "epoch": 1.7488998533137752,
+      "grad_norm": 4.345064640045166,
+      "learning_rate": 4.26554411352022e-08,
+      "loss": 1.6974868774414062,
+      "step": 1640,
+      "token_acc": 0.5931039293777428
+    },
+    {
+      "epoch": 1.759567942392319,
+      "grad_norm": 3.9936347007751465,
+      "learning_rate": 3.916412472329883e-08,
+      "loss": 1.7536365509033203,
+      "step": 1650,
+      "token_acc": 0.583680046326311
+    },
+    {
+      "epoch": 1.770236031470863,
+      "grad_norm": 4.050926685333252,
+      "learning_rate": 3.581603349196371e-08,
+      "loss": 1.7840970993041991,
+      "step": 1660,
+      "token_acc": 0.578690899512709
+    },
+    {
+      "epoch": 1.7809041205494065,
+      "grad_norm": 4.2205491065979,
+      "learning_rate": 3.261220800899323e-08,
+      "loss": 1.7670732498168946,
+      "step": 1670,
+      "token_acc": 0.5830930903003813
+    },
+    {
+      "epoch": 1.7915722096279505,
+      "grad_norm": 4.417363166809082,
+      "learning_rate": 2.9553644005205825e-08,
+      "loss": 1.7118240356445313,
+      "step": 1680,
+      "token_acc": 0.5923614225422198
+    },
+    {
+      "epoch": 1.802240298706494,
+      "grad_norm": 4.421960830688477,
+      "learning_rate": 2.6641292064974786e-08,
+      "loss": 1.7611549377441407,
+      "step": 1690,
+      "token_acc": 0.5802039615357256
+    },
+    {
+      "epoch": 1.812908387785038,
+      "grad_norm": 3.866654396057129,
+      "learning_rate": 2.3876057330792343e-08,
+      "loss": 1.723464584350586,
+      "step": 1700,
+      "token_acc": 0.589326291223976
+    },
+    {
+      "epoch": 1.8235764768635818,
+      "grad_norm": 4.226998805999756,
+      "learning_rate": 2.1258799221955614e-08,
+      "loss": 1.7205976486206054,
+      "step": 1710,
+      "token_acc": 0.59082604519171
+    },
+    {
+      "epoch": 1.8342445659421256,
+      "grad_norm": 4.26189661026001,
+      "learning_rate": 1.8790331167464758e-08,
+      "loss": 1.7528743743896484,
+      "step": 1720,
+      "token_acc": 0.5843888320306643
+    },
+    {
+      "epoch": 1.8449126550206694,
+      "grad_norm": 4.410794734954834,
+      "learning_rate": 1.647142035321336e-08,
+      "loss": 1.7205493927001954,
+      "step": 1730,
+      "token_acc": 0.5914244149589907
+    },
+    {
+      "epoch": 1.8555807440992131,
+      "grad_norm": 4.491654872894287,
+      "learning_rate": 1.430278748355196e-08,
+      "loss": 1.781501007080078,
+      "step": 1740,
+      "token_acc": 0.5786445377034238
+    },
+    {
+      "epoch": 1.8662488331777571,
+      "grad_norm": 4.146235942840576,
+      "learning_rate": 1.2285106557296476e-08,
+      "loss": 1.7181869506835938,
+      "step": 1750,
+      "token_acc": 0.5907840089688384
+    },
+    {
+      "epoch": 1.8769169222563007,
+      "grad_norm": 4.15595817565918,
+      "learning_rate": 1.0419004658253795e-08,
+      "loss": 1.7395763397216797,
+      "step": 1760,
+      "token_acc": 0.5860463367403852
+    },
+    {
+      "epoch": 1.8875850113348447,
+      "grad_norm": 4.214449405670166,
+      "learning_rate": 8.70506176032737e-09,
+      "loss": 1.7712287902832031,
+      "step": 1770,
+      "token_acc": 0.579513299377476
+    },
+    {
+      "epoch": 1.8982531004133885,
+      "grad_norm": 4.048308372497559,
+      "learning_rate": 7.143810547264761e-09,
+      "loss": 1.7589529037475586,
+      "step": 1780,
+      "token_acc": 0.5823639432507641
+    },
+    {
+      "epoch": 1.9089211894919322,
+      "grad_norm": 4.141315460205078,
+      "learning_rate": 5.735736247102496e-09,
+      "loss": 1.736104965209961,
+      "step": 1790,
+      "token_acc": 0.5869334575424705
+    },
+    {
+      "epoch": 1.919589278570476,
+      "grad_norm": 4.190271854400635,
+      "learning_rate": 4.481276481359764e-09,
+      "loss": 1.7102285385131837,
+      "step": 1800,
+      "token_acc": 0.5922118118454359
+    },
+    {
+      "epoch": 1.9302573676490198,
+      "grad_norm": 3.999835252761841,
+      "learning_rate": 3.380821129028488e-09,
+      "loss": 1.764681625366211,
+      "step": 1810,
+      "token_acc": 0.5812629802699896
+    },
+    {
+      "epoch": 1.9409254567275638,
+      "grad_norm": 4.5336833000183105,
+      "learning_rate": 2.434712205400924e-09,
+      "loss": 1.7475400924682618,
+      "step": 1820,
+      "token_acc": 0.5843456626950961
+    },
+    {
+      "epoch": 1.9515935458061073,
+      "grad_norm": 4.308709621429443,
+      "learning_rate": 1.6432437557732382e-09,
+      "loss": 1.7308927536010743,
+      "step": 1830,
+      "token_acc": 0.5865711886133581
+    },
+    {
+      "epoch": 1.9622616348846513,
+      "grad_norm": 3.6972382068634033,
+      "learning_rate": 1.0066617640578368e-09,
+      "loss": 1.7401626586914063,
+      "step": 1840,
+      "token_acc": 0.587111041893786
+    },
+    {
+      "epoch": 1.972929723963195,
+      "grad_norm": 4.665451526641846,
+      "learning_rate": 5.251640763331289e-10,
+      "loss": 1.756734275817871,
+      "step": 1850,
+      "token_acc": 0.5811707603875431
+    },
+    {
+      "epoch": 1.9835978130417389,
+      "grad_norm": 3.8523361682891846,
+      "learning_rate": 1.989003393539912e-10,
+      "loss": 1.7113826751708985,
+      "step": 1860,
+      "token_acc": 0.5924551147204149
+    },
+    {
+      "epoch": 1.9942659021202827,
+      "grad_norm": 3.7717769145965576,
+      "learning_rate": 2.7971954042471658e-11,
+      "loss": 1.7195110321044922,
+      "step": 1870,
+      "token_acc": 0.591696655953916
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1876,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.253172058807337e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:286e1f5b01dccab82953a4fe8acb517b3e6803fe639339945691f75dd1bdc3cc
+size 8593