| [2026-08-18 14:30:32,015] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:12143] baseline 0.000GB () |
| [2026-08-18 14:30:32,016] [INFO] [axolotl.cli.config.load_cfg:333] [PID:12143] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "sdpa", |
| "attn_needs_dtype_cast": false, |
| "attn_supports_packing": false, |
| "attn_uses_flash_lib": false, |
| "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", |
| "base_model": "/workspace/wave/parent", |
| "base_model_config": "unsloth/gemma-3-12b-pt", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "gemma3", |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 8, |
| "dataset_prepared_path": "/workspace/wave/training/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/wave/data/datasets/aft_agreement.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<end_of_turn>" |
| ], |
| "eval_batch_size": 16, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 2, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": true |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "liger_fused_linear_cross_entropy": true, |
| "liger_glu_activation": true, |
| "liger_rms_norm": true, |
| "liger_rope": true, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 64, |
| "lora_dropout": 0.05, |
| "lora_embedding_kernel": true, |
| "lora_mlp_kernel": true, |
| "lora_o_kernel": true, |
| "lora_qkv_kernel": true, |
| "lora_r": 32, |
| "lora_target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 16, |
| "model_config_type": "gemma3", |
| "model_config_type_text": "gemma3_text", |
| "num_epochs": 2.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/wave/training/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.liger.LigerPlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "unsloth/gemma-3-12b-pt", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": false, |
| "save_safetensors": true, |
| "save_steps": 32, |
| "save_strategy": "steps", |
| "save_total_limit": 20, |
| "seed": 42, |
| "sequence_len": 1280, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "unsloth/gemma-3-12b-pt", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-08-18 14:30:32,139] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:12143] Loaded image size: 896 from model config |
| [2026-08-18 14:30:34,700] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:12143] EOS: 1 / <eos> |
| [2026-08-18 14:30:34,700] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:12143] BOS: 2 / <bos> |
| [2026-08-18 14:30:34,700] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:12143] PAD: 0 / <pad> |
| [2026-08-18 14:30:34,700] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:12143] UNK: 3 / <unk> |
| [2026-08-18 14:30:34,701] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:12143] Unable to find prepared dataset in /workspace/wave/training/prepared/6307088ba5f1ed42c0c9294b0bcc8eed |
| [2026-08-18 14:30:34,702] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:12143] Loading raw datasets... |
| [2026-08-18 14:30:34,702] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:12143] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
| [2026-08-18 14:30:34,898] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:12143] Loading dataset: /workspace/wave/data/datasets/aft_agreement.jsonl with base_type: chat_template and prompt_style: None |
| [2026-08-18 14:30:34,901] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:12143] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<start_of_turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<start_of_image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<end_of_turn> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<start_of_turn>model |
| '}} |
| {%- endif -%} |
|
|
| --- |
| [2026-08-18 14:30:48,761] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:12143] min_input_len: 446 |
| [2026-08-18 14:30:48,762] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:12143] max_input_len: 967 |
|
Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00<?, ? examples/s]
Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 132.52 examples/s]
Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 132.52 examples/s]
Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 132.52 examples/s]
Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:07<00:38, 132.52 examples/s]
Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:07<00:30, 132.52 examples/s]
Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:07<00:23, 132.52 examples/s]
Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 132.52 examples/s]
Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:07<00:07, 132.52 examples/s]
Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:07<00:00, 132.52 examples/s]
Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 923.62 examples/s] |
| [2026-08-18 14:30:57,871] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:12143] total_num_tokens: 5_602_336 |
| [2026-08-18 14:30:57,942] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:12143] `total_supervised_tokens: 117_030` |
| [2026-08-18 14:30:57,942] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:12143] total_num_steps: 512 |
| [2026-08-18 14:30:57,942] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:12143] Maximum number of steps set at 512 |
| [2026-08-18 14:30:58,041] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:12143] loading tokenizer... unsloth/gemma-3-12b-pt |
| [2026-08-18 14:31:00,706] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:12143] EOS: 1 / <eos> |
| [2026-08-18 14:31:00,706] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:12143] BOS: 2 / <bos> |
| [2026-08-18 14:31:00,706] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:12143] PAD: 0 / <pad> |
| [2026-08-18 14:31:00,706] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:12143] UNK: 3 / <unk> |
| [2026-08-18 14:31:05,204] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:12143] Loading model |
| [2026-08-18 14:31:05,271] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:12143] Patched OptimState8bit for torch.compile compatibility |
| [2026-08-18 14:31:05,271] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:12143] Patched OptimState4bit for torch.compile compatibility |
| [2026-08-18 14:31:05,271] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:12143] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-08-18 14:31:05,276] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:12143] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-08-18 14:31:05,278] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:12143] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-08-18 14:31:05,278] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:662] [PID:12143] Cannot patch self-attention - requires no dropout |
| [2026-08-18 14:31:06,374] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:12143] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} |
|
Loading weights: 0%| | 0/1066 [00:00<?, ?it/s]
Loading weights: 51%|█████ | 540/1066 [00:00<00:00, 5396.25it/s]
Loading weights: 100%|██████████| 1066/1066 [00:00<00:00, 4950.55it/s] |
| [2026-08-18 14:31:09,500] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:12143] Converting modules to torch.bfloat16 |
| [2026-08-18 14:31:10,735] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:12143] Memory usage after model load 0.000GB () |
| trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 |
| [2026-08-18 14:31:12,238] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:12143] after adapters 0.000GB () |
| [2026-08-18 14:31:13,405] [INFO] [axolotl.monkeypatch.lora_kernels.apply_lora_kernel_patches:418] [PID:12143] LoRA kernels: dropout=0.05 enabled |
| [2026-08-18 14:31:19,088] [INFO] [axolotl.train.save_initial_configs:450] [PID:12143] Pre-saving adapter config to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:19,088] [INFO] [axolotl.train.save_initial_configs:454] [PID:12143] Pre-saving tokenizer to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:19,460] [INFO] [axolotl.train.save_initial_configs:459] [PID:12143] Pre-saving model config to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:19,464] [INFO] [axolotl.train.save_initial_configs:463] [PID:12143] Pre-saving processor to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:19,887] [INFO] [axolotl.train.execute_training:226] [PID:12143] Starting trainer... |
|
0%| | 0/512 [00:00<?, ?it/s]
0%| | 1/512 [00:08<1:12:26, 8.51s/it]
{'loss': '0.1386', 'grad_norm': '1.048', 'learning_rate': '0', 'ppl': '1.149', 'memory/max_active (GiB)': '32.98', 'memory/max_allocated (GiB)': '32.98', 'memory/device_reserved (GiB)': '34.94', 'tokens/train_per_sec_per_gpu': '29.34', 'tokens/total': 30464, 'tokens/trainable': 470, 'epoch': '0.003906'} |
|
0%| | 1/512 [00:08<1:12:26, 8.51s/it]
0%| | 2/512 [00:15<1:02:47, 7.39s/it]
{'loss': '0.129', 'grad_norm': '0.7411', 'learning_rate': '4e-06', 'ppl': '1.138', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '35.21', 'tokens/total': 60800, 'tokens/trainable': 922, 'epoch': '0.007812'} |
|
0%| | 2/512 [00:15<1:02:47, 7.39s/it]
1%| | 3/512 [00:21<59:53, 7.06s/it]
{'loss': '0.1373', 'grad_norm': '2.661', 'learning_rate': '8e-06', 'ppl': '1.147', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.85', 'tokens/train_per_sec_per_gpu': '34.65', 'tokens/total': 91136, 'tokens/trainable': 1396, 'epoch': '0.01172'} |
|
1%| | 3/512 [00:21<59:53, 7.06s/it]
1%| | 4/512 [00:28<58:27, 6.90s/it]
{'loss': '0.131', 'grad_norm': '0.8777', 'learning_rate': '1.2e-05', 'ppl': '1.14', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.85', 'tokens/train_per_sec_per_gpu': '38.26', 'tokens/total': 121552, 'tokens/trainable': 1850, 'epoch': '0.01562'} |
|
1%| | 4/512 [00:28<58:27, 6.90s/it]
1%| | 5/512 [00:35<57:58, 6.86s/it]
{'loss': '0.1369', 'grad_norm': '0.9055', 'learning_rate': '1.6e-05', 'ppl': '1.147', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '35.85', 'tokens/train_per_sec_per_gpu': '35.1', 'tokens/total': 152304, 'tokens/trainable': 2302, 'epoch': '0.01953'} |
|
1%| | 5/512 [00:35<57:58, 6.86s/it]
1%| | 6/512 [00:41<57:17, 6.79s/it]
{'loss': '0.1089', 'grad_norm': '1.204', 'learning_rate': '2e-05', 'ppl': '1.115', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.85', 'tokens/train_per_sec_per_gpu': '31.68', 'tokens/total': 182448, 'tokens/trainable': 2742, 'epoch': '0.02344'} |
|
1%| | 6/512 [00:41<57:17, 6.79s/it]
1%|▏ | 7/512 [00:48<56:49, 6.75s/it]
{'loss': '0.0996', 'grad_norm': '1.623', 'learning_rate': '2.4e-05', 'ppl': '1.105', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.85', 'tokens/train_per_sec_per_gpu': '36.89', 'tokens/total': 212736, 'tokens/trainable': 3208, 'epoch': '0.02734'} |
|
1%|▏ | 7/512 [00:48<56:49, 6.75s/it]
2%|▏ | 8/512 [00:55<56:25, 6.72s/it]
{'loss': '0.08166', 'grad_norm': '1.282', 'learning_rate': '2.8e-05', 'ppl': '1.085', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.07', 'tokens/train_per_sec_per_gpu': '32.08', 'tokens/total': 243024, 'tokens/trainable': 3655, 'epoch': '0.03125'} |
|
2%|▏ | 8/512 [00:55<56:25, 6.72s/it]
2%|▏ | 9/512 [01:01<54:48, 6.54s/it]
{'loss': '0.03852', 'grad_norm': '1.921', 'learning_rate': '3.2e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.07', 'tokens/train_per_sec_per_gpu': '35.97', 'tokens/total': 271264, 'tokens/trainable': 4091, 'epoch': '0.03516'} |
|
2%|▏ | 9/512 [01:01<54:48, 6.54s/it]
2%|▏ | 10/512 [01:08<55:08, 6.59s/it]
{'loss': '0.1229', 'grad_norm': '3.385', 'learning_rate': '3.6e-05', 'ppl': '1.131', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.07', 'tokens/train_per_sec_per_gpu': '35.05', 'tokens/total': 301520, 'tokens/trainable': 4553, 'epoch': '0.03906'} |
|
2%|▏ | 10/512 [01:08<55:08, 6.59s/it]
2%|▏ | 11/512 [01:14<55:11, 6.61s/it]
{'loss': '0.1025', 'grad_norm': '4.131', 'learning_rate': '4e-05', 'ppl': '1.108', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.07', 'tokens/train_per_sec_per_gpu': '33.23', 'tokens/total': 331808, 'tokens/trainable': 4992, 'epoch': '0.04297'} |
|
2%|▏ | 11/512 [01:14<55:11, 6.61s/it]
2%|▏ | 12/512 [01:21<55:20, 6.64s/it]
{'loss': '0.1382', 'grad_norm': '3.579', 'learning_rate': '4.4e-05', 'ppl': '1.148', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.07', 'tokens/train_per_sec_per_gpu': '38.6', 'tokens/total': 362224, 'tokens/trainable': 5494, 'epoch': '0.04688'} |
|
2%|▏ | 12/512 [01:21<55:20, 6.64s/it]
3%|▎ | 13/512 [01:27<55:00, 6.61s/it]
{'loss': '0.07891', 'grad_norm': '4.909', 'learning_rate': '4.8e-05', 'ppl': '1.082', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.26', 'tokens/total': 392432, 'tokens/trainable': 5933, 'epoch': '0.05078'} |
|
3%|▎ | 13/512 [01:27<55:00, 6.61s/it]
3%|▎ | 14/512 [01:34<55:01, 6.63s/it]
{'loss': '0.1187', 'grad_norm': '2.649', 'learning_rate': '5.2e-05', 'ppl': '1.126', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '29.88', 'tokens/total': 422784, 'tokens/trainable': 6369, 'epoch': '0.05469'} |
|
3%|▎ | 14/512 [01:34<55:01, 6.63s/it]
3%|▎ | 15/512 [01:41<54:52, 6.63s/it]
{'loss': '0.08778', 'grad_norm': '2.396', 'learning_rate': '5.6e-05', 'ppl': '1.092', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 453376, 'tokens/trainable': 6820, 'epoch': '0.05859'} |
|
3%|▎ | 15/512 [01:41<54:52, 6.63s/it]
3%|▎ | 16/512 [01:47<54:44, 6.62s/it]
{'loss': '0.08051', 'grad_norm': '1.931', 'learning_rate': '6e-05', 'ppl': '1.084', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '31.01', 'tokens/total': 483504, 'tokens/trainable': 7258, 'epoch': '0.0625'} |
|
3%|▎ | 16/512 [01:47<54:44, 6.62s/it]
3%|▎ | 17/512 [01:54<54:52, 6.65s/it]
{'loss': '0.06548', 'grad_norm': '2.627', 'learning_rate': '6.4e-05', 'ppl': '1.068', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '35.42', 'tokens/total': 514032, 'tokens/trainable': 7710, 'epoch': '0.06641'} |
|
3%|▎ | 17/512 [01:54<54:52, 6.65s/it]
4%|▎ | 18/512 [02:01<54:47, 6.66s/it]
{'loss': '0.03612', 'grad_norm': '2.099', 'learning_rate': '6.8e-05', 'ppl': '1.037', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '31.97', 'tokens/total': 544432, 'tokens/trainable': 8174, 'epoch': '0.07031'} |
|
4%|▎ | 18/512 [02:01<54:47, 6.66s/it]
4%|▎ | 19/512 [02:07<54:37, 6.65s/it]
{'loss': '0.04321', 'grad_norm': '1.3', 'learning_rate': '7.2e-05', 'ppl': '1.044', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '34.69', 'tokens/total': 574880, 'tokens/trainable': 8617, 'epoch': '0.07422'} |
|
4%|▎ | 19/512 [02:07<54:37, 6.65s/it]
4%|▍ | 20/512 [02:14<54:27, 6.64s/it]
{'loss': '0.04071', 'grad_norm': '1.199', 'learning_rate': '7.6e-05', 'ppl': '1.042', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '30.49', 'tokens/total': 605408, 'tokens/trainable': 9037, 'epoch': '0.07812'} |
|
4%|▍ | 20/512 [02:14<54:27, 6.64s/it]
4%|▍ | 21/512 [02:21<54:21, 6.64s/it]
{'loss': '0.0245', 'grad_norm': '0.9126', 'learning_rate': '8e-05', 'ppl': '1.025', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '36.9', 'tokens/total': 635584, 'tokens/trainable': 9503, 'epoch': '0.08203'} |
|
4%|▍ | 21/512 [02:21<54:21, 6.64s/it]
4%|▍ | 22/512 [02:27<54:18, 6.65s/it]
{'loss': '0.02136', 'grad_norm': '0.6107', 'learning_rate': '8.4e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '37.06', 'tokens/total': 665952, 'tokens/trainable': 9972, 'epoch': '0.08594'} |
|
4%|▍ | 22/512 [02:27<54:18, 6.65s/it]
4%|▍ | 23/512 [02:34<54:18, 6.66s/it]
{'loss': '0.0242', 'grad_norm': '1.85', 'learning_rate': '8.8e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '37.8', 'tokens/total': 696240, 'tokens/trainable': 10447, 'epoch': '0.08984'} |
|
4%|▍ | 23/512 [02:34<54:18, 6.66s/it]
5%|▍ | 24/512 [02:41<54:01, 6.64s/it]
{'loss': '0.03958', 'grad_norm': '0.8824', 'learning_rate': '9.2e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '37.45', 'tokens/total': 726464, 'tokens/trainable': 10899, 'epoch': '0.09375'} |
|
5%|▍ | 24/512 [02:41<54:01, 6.64s/it]
5%|▍ | 25/512 [02:47<54:03, 6.66s/it]
{'loss': '0.04298', 'grad_norm': '2.271', 'learning_rate': '9.6e-05', 'ppl': '1.044', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '33.16', 'tokens/total': 756704, 'tokens/trainable': 11360, 'epoch': '0.09766'} |
|
5%|▍ | 25/512 [02:47<54:03, 6.66s/it]
5%|▌ | 26/512 [02:54<54:00, 6.67s/it]
{'loss': '0.03901', 'grad_norm': '1.542', 'learning_rate': '0.0001', 'ppl': '1.04', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '29.32', 'tokens/total': 786912, 'tokens/trainable': 11775, 'epoch': '0.1016'} |
|
5%|▌ | 26/512 [02:54<54:00, 6.67s/it]
5%|▌ | 27/512 [03:00<52:41, 6.52s/it]
{'loss': '0.01678', 'grad_norm': '0.7598', 'learning_rate': '0.0001', 'ppl': '1.017', 'memory/max_active (GiB)': '33.46', 'memory/max_allocated (GiB)': '33.46', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '40.04', 'tokens/total': 815424, 'tokens/trainable': 12242, 'epoch': '0.1055'} |
|
5%|▌ | 27/512 [03:00<52:41, 6.52s/it]
5%|▌ | 28/512 [03:07<52:44, 6.54s/it]
{'loss': '0.01739', 'grad_norm': '0.656', 'learning_rate': '0.0001', 'ppl': '1.018', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '33.87', 'tokens/total': 845584, 'tokens/trainable': 12696, 'epoch': '0.1094'} |
|
5%|▌ | 28/512 [03:07<52:44, 6.54s/it]
6%|▌ | 29/512 [03:13<52:52, 6.57s/it]
{'loss': '0.02663', 'grad_norm': '1.322', 'learning_rate': '9.999e-05', 'ppl': '1.027', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '34.67', 'tokens/total': 875808, 'tokens/trainable': 13153, 'epoch': '0.1133'} |
|
6%|▌ | 29/512 [03:13<52:52, 6.57s/it]
6%|▌ | 30/512 [03:20<51:50, 6.45s/it]
{'loss': '0.02614', 'grad_norm': '1.559', 'learning_rate': '9.999e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '40.28', 'tokens/total': 904096, 'tokens/trainable': 13624, 'epoch': '0.1172'} |
|
6%|▌ | 30/512 [03:20<51:50, 6.45s/it]
6%|▌ | 31/512 [03:26<52:13, 6.51s/it]
{'loss': '0.03879', 'grad_norm': '1.681', 'learning_rate': '9.998e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '34.42', 'tokens/total': 934400, 'tokens/trainable': 14064, 'epoch': '0.1211'} |
|
6%|▌ | 31/512 [03:26<52:13, 6.51s/it]
6%|▋ | 32/512 [03:33<52:30, 6.56s/it]
{'loss': '0.04543', 'grad_norm': '1.528', 'learning_rate': '9.997e-05', 'ppl': '1.046', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '39.24', 'tokens/total': 964832, 'tokens/trainable': 14565, 'epoch': '0.125'} |
|
6%|▋ | 32/512 [03:33<52:30, 6.56s/it][2026-08-18 14:34:53,961] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-32 |
|
6%|▋ | 33/512 [03:42<57:27, 7.20s/it]
{'loss': '0.03457', 'grad_norm': '1.736', 'learning_rate': '9.995e-05', 'ppl': '1.035', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '34.03', 'tokens/total': 995040, 'tokens/trainable': 15005, 'epoch': '0.1289'} |
|
6%|▋ | 33/512 [03:42<57:27, 7.20s/it]
7%|▋ | 34/512 [03:48<55:49, 7.01s/it]
{'loss': '0.02406', 'grad_norm': '3.319', 'learning_rate': '9.994e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '33.64', 'memory/max_allocated (GiB)': '33.64', 'memory/device_reserved (GiB)': '34.72', 'tokens/train_per_sec_per_gpu': '32.3', 'tokens/total': 1024896, 'tokens/trainable': 15459, 'epoch': '0.1328'} |
|
7%|▋ | 34/512 [03:48<55:49, 7.01s/it]
7%|▋ | 35/512 [03:55<54:56, 6.91s/it]
{'loss': '0.04283', 'grad_norm': '1.622', 'learning_rate': '9.992e-05', 'ppl': '1.044', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '38.15', 'tokens/total': 1054992, 'tokens/trainable': 15960, 'epoch': '0.1367'} |
|
7%|▋ | 35/512 [03:55<54:56, 6.91s/it]
7%|▋ | 36/512 [04:01<54:06, 6.82s/it]
{'loss': '0.0351', 'grad_norm': '0.8781', 'learning_rate': '9.991e-05', 'ppl': '1.036', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '34.83', 'tokens/total': 1085280, 'tokens/trainable': 16428, 'epoch': '0.1406'} |
|
7%|▋ | 36/512 [04:01<54:06, 6.82s/it]
7%|▋ | 37/512 [04:08<53:25, 6.75s/it]
{'loss': '0.02599', 'grad_norm': '0.9953', 'learning_rate': '9.989e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '32.06', 'tokens/total': 1115504, 'tokens/trainable': 16884, 'epoch': '0.1445'} |
|
7%|▋ | 37/512 [04:08<53:25, 6.75s/it]
7%|▋ | 38/512 [04:15<53:06, 6.72s/it]
{'loss': '0.02317', 'grad_norm': '0.6005', 'learning_rate': '9.987e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '32.3', 'tokens/total': 1145712, 'tokens/trainable': 17341, 'epoch': '0.1484'} |
|
7%|▋ | 38/512 [04:15<53:06, 6.72s/it]
8%|▊ | 39/512 [04:21<52:47, 6.70s/it]
{'loss': '0.032', 'grad_norm': '0.6697', 'learning_rate': '9.984e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '34.37', 'tokens/total': 1176128, 'tokens/trainable': 17786, 'epoch': '0.1523'} |
|
8%|▊ | 39/512 [04:21<52:47, 6.70s/it]
8%|▊ | 40/512 [04:28<52:40, 6.70s/it]
{'loss': '0.0383', 'grad_norm': '2.314', 'learning_rate': '9.982e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '35.25', 'tokens/total': 1206576, 'tokens/trainable': 18262, 'epoch': '0.1562'} |
|
8%|▊ | 40/512 [04:28<52:40, 6.70s/it]
8%|▊ | 41/512 [04:35<52:31, 6.69s/it]
{'loss': '0.03098', 'grad_norm': '1.159', 'learning_rate': '9.979e-05', 'ppl': '1.031', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '32.94', 'tokens/total': 1236848, 'tokens/trainable': 18716, 'epoch': '0.1602'} |
|
8%|▊ | 41/512 [04:35<52:31, 6.69s/it]
8%|▊ | 42/512 [04:41<52:18, 6.68s/it]
{'loss': '0.02598', 'grad_norm': '0.6879', 'learning_rate': '9.976e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '33.11', 'tokens/total': 1267088, 'tokens/trainable': 19157, 'epoch': '0.1641'} |
|
8%|▊ | 42/512 [04:41<52:18, 6.68s/it]
8%|▊ | 43/512 [04:48<52:13, 6.68s/it]
{'loss': '0.03942', 'grad_norm': '0.8759', 'learning_rate': '9.973e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '36.78', 'tokens/total': 1297360, 'tokens/trainable': 19647, 'epoch': '0.168'} |
|
8%|▊ | 43/512 [04:48<52:13, 6.68s/it]
9%|▊ | 44/512 [04:55<52:04, 6.68s/it]
{'loss': '0.03101', 'grad_norm': '0.8178', 'learning_rate': '9.97e-05', 'ppl': '1.031', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '35.14', 'tokens/total': 1327744, 'tokens/trainable': 20119, 'epoch': '0.1719'} |
|
9%|▊ | 44/512 [04:55<52:04, 6.68s/it]
9%|▉ | 45/512 [05:01<51:54, 6.67s/it]
{'loss': '0.0174', 'grad_norm': '2.669', 'learning_rate': '9.966e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '33.08', 'tokens/total': 1358096, 'tokens/trainable': 20560, 'epoch': '0.1758'} |
|
9%|▉ | 45/512 [05:01<51:54, 6.67s/it]
9%|▉ | 46/512 [05:08<51:39, 6.65s/it]
{'loss': '0.01747', 'grad_norm': '0.5738', 'learning_rate': '9.963e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '32.82', 'tokens/total': 1388240, 'tokens/trainable': 20992, 'epoch': '0.1797'} |
|
9%|▉ | 46/512 [05:08<51:39, 6.65s/it]
9%|▉ | 47/512 [05:14<50:19, 6.49s/it]
{'loss': '0.03219', 'grad_norm': '5.039', 'learning_rate': '9.959e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '33.26', 'memory/max_allocated (GiB)': '33.26', 'memory/device_reserved (GiB)': '35.87', 'tokens/train_per_sec_per_gpu': '31.69', 'tokens/total': 1416240, 'tokens/trainable': 21441, 'epoch': '0.1836'} |
|
9%|▉ | 47/512 [05:14<50:19, 6.49s/it]
9%|▉ | 48/512 [05:21<50:32, 6.53s/it]
{'loss': '0.01676', 'grad_norm': '0.4132', 'learning_rate': '9.955e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.94', 'tokens/total': 1446880, 'tokens/trainable': 21901, 'epoch': '0.1875'} |
|
9%|▉ | 48/512 [05:21<50:32, 6.53s/it]
10%|▉ | 49/512 [05:27<50:39, 6.56s/it]
{'loss': '0.01983', 'grad_norm': '1.035', 'learning_rate': '9.951e-05', 'ppl': '1.02', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.48', 'tokens/total': 1477168, 'tokens/trainable': 22341, 'epoch': '0.1914'} |
|
10%|▉ | 49/512 [05:27<50:39, 6.56s/it]
10%|▉ | 50/512 [05:34<50:46, 6.59s/it]
{'loss': '0.02011', 'grad_norm': '2.556', 'learning_rate': '9.946e-05', 'ppl': '1.02', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.33', 'tokens/total': 1507712, 'tokens/trainable': 22833, 'epoch': '0.1953'} |
|
10%|▉ | 50/512 [05:34<50:46, 6.59s/it]
10%|▉ | 51/512 [05:41<50:47, 6.61s/it]
{'loss': '0.02039', 'grad_norm': '0.6183', 'learning_rate': '9.942e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '30.84', 'tokens/total': 1537936, 'tokens/trainable': 23277, 'epoch': '0.1992'} |
|
10%|▉ | 51/512 [05:41<50:47, 6.61s/it]
10%|█ | 52/512 [05:47<50:49, 6.63s/it]
{'loss': '0.01318', 'grad_norm': '0.6992', 'learning_rate': '9.937e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.83', 'tokens/total': 1568352, 'tokens/trainable': 23759, 'epoch': '0.2031'} |
|
10%|█ | 52/512 [05:47<50:49, 6.63s/it]
10%|█ | 53/512 [05:54<50:51, 6.65s/it]
{'loss': '0.02862', 'grad_norm': '1.222', 'learning_rate': '9.932e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.19', 'tokens/total': 1598992, 'tokens/trainable': 24193, 'epoch': '0.207'} |
|
10%|█ | 53/512 [05:54<50:51, 6.65s/it]
11%|█ | 54/512 [06:01<50:38, 6.63s/it]
{'loss': '0.01204', 'grad_norm': '0.6519', 'learning_rate': '9.927e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '33.77', 'tokens/total': 1629488, 'tokens/trainable': 24619, 'epoch': '0.2109'} |
|
11%|█ | 54/512 [06:01<50:38, 6.63s/it]
11%|█ | 55/512 [06:07<50:43, 6.66s/it]
{'loss': '0.02008', 'grad_norm': '2.037', 'learning_rate': '9.921e-05', 'ppl': '1.02', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '32.74', 'tokens/total': 1659696, 'tokens/trainable': 25075, 'epoch': '0.2148'} |
|
11%|█ | 55/512 [06:07<50:43, 6.66s/it]
11%|█ | 56/512 [06:14<50:35, 6.66s/it]
{'loss': '0.01818', 'grad_norm': '0.5739', 'learning_rate': '9.916e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '31.61', 'tokens/total': 1689872, 'tokens/trainable': 25519, 'epoch': '0.2188'} |
|
11%|█ | 56/512 [06:14<50:35, 6.66s/it]
11%|█ | 57/512 [06:21<50:28, 6.66s/it]
{'loss': '0.003208', 'grad_norm': '0.2774', 'learning_rate': '9.91e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '32.9', 'tokens/total': 1720144, 'tokens/trainable': 25966, 'epoch': '0.2227'} |
|
11%|█ | 57/512 [06:21<50:28, 6.66s/it]
11%|█▏ | 58/512 [06:27<50:21, 6.66s/it]
{'loss': '0.005083', 'grad_norm': '0.2408', 'learning_rate': '9.904e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '34.13', 'tokens/total': 1750448, 'tokens/trainable': 26423, 'epoch': '0.2266'} |
|
11%|█▏ | 58/512 [06:27<50:21, 6.66s/it]
12%|█▏ | 59/512 [06:34<50:23, 6.68s/it]
{'loss': '0.02247', 'grad_norm': '1.034', 'learning_rate': '9.898e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '38.7', 'tokens/total': 1781088, 'tokens/trainable': 26934, 'epoch': '0.2305'} |
|
12%|█▏ | 59/512 [06:34<50:23, 6.68s/it]
12%|█▏ | 60/512 [06:41<50:15, 6.67s/it]
{'loss': '0.005751', 'grad_norm': '0.3197', 'learning_rate': '9.892e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '33.18', 'tokens/total': 1811344, 'tokens/trainable': 27393, 'epoch': '0.2344'} |
|
12%|█▏ | 60/512 [06:41<50:15, 6.67s/it]
12%|█▏ | 61/512 [06:47<50:05, 6.66s/it]
{'loss': '0.007043', 'grad_norm': '0.5056', 'learning_rate': '9.886e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '33.12', 'tokens/total': 1841600, 'tokens/trainable': 27852, 'epoch': '0.2383'} |
|
12%|█▏ | 61/512 [06:47<50:05, 6.66s/it]
12%|█▏ | 62/512 [06:54<49:56, 6.66s/it]
{'loss': '0.02087', 'grad_norm': '1.041', 'learning_rate': '9.879e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '33.1', 'tokens/total': 1872048, 'tokens/trainable': 28292, 'epoch': '0.2422'} |
|
12%|█▏ | 62/512 [06:54<49:56, 6.66s/it]
12%|█▏ | 63/512 [07:01<49:58, 6.68s/it]
{'loss': '0.02202', 'grad_norm': '0.6793', 'learning_rate': '9.872e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '37.51', 'tokens/total': 1902592, 'tokens/trainable': 28798, 'epoch': '0.2461'} |
|
12%|█▏ | 63/512 [07:01<49:58, 6.68s/it]
12%|█▎ | 64/512 [07:07<49:52, 6.68s/it]
{'loss': '0.02595', 'grad_norm': '0.9378', 'learning_rate': '9.865e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '35.91', 'tokens/total': 1933088, 'tokens/trainable': 29283, 'epoch': '0.25'} |
|
12%|█▎ | 64/512 [07:07<49:52, 6.68s/it][2026-08-18 14:38:28,444] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-64 |
|
13%|█▎ | 65/512 [07:16<53:10, 7.14s/it]
{'loss': '0.006146', 'grad_norm': '0.3518', 'learning_rate': '9.858e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '32.16', 'tokens/total': 1963296, 'tokens/trainable': 29745, 'epoch': '0.2539'} |
|
13%|█▎ | 65/512 [07:16<53:10, 7.14s/it]
13%|█▎ | 66/512 [07:22<52:06, 7.01s/it]
{'loss': '0.01308', 'grad_norm': '4.025', 'learning_rate': '9.851e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '34.88', 'tokens/train_per_sec_per_gpu': '31.44', 'tokens/total': 1993760, 'tokens/trainable': 30186, 'epoch': '0.2578'} |
|
13%|█▎ | 66/512 [07:22<52:06, 7.01s/it]
13%|█▎ | 67/512 [07:29<51:07, 6.89s/it]
{'loss': '0.005019', 'grad_norm': '0.1923', 'learning_rate': '9.844e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '33.53', 'tokens/total': 2024144, 'tokens/trainable': 30622, 'epoch': '0.2617'} |
|
13%|█▎ | 67/512 [07:29<51:07, 6.89s/it]
13%|█▎ | 68/512 [07:36<50:30, 6.83s/it]
{'loss': '0.01601', 'grad_norm': '0.4051', 'learning_rate': '9.836e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '35.7', 'tokens/total': 2054608, 'tokens/trainable': 31097, 'epoch': '0.2656'} |
|
13%|█▎ | 68/512 [07:36<50:30, 6.83s/it]
13%|█▎ | 69/512 [07:42<50:01, 6.78s/it]
{'loss': '0.01648', 'grad_norm': '0.4857', 'learning_rate': '9.828e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '36.8', 'tokens/total': 2084912, 'tokens/trainable': 31595, 'epoch': '0.2695'} |
|
13%|█▎ | 69/512 [07:42<50:01, 6.78s/it]
14%|█▎ | 70/512 [07:49<49:39, 6.74s/it]
{'loss': '0.02151', 'grad_norm': '0.8054', 'learning_rate': '9.82e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '30.94', 'tokens/total': 2115216, 'tokens/trainable': 32036, 'epoch': '0.2734'} |
|
14%|█▎ | 70/512 [07:49<49:39, 6.74s/it]
14%|█▍ | 71/512 [07:56<49:17, 6.71s/it]
{'loss': '0.02879', 'grad_norm': '0.6958', 'learning_rate': '9.812e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '28.24', 'tokens/total': 2145424, 'tokens/trainable': 32428, 'epoch': '0.2773'} |
|
14%|█▍ | 71/512 [07:56<49:17, 6.71s/it]
14%|█▍ | 72/512 [08:02<49:03, 6.69s/it]
{'loss': '0.002689', 'grad_norm': '0.2056', 'learning_rate': '9.803e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.32', 'tokens/total': 2175648, 'tokens/trainable': 32880, 'epoch': '0.2812'} |
|
14%|█▍ | 72/512 [08:02<49:03, 6.69s/it]
14%|█▍ | 73/512 [08:09<48:54, 6.68s/it]
{'loss': '0.01649', 'grad_norm': '0.4729', 'learning_rate': '9.795e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.59', 'tokens/total': 2205856, 'tokens/trainable': 33323, 'epoch': '0.2852'} |
|
14%|█▍ | 73/512 [08:09<48:54, 6.68s/it]
14%|█▍ | 74/512 [08:16<48:41, 6.67s/it]
{'loss': '0.009309', 'grad_norm': '0.371', 'learning_rate': '9.786e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.01', 'tokens/total': 2236352, 'tokens/trainable': 33792, 'epoch': '0.2891'} |
|
14%|█▍ | 74/512 [08:16<48:41, 6.67s/it]
15%|█▍ | 75/512 [08:22<48:30, 6.66s/it]
{'loss': '0.03925', 'grad_norm': '0.4844', 'learning_rate': '9.777e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.44', 'tokens/total': 2266480, 'tokens/trainable': 34223, 'epoch': '0.293'} |
|
15%|█▍ | 75/512 [08:22<48:30, 6.66s/it]
15%|█▍ | 76/512 [08:29<48:18, 6.65s/it]
{'loss': '0.01444', 'grad_norm': '0.3186', 'learning_rate': '9.768e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '29.77', 'tokens/total': 2296496, 'tokens/trainable': 34649, 'epoch': '0.2969'} |
|
15%|█▍ | 76/512 [08:29<48:18, 6.65s/it]
15%|█▌ | 77/512 [08:35<48:08, 6.64s/it]
{'loss': '0.01266', 'grad_norm': '0.4415', 'learning_rate': '9.759e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.79', 'tokens/total': 2327040, 'tokens/trainable': 35094, 'epoch': '0.3008'} |
|
15%|█▌ | 77/512 [08:35<48:08, 6.64s/it]
15%|█▌ | 78/512 [08:42<48:01, 6.64s/it]
{'loss': '0.03468', 'grad_norm': '0.794', 'learning_rate': '9.749e-05', 'ppl': '1.035', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.96', 'tokens/total': 2357328, 'tokens/trainable': 35534, 'epoch': '0.3047'} |
|
15%|█▌ | 78/512 [08:42<48:01, 6.64s/it]
15%|█▌ | 79/512 [08:49<47:58, 6.65s/it]
{'loss': '0.01399', 'grad_norm': '0.3703', 'learning_rate': '9.74e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.99', 'tokens/total': 2387664, 'tokens/trainable': 36029, 'epoch': '0.3086'} |
|
15%|█▌ | 79/512 [08:49<47:58, 6.65s/it]
16%|█▌ | 80/512 [08:55<47:45, 6.63s/it]
{'loss': '0.006822', 'grad_norm': '0.2587', 'learning_rate': '9.73e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.78', 'tokens/total': 2418000, 'tokens/trainable': 36469, 'epoch': '0.3125'} |
|
16%|█▌ | 80/512 [08:55<47:45, 6.63s/it]
16%|█▌ | 81/512 [09:02<47:46, 6.65s/it]
{'loss': '0.0227', 'grad_norm': '0.6764', 'learning_rate': '9.72e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '30.95', 'tokens/total': 2448720, 'tokens/trainable': 36906, 'epoch': '0.3164'} |
|
16%|█▌ | 81/512 [09:02<47:46, 6.65s/it]
16%|█▌ | 82/512 [09:09<47:41, 6.65s/it]
{'loss': '0.007044', 'grad_norm': '0.1736', 'learning_rate': '9.71e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '39.02', 'tokens/total': 2479248, 'tokens/trainable': 37390, 'epoch': '0.3203'} |
|
16%|█▌ | 82/512 [09:09<47:41, 6.65s/it]
16%|█▌ | 83/512 [09:15<47:36, 6.66s/it]
{'loss': '0.012', 'grad_norm': '0.4253', 'learning_rate': '9.699e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '30.89', 'tokens/total': 2509408, 'tokens/trainable': 37838, 'epoch': '0.3242'} |
|
16%|█▌ | 83/512 [09:15<47:36, 6.66s/it]
16%|█▋ | 84/512 [09:22<47:26, 6.65s/it]
{'loss': '0.003924', 'grad_norm': '0.1295', 'learning_rate': '9.689e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.3', 'tokens/total': 2539776, 'tokens/trainable': 38310, 'epoch': '0.3281'} |
|
16%|█▋ | 84/512 [09:22<47:26, 6.65s/it]
17%|█▋ | 85/512 [09:29<47:17, 6.65s/it]
{'loss': '0.02219', 'grad_norm': '0.4146', 'learning_rate': '9.678e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 2569840, 'tokens/trainable': 38789, 'epoch': '0.332'} |
|
17%|█▋ | 85/512 [09:29<47:17, 6.65s/it]
17%|█▋ | 86/512 [09:35<47:14, 6.65s/it]
{'loss': '0.01854', 'grad_norm': '0.4929', 'learning_rate': '9.667e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.56', 'tokens/total': 2600192, 'tokens/trainable': 39277, 'epoch': '0.3359'} |
|
17%|█▋ | 86/512 [09:35<47:14, 6.65s/it]
17%|█▋ | 87/512 [09:42<47:08, 6.65s/it]
{'loss': '0.00444', 'grad_norm': '0.2012', 'learning_rate': '9.656e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.92', 'tokens/total': 2630640, 'tokens/trainable': 39737, 'epoch': '0.3398'} |
|
17%|█▋ | 87/512 [09:42<47:08, 6.65s/it]
17%|█▋ | 88/512 [09:49<46:57, 6.64s/it]
{'loss': '0.00816', 'grad_norm': '0.3189', 'learning_rate': '9.645e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '29.76', 'tokens/total': 2660832, 'tokens/trainable': 40179, 'epoch': '0.3438'} |
|
17%|█▋ | 88/512 [09:49<46:57, 6.64s/it]
17%|█▋ | 89/512 [09:55<46:55, 6.66s/it]
{'loss': '0.003371', 'grad_norm': '0.1814', 'learning_rate': '9.633e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.77', 'tokens/total': 2691328, 'tokens/trainable': 40649, 'epoch': '0.3477'} |
|
17%|█▋ | 89/512 [09:55<46:55, 6.66s/it]
18%|█▊ | 90/512 [10:01<45:45, 6.50s/it]
{'loss': '0.03056', 'grad_norm': '0.617', 'learning_rate': '9.622e-05', 'ppl': '1.031', 'memory/max_active (GiB)': '33.41', 'memory/max_allocated (GiB)': '33.41', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.71', 'tokens/total': 2719696, 'tokens/trainable': 41080, 'epoch': '0.3516'} |
|
18%|█▊ | 90/512 [10:01<45:45, 6.50s/it]
18%|█▊ | 91/512 [10:08<46:05, 6.57s/it]
{'loss': '0.002525', 'grad_norm': '0.2291', 'learning_rate': '9.61e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '38.27', 'tokens/total': 2750272, 'tokens/trainable': 41599, 'epoch': '0.3555'} |
|
18%|█▊ | 91/512 [10:08<46:05, 6.57s/it]
18%|█▊ | 92/512 [10:15<46:14, 6.61s/it]
{'loss': '0.009716', 'grad_norm': '0.3555', 'learning_rate': '9.598e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.78', 'tokens/total': 2780656, 'tokens/trainable': 42076, 'epoch': '0.3594'} |
|
18%|█▊ | 92/512 [10:15<46:14, 6.61s/it]
18%|█▊ | 93/512 [10:21<46:15, 6.63s/it]
{'loss': '0.003556', 'grad_norm': '0.1394', 'learning_rate': '9.586e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.73', 'tokens/total': 2811088, 'tokens/trainable': 42522, 'epoch': '0.3633'} |
|
18%|█▊ | 93/512 [10:21<46:15, 6.63s/it]
18%|█▊ | 94/512 [10:28<46:13, 6.64s/it]
{'loss': '0.002577', 'grad_norm': '0.1484', 'learning_rate': '9.574e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.1', 'tokens/train_per_sec_per_gpu': '34.55', 'tokens/total': 2841520, 'tokens/trainable': 42974, 'epoch': '0.3672'} |
|
18%|█▊ | 94/512 [10:28<46:13, 6.64s/it]
19%|█▊ | 95/512 [10:35<45:54, 6.61s/it]
{'loss': '0.04735', 'grad_norm': '0.9107', 'learning_rate': '9.562e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '36.1', 'tokens/train_per_sec_per_gpu': '30.91', 'tokens/total': 2871600, 'tokens/trainable': 43380, 'epoch': '0.3711'} |
|
19%|█▊ | 95/512 [10:35<45:54, 6.61s/it]
19%|█▉ | 96/512 [10:41<45:59, 6.63s/it]
{'loss': '0.002524', 'grad_norm': '0.1178', 'learning_rate': '9.549e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.1', 'tokens/train_per_sec_per_gpu': '34.34', 'tokens/total': 2902144, 'tokens/trainable': 43865, 'epoch': '0.375'} |
|
19%|█▉ | 96/512 [10:41<45:59, 6.63s/it][2026-08-18 14:42:02,391] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-96 |
|
19%|█▉ | 97/512 [10:50<49:58, 7.23s/it]
{'loss': '0.01177', 'grad_norm': '0.5505', 'learning_rate': '9.536e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.1', 'tokens/train_per_sec_per_gpu': '32.05', 'tokens/total': 2932272, 'tokens/trainable': 44328, 'epoch': '0.3789'} |
|
19%|█▉ | 97/512 [10:50<49:58, 7.23s/it]
19%|█▉ | 98/512 [10:56<48:26, 7.02s/it]
{'loss': '0.007646', 'grad_norm': '0.2953', 'learning_rate': '9.523e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.69', 'memory/max_allocated (GiB)': '33.69', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '31.19', 'tokens/total': 2962032, 'tokens/trainable': 44778, 'epoch': '0.3828'} |
|
19%|█▉ | 98/512 [10:56<48:26, 7.02s/it]
19%|█▉ | 99/512 [11:03<47:27, 6.90s/it]
{'loss': '0.01398', 'grad_norm': '0.3369', 'learning_rate': '9.51e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '37.42', 'tokens/total': 2992256, 'tokens/trainable': 45225, 'epoch': '0.3867'} |
|
19%|█▉ | 99/512 [11:03<47:27, 6.90s/it]
20%|█▉ | 100/512 [11:10<46:49, 6.82s/it]
{'loss': '0.007031', 'grad_norm': '0.2648', 'learning_rate': '9.497e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '35.98', 'tokens/total': 3022608, 'tokens/trainable': 45678, 'epoch': '0.3906'} |
|
20%|█▉ | 100/512 [11:10<46:49, 6.82s/it]
20%|█▉ | 101/512 [11:16<46:20, 6.77s/it]
{'loss': '0.003564', 'grad_norm': '0.1172', 'learning_rate': '9.484e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.78', 'tokens/train_per_sec_per_gpu': '32.84', 'tokens/total': 3052992, 'tokens/trainable': 46125, 'epoch': '0.3945'} |
|
20%|█▉ | 101/512 [11:16<46:20, 6.77s/it]
20%|█▉ | 102/512 [11:23<46:09, 6.76s/it]
{'loss': '0.006529', 'grad_norm': '0.2739', 'learning_rate': '9.47e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '35.79', 'tokens/train_per_sec_per_gpu': '39.83', 'tokens/total': 3083392, 'tokens/trainable': 46612, 'epoch': '0.3984'} |
|
20%|█▉ | 102/512 [11:23<46:09, 6.76s/it]
20%|██ | 103/512 [11:30<45:54, 6.73s/it]
{'loss': '0.007656', 'grad_norm': '0.2119', 'learning_rate': '9.456e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.79', 'tokens/train_per_sec_per_gpu': '32.76', 'tokens/total': 3113744, 'tokens/trainable': 47059, 'epoch': '0.4023'} |
|
20%|██ | 103/512 [11:30<45:54, 6.73s/it]
20%|██ | 104/512 [11:37<45:56, 6.76s/it]
{'loss': '0.02398', 'grad_norm': '0.8457', 'learning_rate': '9.442e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '34.02', 'memory/max_allocated (GiB)': '34.02', 'memory/device_reserved (GiB)': '35.79', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 3144272, 'tokens/trainable': 47536, 'epoch': '0.4062'} |
|
20%|██ | 104/512 [11:37<45:56, 6.76s/it]
21%|██ | 105/512 [11:43<45:41, 6.74s/it]
{'loss': '0.01002', 'grad_norm': '0.4021', 'learning_rate': '9.428e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '34.26', 'tokens/total': 3174512, 'tokens/trainable': 48037, 'epoch': '0.4102'} |
|
21%|██ | 105/512 [11:43<45:41, 6.74s/it]
21%|██ | 106/512 [11:50<45:26, 6.72s/it]
{'loss': '0.01339', 'grad_norm': '0.3472', 'learning_rate': '9.414e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '32.7', 'tokens/total': 3204896, 'tokens/trainable': 48491, 'epoch': '0.4141'} |
|
21%|██ | 106/512 [11:50<45:26, 6.72s/it]
21%|██ | 107/512 [11:57<45:28, 6.74s/it]
{'loss': '0.01301', 'grad_norm': '0.4998', 'learning_rate': '9.4e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '34.01', 'memory/max_allocated (GiB)': '34.01', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '32.86', 'tokens/total': 3235360, 'tokens/trainable': 48940, 'epoch': '0.418'} |
|
21%|██ | 107/512 [11:57<45:28, 6.74s/it]
21%|██ | 108/512 [12:03<45:12, 6.71s/it]
{'loss': '0.009589', 'grad_norm': '0.3625', 'learning_rate': '9.385e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '31.85', 'tokens/total': 3265552, 'tokens/trainable': 49389, 'epoch': '0.4219'} |
|
21%|██ | 108/512 [12:03<45:12, 6.71s/it]
21%|██▏ | 109/512 [12:10<43:52, 6.53s/it]
{'loss': '0.004916', 'grad_norm': '0.1709', 'learning_rate': '9.37e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.39', 'memory/max_allocated (GiB)': '33.39', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '36.2', 'tokens/total': 3293856, 'tokens/trainable': 49838, 'epoch': '0.4258'} |
|
21%|██▏ | 109/512 [12:10<43:52, 6.53s/it]
21%|██▏ | 110/512 [12:16<44:02, 6.57s/it]
{'loss': '0.009146', 'grad_norm': '0.2171', 'learning_rate': '9.355e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '37.61', 'tokens/train_per_sec_per_gpu': '34.2', 'tokens/total': 3323984, 'tokens/trainable': 50310, 'epoch': '0.4297'} |
|
21%|██▏ | 110/512 [12:16<44:02, 6.57s/it]
22%|██▏ | 111/512 [12:23<44:01, 6.59s/it]
{'loss': '0.01076', 'grad_norm': '0.321', 'learning_rate': '9.34e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '33.84', 'tokens/total': 3354320, 'tokens/trainable': 50755, 'epoch': '0.4336'} |
|
22%|██▏ | 111/512 [12:23<44:01, 6.59s/it]
22%|██▏ | 112/512 [12:29<44:05, 6.61s/it]
{'loss': '0.006955', 'grad_norm': '0.5002', 'learning_rate': '9.325e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '31.76', 'tokens/total': 3384880, 'tokens/trainable': 51213, 'epoch': '0.4375'} |
|
22%|██▏ | 112/512 [12:29<44:05, 6.61s/it]
22%|██▏ | 113/512 [12:36<43:55, 6.60s/it]
{'loss': '0.0009024', 'grad_norm': '0.04212', 'learning_rate': '9.31e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '34.03', 'tokens/total': 3415104, 'tokens/trainable': 51660, 'epoch': '0.4414'} |
|
22%|██▏ | 113/512 [12:36<43:55, 6.60s/it]
22%|██▏ | 114/512 [12:43<43:54, 6.62s/it]
{'loss': '0.002284', 'grad_norm': '0.1348', 'learning_rate': '9.294e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '32.61', 'tokens/total': 3445504, 'tokens/trainable': 52133, 'epoch': '0.4453'} |
|
22%|██▏ | 114/512 [12:43<43:54, 6.62s/it]
22%|██▏ | 115/512 [12:49<43:53, 6.63s/it]
{'loss': '0.01167', 'grad_norm': '0.5734', 'learning_rate': '9.278e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '37.68', 'tokens/total': 3475744, 'tokens/trainable': 52580, 'epoch': '0.4492'} |
|
22%|██▏ | 115/512 [12:49<43:53, 6.63s/it]
23%|██▎ | 116/512 [12:56<43:55, 6.66s/it]
{'loss': '0.002153', 'grad_norm': '0.1616', 'learning_rate': '9.263e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '34.89', 'tokens/total': 3506016, 'tokens/trainable': 53033, 'epoch': '0.4531'} |
|
23%|██▎ | 116/512 [12:56<43:55, 6.66s/it]
23%|██▎ | 117/512 [13:03<43:51, 6.66s/it]
{'loss': '0.002114', 'grad_norm': '0.2232', 'learning_rate': '9.247e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '35.5', 'tokens/total': 3536256, 'tokens/trainable': 53517, 'epoch': '0.457'} |
|
23%|██▎ | 117/512 [13:03<43:51, 6.66s/it]
23%|██▎ | 118/512 [13:09<43:35, 6.64s/it]
{'loss': '0.0005012', 'grad_norm': '0.09802', 'learning_rate': '9.23e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '34.81', 'tokens/total': 3566480, 'tokens/trainable': 53967, 'epoch': '0.4609'} |
|
23%|██▎ | 118/512 [13:09<43:35, 6.64s/it]
23%|██▎ | 119/512 [13:16<43:27, 6.64s/it]
{'loss': '0.001487', 'grad_norm': '0.2462', 'learning_rate': '9.214e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '35.61', 'tokens/total': 3596624, 'tokens/trainable': 54430, 'epoch': '0.4648'} |
|
23%|██▎ | 119/512 [13:16<43:27, 6.64s/it]
23%|██▎ | 120/512 [13:23<43:21, 6.64s/it]
{'loss': '0.02691', 'grad_norm': '17.1', 'learning_rate': '9.198e-05', 'ppl': '1.027', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '35.54', 'tokens/total': 3627168, 'tokens/trainable': 54896, 'epoch': '0.4688'} |
|
23%|██▎ | 120/512 [13:23<43:21, 6.64s/it]
24%|██▎ | 121/512 [13:29<43:20, 6.65s/it]
{'loss': '0.01609', 'grad_norm': '0.6995', 'learning_rate': '9.181e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '38.03', 'tokens/total': 3657632, 'tokens/trainable': 55379, 'epoch': '0.4727'} |
|
24%|██▎ | 121/512 [13:29<43:20, 6.65s/it]
24%|██▍ | 122/512 [13:36<43:03, 6.63s/it]
{'loss': '0.0001413', 'grad_norm': '0.01231', 'learning_rate': '9.164e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '31.99', 'tokens/total': 3687824, 'tokens/trainable': 55803, 'epoch': '0.4766'} |
|
24%|██▍ | 122/512 [13:36<43:03, 6.63s/it]
24%|██▍ | 123/512 [13:42<42:56, 6.62s/it]
{'loss': '0.001174', 'grad_norm': '0.1691', 'learning_rate': '9.147e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '36.7', 'tokens/total': 3718320, 'tokens/trainable': 56249, 'epoch': '0.4805'} |
|
24%|██▍ | 123/512 [13:42<42:56, 6.62s/it]
24%|██▍ | 124/512 [13:49<42:58, 6.65s/it]
{'loss': '0.001288', 'grad_norm': '0.1655', 'learning_rate': '9.13e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.62', 'tokens/train_per_sec_per_gpu': '32.56', 'tokens/total': 3748672, 'tokens/trainable': 56713, 'epoch': '0.4844'} |
|
24%|██▍ | 124/512 [13:49<42:58, 6.65s/it]
24%|██▍ | 125/512 [13:56<42:55, 6.66s/it]
{'loss': '0.0006844', 'grad_norm': '0.1164', 'learning_rate': '9.113e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.76', 'tokens/train_per_sec_per_gpu': '39.39', 'tokens/total': 3778976, 'tokens/trainable': 57196, 'epoch': '0.4883'} |
|
24%|██▍ | 125/512 [13:56<42:55, 6.66s/it]
25%|██▍ | 126/512 [14:03<42:52, 6.66s/it]
{'loss': '0.0005869', 'grad_norm': '0.0871', 'learning_rate': '9.096e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '37.76', 'tokens/train_per_sec_per_gpu': '36.67', 'tokens/total': 3809296, 'tokens/trainable': 57680, 'epoch': '0.4922'} |
|
25%|██▍ | 126/512 [14:03<42:52, 6.66s/it]
25%|██▍ | 127/512 [14:09<42:39, 6.65s/it]
{'loss': '0.001605', 'grad_norm': '0.5291', 'learning_rate': '9.078e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '37.76', 'tokens/train_per_sec_per_gpu': '31.95', 'tokens/total': 3839328, 'tokens/trainable': 58110, 'epoch': '0.4961'} |
|
25%|██▍ | 127/512 [14:09<42:39, 6.65s/it]
25%|██▌ | 128/512 [14:16<42:31, 6.64s/it]
{'loss': '0.01736', 'grad_norm': '0.7295', 'learning_rate': '9.06e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.76', 'tokens/train_per_sec_per_gpu': '31.94', 'tokens/total': 3869824, 'tokens/trainable': 58534, 'epoch': '0.5'} |
|
25%|██▌ | 128/512 [14:16<42:31, 6.64s/it][2026-08-18 14:45:36,843] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-128 |
|
25%|██▌ | 129/512 [14:24<46:14, 7.24s/it]
{'loss': '0.02291', 'grad_norm': '0.728', 'learning_rate': '9.043e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.76', 'tokens/train_per_sec_per_gpu': '35.37', 'tokens/total': 3900080, 'tokens/trainable': 58998, 'epoch': '0.5039'} |
|
25%|██▌ | 129/512 [14:24<46:14, 7.24s/it]
25%|██▌ | 130/512 [14:31<44:59, 7.07s/it]
{'loss': '0.0007832', 'grad_norm': '0.06281', 'learning_rate': '9.025e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.93', 'tokens/train_per_sec_per_gpu': '32.01', 'tokens/total': 3930560, 'tokens/trainable': 59448, 'epoch': '0.5078'} |
|
25%|██▌ | 130/512 [14:31<44:59, 7.07s/it]
26%|██▌ | 131/512 [14:38<43:55, 6.92s/it]
{'loss': '0.004909', 'grad_norm': '0.315', 'learning_rate': '9.007e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.93', 'tokens/train_per_sec_per_gpu': '32.28', 'tokens/total': 3960496, 'tokens/trainable': 59906, 'epoch': '0.5117'} |
|
26%|██▌ | 131/512 [14:38<43:55, 6.92s/it]
26%|██▌ | 132/512 [14:44<43:17, 6.84s/it]
{'loss': '0.005764', 'grad_norm': '0.1716', 'learning_rate': '8.988e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.48', 'tokens/total': 3990928, 'tokens/trainable': 60350, 'epoch': '0.5156'} |
|
26%|██▌ | 132/512 [14:44<43:17, 6.84s/it]
26%|██▌ | 133/512 [14:51<42:49, 6.78s/it]
{'loss': '0.002609', 'grad_norm': '0.5106', 'learning_rate': '8.97e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '36.41', 'tokens/total': 4021264, 'tokens/trainable': 60831, 'epoch': '0.5195'} |
|
26%|██▌ | 133/512 [14:51<42:49, 6.78s/it]
26%|██▌ | 134/512 [14:58<42:27, 6.74s/it]
{'loss': '0.005374', 'grad_norm': '0.173', 'learning_rate': '8.951e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '34.16', 'tokens/total': 4051728, 'tokens/trainable': 61305, 'epoch': '0.5234'} |
|
26%|██▌ | 134/512 [14:58<42:27, 6.74s/it]
26%|██▋ | 135/512 [15:04<42:19, 6.74s/it]
{'loss': '0.002464', 'grad_norm': '0.07392', 'learning_rate': '8.933e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '37.62', 'tokens/total': 4081920, 'tokens/trainable': 61792, 'epoch': '0.5273'} |
|
26%|██▋ | 135/512 [15:04<42:19, 6.74s/it]
27%|██▋ | 136/512 [15:11<42:07, 6.72s/it]
{'loss': '0.00858', 'grad_norm': '0.6117', 'learning_rate': '8.914e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '31.52', 'tokens/total': 4112304, 'tokens/trainable': 62253, 'epoch': '0.5312'} |
|
27%|██▋ | 136/512 [15:11<42:07, 6.72s/it]
27%|██▋ | 137/512 [15:18<41:54, 6.70s/it]
{'loss': '0.008749', 'grad_norm': '0.4993', 'learning_rate': '8.895e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '31.83', 'tokens/total': 4142512, 'tokens/trainable': 62707, 'epoch': '0.5352'} |
|
27%|██▋ | 137/512 [15:18<41:54, 6.70s/it]
27%|██▋ | 138/512 [15:24<41:37, 6.68s/it]
{'loss': '0.003255', 'grad_norm': '0.1438', 'learning_rate': '8.876e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '29.5', 'tokens/total': 4172992, 'tokens/trainable': 63145, 'epoch': '0.5391'} |
|
27%|██▋ | 138/512 [15:24<41:37, 6.68s/it]
27%|██▋ | 139/512 [15:31<41:26, 6.67s/it]
{'loss': '0.003425', 'grad_norm': '0.2352', 'learning_rate': '8.856e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '34.18', 'tokens/total': 4203136, 'tokens/trainable': 63587, 'epoch': '0.543'} |
|
27%|██▋ | 139/512 [15:31<41:26, 6.67s/it]
27%|██▋ | 140/512 [15:38<41:17, 6.66s/it]
{'loss': '0.004791', 'grad_norm': '0.3351', 'learning_rate': '8.837e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '37.03', 'tokens/total': 4233456, 'tokens/trainable': 64052, 'epoch': '0.5469'} |
|
27%|██▋ | 140/512 [15:38<41:17, 6.66s/it]
28%|██▊ | 141/512 [15:44<41:09, 6.66s/it]
{'loss': '0.004646', 'grad_norm': '0.2568', 'learning_rate': '8.817e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '34.62', 'tokens/total': 4263728, 'tokens/trainable': 64526, 'epoch': '0.5508'} |
|
28%|██▊ | 141/512 [15:44<41:09, 6.66s/it]
28%|██▊ | 142/512 [15:51<41:16, 6.69s/it]
{'loss': '0.0007791', 'grad_norm': '0.03925', 'learning_rate': '8.798e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '34.01', 'memory/max_allocated (GiB)': '34.01', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '34.83', 'tokens/total': 4294432, 'tokens/trainable': 64983, 'epoch': '0.5547'} |
|
28%|██▊ | 142/512 [15:51<41:16, 6.69s/it]
28%|██▊ | 143/512 [15:58<41:22, 6.73s/it]
{'loss': '0.01175', 'grad_norm': '0.2485', 'learning_rate': '8.778e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '37.46', 'tokens/total': 4324960, 'tokens/trainable': 65492, 'epoch': '0.5586'} |
|
28%|██▊ | 143/512 [15:58<41:22, 6.73s/it]
28%|██▊ | 144/512 [16:04<41:03, 6.70s/it]
{'loss': '0.009142', 'grad_norm': '0.4652', 'learning_rate': '8.758e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '35.81', 'tokens/total': 4355424, 'tokens/trainable': 65925, 'epoch': '0.5625'} |
|
28%|██▊ | 144/512 [16:04<41:03, 6.70s/it]
28%|██▊ | 145/512 [16:11<40:52, 6.68s/it]
{'loss': '0.004863', 'grad_norm': '0.1651', 'learning_rate': '8.738e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '31.56', 'tokens/total': 4385712, 'tokens/trainable': 66383, 'epoch': '0.5664'} |
|
28%|██▊ | 145/512 [16:11<40:52, 6.68s/it]
29%|██▊ | 146/512 [16:18<40:43, 6.68s/it]
{'loss': '0.0116', 'grad_norm': '0.4035', 'learning_rate': '8.718e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.55', 'tokens/train_per_sec_per_gpu': '31.56', 'tokens/total': 4416016, 'tokens/trainable': 66840, 'epoch': '0.5703'} |
|
29%|██▊ | 146/512 [16:18<40:43, 6.68s/it]
29%|██▊ | 147/512 [16:24<40:32, 6.66s/it]
{'loss': '0.0209', 'grad_norm': '0.6244', 'learning_rate': '8.697e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '31.94', 'tokens/total': 4446368, 'tokens/trainable': 67297, 'epoch': '0.5742'} |
|
29%|██▊ | 147/512 [16:24<40:32, 6.66s/it]
29%|██▉ | 148/512 [16:31<40:27, 6.67s/it]
{'loss': '0.001415', 'grad_norm': '0.1233', 'learning_rate': '8.677e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '37.58', 'tokens/total': 4476880, 'tokens/trainable': 67769, 'epoch': '0.5781'} |
|
29%|██▉ | 148/512 [16:31<40:27, 6.67s/it]
29%|██▉ | 149/512 [16:38<40:19, 6.67s/it]
{'loss': '0.002762', 'grad_norm': '0.2333', 'learning_rate': '8.656e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '33.05', 'tokens/total': 4507232, 'tokens/trainable': 68257, 'epoch': '0.582'} |
|
29%|██▉ | 149/512 [16:38<40:19, 6.67s/it]
29%|██▉ | 150/512 [16:44<40:06, 6.65s/it]
{'loss': '0.005425', 'grad_norm': '0.427', 'learning_rate': '8.635e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '35.73', 'tokens/total': 4537376, 'tokens/trainable': 68706, 'epoch': '0.5859'} |
|
29%|██▉ | 150/512 [16:44<40:06, 6.65s/it]
29%|██▉ | 151/512 [16:51<39:58, 6.64s/it]
{'loss': '0.0003616', 'grad_norm': '0.04172', 'learning_rate': '8.615e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '33.49', 'tokens/total': 4567584, 'tokens/trainable': 69131, 'epoch': '0.5898'} |
|
29%|██▉ | 151/512 [16:51<39:58, 6.64s/it]
30%|██▉ | 152/512 [16:58<39:50, 6.64s/it]
{'loss': '0.01979', 'grad_norm': '0.4934', 'learning_rate': '8.594e-05', 'ppl': '1.02', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '31.98', 'tokens/total': 4597792, 'tokens/trainable': 69586, 'epoch': '0.5938'} |
|
30%|██▉ | 152/512 [16:58<39:50, 6.64s/it]
30%|██▉ | 153/512 [17:04<39:52, 6.67s/it]
{'loss': '0.01604', 'grad_norm': '0.4899', 'learning_rate': '8.572e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '32.72', 'tokens/total': 4627936, 'tokens/trainable': 70061, 'epoch': '0.5977'} |
|
30%|██▉ | 153/512 [17:04<39:52, 6.67s/it]
30%|███ | 154/512 [17:11<39:41, 6.65s/it]
{'loss': '0.01073', 'grad_norm': '0.7293', 'learning_rate': '8.551e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '28.1', 'tokens/total': 4658352, 'tokens/trainable': 70467, 'epoch': '0.6016'} |
|
30%|███ | 154/512 [17:11<39:41, 6.65s/it]
30%|███ | 155/512 [17:18<39:37, 6.66s/it]
{'loss': '0.001413', 'grad_norm': '0.0669', 'learning_rate': '8.53e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 4688896, 'tokens/trainable': 70933, 'epoch': '0.6055'} |
|
30%|███ | 155/512 [17:18<39:37, 6.66s/it]
30%|███ | 156/512 [17:24<39:31, 6.66s/it]
{'loss': '0.00884', 'grad_norm': '0.4426', 'learning_rate': '8.508e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '34.5', 'tokens/total': 4719168, 'tokens/trainable': 71382, 'epoch': '0.6094'} |
|
30%|███ | 156/512 [17:24<39:31, 6.66s/it]
31%|███ | 157/512 [17:31<39:14, 6.63s/it]
{'loss': '0.002818', 'grad_norm': '0.1966', 'learning_rate': '8.487e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '30.93', 'tokens/total': 4749136, 'tokens/trainable': 71808, 'epoch': '0.6133'} |
|
31%|███ | 157/512 [17:31<39:14, 6.63s/it]
31%|███ | 158/512 [17:37<39:10, 6.64s/it]
{'loss': '0.0154', 'grad_norm': '0.557', 'learning_rate': '8.465e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '33.05', 'tokens/total': 4779472, 'tokens/trainable': 72249, 'epoch': '0.6172'} |
|
31%|███ | 158/512 [17:37<39:10, 6.64s/it]
31%|███ | 159/512 [17:44<39:04, 6.64s/it]
{'loss': '0.008824', 'grad_norm': '0.2506', 'learning_rate': '8.443e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '35.36', 'tokens/total': 4809632, 'tokens/trainable': 72726, 'epoch': '0.6211'} |
|
31%|███ | 159/512 [17:44<39:04, 6.64s/it]
31%|███▏ | 160/512 [17:51<38:57, 6.64s/it]
{'loss': '0.003945', 'grad_norm': '0.1482', 'learning_rate': '8.421e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '32.87', 'tokens/total': 4840112, 'tokens/trainable': 73148, 'epoch': '0.625'} |
|
31%|███▏ | 160/512 [17:51<38:57, 6.64s/it][2026-08-18 14:49:11,834] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-160 |
|
31%|███▏ | 161/512 [17:59<42:15, 7.22s/it]
{'loss': '0.00313', 'grad_norm': '0.09184', 'learning_rate': '8.399e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '34.04', 'memory/max_allocated (GiB)': '34.04', 'memory/device_reserved (GiB)': '38.65', 'tokens/train_per_sec_per_gpu': '31.47', 'tokens/total': 4870656, 'tokens/trainable': 73600, 'epoch': '0.6289'} |
|
31%|███▏ | 161/512 [17:59<42:15, 7.22s/it]
32%|███▏ | 162/512 [18:06<41:05, 7.04s/it]
{'loss': '0.00897', 'grad_norm': '0.2', 'learning_rate': '8.376e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.66', 'memory/max_allocated (GiB)': '33.66', 'memory/device_reserved (GiB)': '35.74', 'tokens/train_per_sec_per_gpu': '35.65', 'tokens/total': 4900608, 'tokens/trainable': 74068, 'epoch': '0.6328'} |
|
32%|███▏ | 162/512 [18:06<41:05, 7.04s/it]
32%|███▏ | 163/512 [18:13<40:18, 6.93s/it]
{'loss': '0.002803', 'grad_norm': '0.2488', 'learning_rate': '8.354e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.74', 'tokens/train_per_sec_per_gpu': '35.71', 'tokens/total': 4930752, 'tokens/trainable': 74527, 'epoch': '0.6367'} |
|
32%|███▏ | 163/512 [18:13<40:18, 6.93s/it]
32%|███▏ | 164/512 [18:19<39:39, 6.84s/it]
{'loss': '0.007594', 'grad_norm': '0.5618', 'learning_rate': '8.332e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.74', 'tokens/train_per_sec_per_gpu': '32.03', 'tokens/total': 4961008, 'tokens/trainable': 74992, 'epoch': '0.6406'} |
|
32%|███▏ | 164/512 [18:19<39:39, 6.84s/it]
32%|███▏ | 165/512 [18:26<39:13, 6.78s/it]
{'loss': '0.001276', 'grad_norm': '0.05916', 'learning_rate': '8.309e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.78', 'tokens/train_per_sec_per_gpu': '35.47', 'tokens/total': 4991200, 'tokens/trainable': 75442, 'epoch': '0.6445'} |
|
32%|███▏ | 165/512 [18:26<39:13, 6.78s/it]
32%|███▏ | 166/512 [18:33<38:57, 6.76s/it]
{'loss': '0.01418', 'grad_norm': '0.5386', 'learning_rate': '8.286e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '31.4', 'tokens/total': 5021600, 'tokens/trainable': 75887, 'epoch': '0.6484'} |
|
32%|███▏ | 166/512 [18:33<38:57, 6.76s/it]
33%|███▎ | 167/512 [18:39<38:39, 6.72s/it]
{'loss': '0.01045', 'grad_norm': '0.8136', 'learning_rate': '8.263e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.32', 'tokens/total': 5052032, 'tokens/trainable': 76331, 'epoch': '0.6523'} |
|
33%|███▎ | 167/512 [18:39<38:39, 6.72s/it]
33%|███▎ | 168/512 [18:45<37:33, 6.55s/it]
{'loss': '0.0001703', 'grad_norm': '0.01064', 'learning_rate': '8.24e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '36.1', 'tokens/total': 5080256, 'tokens/trainable': 76745, 'epoch': '0.6562'} |
|
33%|███▎ | 168/512 [18:45<37:33, 6.55s/it]
33%|███▎ | 169/512 [18:52<37:34, 6.57s/it]
{'loss': '0.002176', 'grad_norm': '0.2484', 'learning_rate': '8.217e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.44', 'tokens/total': 5110784, 'tokens/trainable': 77201, 'epoch': '0.6602'} |
|
33%|███▎ | 169/512 [18:52<37:34, 6.57s/it]
33%|███▎ | 170/512 [18:59<37:38, 6.60s/it]
{'loss': '0.0009023', 'grad_norm': '0.08222', 'learning_rate': '8.194e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '33.41', 'tokens/total': 5141200, 'tokens/trainable': 77655, 'epoch': '0.6641'} |
|
33%|███▎ | 170/512 [18:59<37:38, 6.60s/it]
33%|███▎ | 171/512 [19:05<37:38, 6.62s/it]
{'loss': '0.0004278', 'grad_norm': '0.03091', 'learning_rate': '8.171e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '39.29', 'tokens/total': 5171760, 'tokens/trainable': 78122, 'epoch': '0.668'} |
|
33%|███▎ | 171/512 [19:05<37:38, 6.62s/it]
34%|███▎ | 172/512 [19:12<37:36, 6.64s/it]
{'loss': '0.004756', 'grad_norm': '0.2397', 'learning_rate': '8.147e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '33.75', 'tokens/total': 5202272, 'tokens/trainable': 78576, 'epoch': '0.6719'} |
|
34%|███▎ | 172/512 [19:12<37:36, 6.64s/it]
34%|███▍ | 173/512 [19:19<37:31, 6.64s/it]
{'loss': '0.01252', 'grad_norm': '3.947', 'learning_rate': '8.124e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '36.7', 'tokens/total': 5232800, 'tokens/trainable': 79059, 'epoch': '0.6758'} |
|
34%|███▍ | 173/512 [19:19<37:31, 6.64s/it]
34%|███▍ | 174/512 [19:25<37:16, 6.62s/it]
{'loss': '0.00105', 'grad_norm': '0.07857', 'learning_rate': '8.1e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.66', 'memory/max_allocated (GiB)': '33.66', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '29.9', 'tokens/total': 5262672, 'tokens/trainable': 79498, 'epoch': '0.6797'} |
|
34%|███▍ | 174/512 [19:25<37:16, 6.62s/it]
34%|███▍ | 175/512 [19:32<37:07, 6.61s/it]
{'loss': '0.0003683', 'grad_norm': '0.02755', 'learning_rate': '8.076e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '30.63', 'tokens/total': 5293072, 'tokens/trainable': 79922, 'epoch': '0.6836'} |
|
34%|███▍ | 175/512 [19:32<37:07, 6.61s/it]
34%|███▍ | 176/512 [19:38<36:12, 6.47s/it]
{'loss': '0.0009432', 'grad_norm': '0.1078', 'learning_rate': '8.053e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.46', 'memory/max_allocated (GiB)': '33.46', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.9', 'tokens/total': 5321520, 'tokens/trainable': 80383, 'epoch': '0.6875'} |
|
34%|███▍ | 176/512 [19:38<36:12, 6.47s/it]
35%|███▍ | 177/512 [19:45<36:23, 6.52s/it]
{'loss': '0.004537', 'grad_norm': '0.4462', 'learning_rate': '8.029e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '33.29', 'tokens/total': 5351904, 'tokens/trainable': 80824, 'epoch': '0.6914'} |
|
35%|███▍ | 177/512 [19:45<36:23, 6.52s/it]
35%|███▍ | 178/512 [19:51<36:25, 6.54s/it]
{'loss': '0.01462', 'grad_norm': '0.3815', 'learning_rate': '8.005e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '32.44', 'tokens/total': 5382432, 'tokens/trainable': 81243, 'epoch': '0.6953'} |
|
35%|███▍ | 178/512 [19:51<36:25, 6.54s/it]
35%|███▍ | 179/512 [19:58<36:34, 6.59s/it]
{'loss': '0.002184', 'grad_norm': '0.2383', 'learning_rate': '7.98e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.19', 'tokens/total': 5412960, 'tokens/trainable': 81716, 'epoch': '0.6992'} |
|
35%|███▍ | 179/512 [19:58<36:34, 6.59s/it]
35%|███▌ | 180/512 [20:05<36:38, 6.62s/it]
{'loss': '0.002834', 'grad_norm': '0.2181', 'learning_rate': '7.956e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.7', 'tokens/total': 5443232, 'tokens/trainable': 82181, 'epoch': '0.7031'} |
|
35%|███▌ | 180/512 [20:05<36:38, 6.62s/it]
35%|███▌ | 181/512 [20:11<35:44, 6.48s/it]
{'loss': '0.01131', 'grad_norm': '0.4715', 'learning_rate': '7.932e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.29', 'memory/max_allocated (GiB)': '33.29', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.82', 'tokens/total': 5471440, 'tokens/trainable': 82626, 'epoch': '0.707'} |
|
35%|███▌ | 181/512 [20:11<35:44, 6.48s/it]
36%|███▌ | 182/512 [20:17<35:55, 6.53s/it]
{'loss': '0.001027', 'grad_norm': '0.07458', 'learning_rate': '7.907e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.34', 'tokens/total': 5501744, 'tokens/trainable': 83089, 'epoch': '0.7109'} |
|
36%|███▌ | 182/512 [20:17<35:55, 6.53s/it]
36%|███▌ | 183/512 [20:24<36:02, 6.57s/it]
{'loss': '0.008842', 'grad_norm': '0.3087', 'learning_rate': '7.883e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.53', 'tokens/total': 5532272, 'tokens/trainable': 83590, 'epoch': '0.7148'} |
|
36%|███▌ | 183/512 [20:24<36:02, 6.57s/it]
36%|███▌ | 184/512 [20:31<36:06, 6.61s/it]
{'loss': '0.0107', 'grad_norm': '0.3023', 'learning_rate': '7.858e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.48', 'tokens/total': 5562784, 'tokens/trainable': 84046, 'epoch': '0.7188'} |
|
36%|███▌ | 184/512 [20:31<36:06, 6.61s/it]
36%|███▌ | 185/512 [20:37<35:59, 6.60s/it]
{'loss': '0.001826', 'grad_norm': '0.5271', 'learning_rate': '7.833e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.27', 'tokens/total': 5592800, 'tokens/trainable': 84517, 'epoch': '0.7227'} |
|
36%|███▌ | 185/512 [20:37<35:59, 6.60s/it]
36%|███▋ | 186/512 [20:44<35:59, 6.63s/it]
{'loss': '0.01045', 'grad_norm': '0.4302', 'learning_rate': '7.808e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.15', 'tokens/total': 5623088, 'tokens/trainable': 84974, 'epoch': '0.7266'} |
|
36%|███▋ | 186/512 [20:44<35:59, 6.63s/it]
37%|███▋ | 187/512 [20:51<35:59, 6.64s/it]
{'loss': '0.002091', 'grad_norm': '0.1001', 'learning_rate': '7.783e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '35.31', 'tokens/total': 5653344, 'tokens/trainable': 85460, 'epoch': '0.7305'} |
|
37%|███▋ | 187/512 [20:51<35:59, 6.64s/it]
37%|███▋ | 188/512 [20:57<35:53, 6.65s/it]
{'loss': '0.01313', 'grad_norm': '0.2164', 'learning_rate': '7.758e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '36.25', 'tokens/total': 5683600, 'tokens/trainable': 85939, 'epoch': '0.7344'} |
|
37%|███▋ | 188/512 [20:57<35:53, 6.65s/it]
37%|███▋ | 189/512 [21:03<34:53, 6.48s/it]
{'loss': '0.004088', 'grad_norm': '0.1495', 'learning_rate': '7.733e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '38.68', 'tokens/total': 5711952, 'tokens/trainable': 86377, 'epoch': '0.7383'} |
|
37%|███▋ | 189/512 [21:03<34:53, 6.48s/it]
37%|███▋ | 190/512 [21:10<35:03, 6.53s/it]
{'loss': '0.006507', 'grad_norm': '0.2597', 'learning_rate': '7.708e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.13', 'tokens/total': 5742336, 'tokens/trainable': 86834, 'epoch': '0.7422'} |
|
37%|███▋ | 190/512 [21:10<35:03, 6.53s/it]
37%|███▋ | 191/512 [21:17<35:04, 6.56s/it]
{'loss': '0.0086', 'grad_norm': '0.3106', 'learning_rate': '7.683e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '32.25', 'tokens/total': 5772560, 'tokens/trainable': 87265, 'epoch': '0.7461'} |
|
37%|███▋ | 191/512 [21:17<35:04, 6.56s/it]
38%|███▊ | 192/512 [21:23<35:14, 6.61s/it]
{'loss': '0.003339', 'grad_norm': '0.1021', 'learning_rate': '7.657e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '34.34', 'tokens/total': 5803136, 'tokens/trainable': 87747, 'epoch': '0.75'} |
|
38%|███▊ | 192/512 [21:23<35:14, 6.61s/it][2026-08-18 14:52:44,530] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-192 |
|
38%|███▊ | 193/512 [21:32<38:37, 7.27s/it]
{'loss': '0.002802', 'grad_norm': '0.2163', 'learning_rate': '7.632e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.57', 'tokens/train_per_sec_per_gpu': '36.37', 'tokens/total': 5833440, 'tokens/trainable': 88239, 'epoch': '0.7539'} |
|
38%|███▊ | 193/512 [21:32<38:37, 7.27s/it]
38%|███▊ | 194/512 [21:39<37:32, 7.08s/it]
{'loss': '0.001716', 'grad_norm': '0.1151', 'learning_rate': '7.606e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.56', 'tokens/train_per_sec_per_gpu': '36.49', 'tokens/total': 5863552, 'tokens/trainable': 88718, 'epoch': '0.7578'} |
|
38%|███▊ | 194/512 [21:39<37:32, 7.08s/it]
38%|███▊ | 195/512 [21:46<36:41, 6.94s/it]
{'loss': '0.008859', 'grad_norm': '0.435', 'learning_rate': '7.58e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.56', 'tokens/train_per_sec_per_gpu': '31.44', 'tokens/total': 5893680, 'tokens/trainable': 89129, 'epoch': '0.7617'} |
|
38%|███▊ | 195/512 [21:46<36:41, 6.94s/it]
38%|███▊ | 196/512 [21:52<36:02, 6.84s/it]
{'loss': '0.001412', 'grad_norm': '0.1091', 'learning_rate': '7.555e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.56', 'tokens/train_per_sec_per_gpu': '32.54', 'tokens/total': 5923744, 'tokens/trainable': 89567, 'epoch': '0.7656'} |
|
38%|███▊ | 196/512 [21:52<36:02, 6.84s/it]
38%|███▊ | 197/512 [21:59<35:34, 6.78s/it]
{'loss': '0.01045', 'grad_norm': '0.466', 'learning_rate': '7.529e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.57', 'tokens/train_per_sec_per_gpu': '30.53', 'tokens/total': 5954128, 'tokens/trainable': 89988, 'epoch': '0.7695'} |
|
38%|███▊ | 197/512 [21:59<35:34, 6.78s/it]
39%|███▊ | 198/512 [22:05<35:16, 6.74s/it]
{'loss': '0.0001136', 'grad_norm': '0.01106', 'learning_rate': '7.503e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.57', 'tokens/train_per_sec_per_gpu': '36.34', 'tokens/total': 5984352, 'tokens/trainable': 90434, 'epoch': '0.7734'} |
|
39%|███▊ | 198/512 [22:05<35:16, 6.74s/it]
39%|███▉ | 199/512 [22:12<35:02, 6.72s/it]
{'loss': '0.002169', 'grad_norm': '0.1384', 'learning_rate': '7.477e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.61', 'tokens/train_per_sec_per_gpu': '34.39', 'tokens/total': 6014704, 'tokens/trainable': 90913, 'epoch': '0.7773'} |
|
39%|███▉ | 199/512 [22:12<35:02, 6.72s/it]
39%|███▉ | 200/512 [22:19<34:44, 6.68s/it]
{'loss': '0.01668', 'grad_norm': '0.3563', 'learning_rate': '7.451e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '34.44', 'tokens/total': 6045056, 'tokens/trainable': 91355, 'epoch': '0.7812'} |
|
39%|███▉ | 200/512 [22:19<34:44, 6.68s/it]
39%|███▉ | 201/512 [22:25<34:46, 6.71s/it]
{'loss': '0.0003422', 'grad_norm': '0.0225', 'learning_rate': '7.424e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '37.52', 'tokens/total': 6075504, 'tokens/trainable': 91856, 'epoch': '0.7852'} |
|
39%|███▉ | 201/512 [22:25<34:46, 6.71s/it]
39%|███▉ | 202/512 [22:32<34:32, 6.68s/it]
{'loss': '0.004311', 'grad_norm': '0.3169', 'learning_rate': '7.398e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 6105904, 'tokens/trainable': 92311, 'epoch': '0.7891'} |
|
39%|███▉ | 202/512 [22:32<34:32, 6.68s/it]
40%|███▉ | 203/512 [22:39<34:25, 6.68s/it]
{'loss': '9.595e-05', 'grad_norm': '0.006777', 'learning_rate': '7.372e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '34.89', 'tokens/total': 6136272, 'tokens/trainable': 92764, 'epoch': '0.793'} |
|
40%|███▉ | 203/512 [22:39<34:25, 6.68s/it]
40%|███▉ | 204/512 [22:45<34:10, 6.66s/it]
{'loss': '0.003057', 'grad_norm': '0.2564', 'learning_rate': '7.345e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.65', 'memory/max_allocated (GiB)': '33.65', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.21', 'tokens/total': 6166336, 'tokens/trainable': 93227, 'epoch': '0.7969'} |
|
40%|███▉ | 204/512 [22:45<34:10, 6.66s/it]
40%|████ | 205/512 [22:52<33:59, 6.64s/it]
{'loss': '0.001098', 'grad_norm': '0.09476', 'learning_rate': '7.319e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.27', 'tokens/total': 6196720, 'tokens/trainable': 93675, 'epoch': '0.8008'} |
|
40%|████ | 205/512 [22:52<33:59, 6.64s/it]
40%|████ | 206/512 [22:59<33:58, 6.66s/it]
{'loss': '0.0003043', 'grad_norm': '0.01866', 'learning_rate': '7.292e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '32.53', 'tokens/total': 6227424, 'tokens/trainable': 94120, 'epoch': '0.8047'} |
|
40%|████ | 206/512 [22:59<33:58, 6.66s/it]
40%|████ | 207/512 [23:05<33:48, 6.65s/it]
{'loss': '0.0002365', 'grad_norm': '0.02226', 'learning_rate': '7.266e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '35.5', 'tokens/total': 6257616, 'tokens/trainable': 94557, 'epoch': '0.8086'} |
|
40%|████ | 207/512 [23:05<33:48, 6.65s/it]
41%|████ | 208/512 [23:12<33:45, 6.66s/it]
{'loss': '0.0003968', 'grad_norm': '0.03336', 'learning_rate': '7.239e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '40.8', 'tokens/total': 6288000, 'tokens/trainable': 95035, 'epoch': '0.8125'} |
|
41%|████ | 208/512 [23:12<33:45, 6.66s/it]
41%|████ | 209/512 [23:19<33:38, 6.66s/it]
{'loss': '0.000308', 'grad_norm': '0.0507', 'learning_rate': '7.212e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '37.68', 'tokens/total': 6318336, 'tokens/trainable': 95517, 'epoch': '0.8164'} |
|
41%|████ | 209/512 [23:19<33:38, 6.66s/it]
41%|████ | 210/512 [23:25<33:32, 6.67s/it]
{'loss': '0.001556', 'grad_norm': '0.1036', 'learning_rate': '7.185e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '36.59', 'tokens/total': 6348656, 'tokens/trainable': 96026, 'epoch': '0.8203'} |
|
41%|████ | 210/512 [23:25<33:32, 6.67s/it]
41%|████ | 211/512 [23:32<33:25, 6.66s/it]
{'loss': '3.462e-05', 'grad_norm': '0.001788', 'learning_rate': '7.158e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '35.62', 'tokens/total': 6379040, 'tokens/trainable': 96477, 'epoch': '0.8242'} |
|
41%|████ | 211/512 [23:32<33:25, 6.66s/it]
41%|████▏ | 212/512 [23:39<33:19, 6.67s/it]
{'loss': '0.000558', 'grad_norm': '0.05056', 'learning_rate': '7.131e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.12', 'tokens/total': 6409312, 'tokens/trainable': 96951, 'epoch': '0.8281'} |
|
41%|████▏ | 212/512 [23:39<33:19, 6.67s/it]
42%|████▏ | 213/512 [23:45<33:10, 6.66s/it]
{'loss': '0.001203', 'grad_norm': '0.1382', 'learning_rate': '7.104e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '30.88', 'tokens/total': 6439520, 'tokens/trainable': 97350, 'epoch': '0.832'} |
|
42%|████▏ | 213/512 [23:45<33:10, 6.66s/it]
42%|████▏ | 214/512 [23:52<33:02, 6.65s/it]
{'loss': '0.001068', 'grad_norm': '0.2137', 'learning_rate': '7.077e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '29.99', 'tokens/total': 6469888, 'tokens/trainable': 97792, 'epoch': '0.8359'} |
|
42%|████▏ | 214/512 [23:52<33:02, 6.65s/it]
42%|████▏ | 215/512 [23:59<32:56, 6.65s/it]
{'loss': '0.001371', 'grad_norm': '0.1477', 'learning_rate': '7.05e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '36.63', 'tokens/total': 6500128, 'tokens/trainable': 98257, 'epoch': '0.8398'} |
|
42%|████▏ | 215/512 [23:59<32:56, 6.65s/it]
42%|████▏ | 216/512 [24:05<32:57, 6.68s/it]
{'loss': '0.001662', 'grad_norm': '0.08311', 'learning_rate': '7.022e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '34.03', 'memory/max_allocated (GiB)': '34.03', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '32.8', 'tokens/total': 6530832, 'tokens/trainable': 98731, 'epoch': '0.8438'} |
|
42%|████▏ | 216/512 [24:05<32:57, 6.68s/it]
42%|████▏ | 217/512 [24:12<32:50, 6.68s/it]
{'loss': '0.000104', 'grad_norm': '0.02766', 'learning_rate': '6.995e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '36.57', 'tokens/total': 6561248, 'tokens/trainable': 99212, 'epoch': '0.8477'} |
|
42%|████▏ | 217/512 [24:12<32:50, 6.68s/it]
43%|████▎ | 218/512 [24:19<32:37, 6.66s/it]
{'loss': '0.003625', 'grad_norm': '0.5895', 'learning_rate': '6.968e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.16', 'tokens/total': 6591648, 'tokens/trainable': 99654, 'epoch': '0.8516'} |
|
43%|████▎ | 218/512 [24:19<32:37, 6.66s/it]
43%|████▎ | 219/512 [24:25<32:34, 6.67s/it]
{'loss': '0.002938', 'grad_norm': '0.3624', 'learning_rate': '6.94e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '32.97', 'tokens/total': 6622368, 'tokens/trainable': 100086, 'epoch': '0.8555'} |
|
43%|████▎ | 219/512 [24:25<32:34, 6.67s/it]
43%|████▎ | 220/512 [24:32<32:26, 6.67s/it]
{'loss': '0.006389', 'grad_norm': '0.5691', 'learning_rate': '6.913e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.35', 'tokens/total': 6652848, 'tokens/trainable': 100524, 'epoch': '0.8594'} |
|
43%|████▎ | 220/512 [24:32<32:26, 6.67s/it]
43%|████▎ | 221/512 [24:39<32:26, 6.69s/it]
{'loss': '0.00503', 'grad_norm': '0.627', 'learning_rate': '6.885e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.52', 'tokens/total': 6683392, 'tokens/trainable': 100996, 'epoch': '0.8633'} |
|
43%|████▎ | 221/512 [24:39<32:26, 6.69s/it]
43%|████▎ | 222/512 [24:45<32:15, 6.68s/it]
{'loss': '0.000453', 'grad_norm': '0.06276', 'learning_rate': '6.857e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '32.07', 'tokens/total': 6713712, 'tokens/trainable': 101449, 'epoch': '0.8672'} |
|
43%|████▎ | 222/512 [24:45<32:15, 6.68s/it]
44%|████▎ | 223/512 [24:52<32:05, 6.66s/it]
{'loss': '0.0007312', 'grad_norm': '0.1305', 'learning_rate': '6.83e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '33.19', 'tokens/total': 6744176, 'tokens/trainable': 101905, 'epoch': '0.8711'} |
|
44%|████▎ | 223/512 [24:52<32:05, 6.66s/it]
44%|████▍ | 224/512 [24:59<32:01, 6.67s/it]
{'loss': '0.01871', 'grad_norm': '0.7504', 'learning_rate': '6.802e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '36.6', 'tokens/total': 6774416, 'tokens/trainable': 102411, 'epoch': '0.875'} |
|
44%|████▍ | 224/512 [24:59<32:01, 6.67s/it][2026-08-18 14:56:19,762] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-224 |
|
44%|████▍ | 225/512 [25:07<34:25, 7.20s/it]
{'loss': '0.0004673', 'grad_norm': '0.3213', 'learning_rate': '6.774e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '37.19', 'tokens/train_per_sec_per_gpu': '34.07', 'tokens/total': 6804592, 'tokens/trainable': 102882, 'epoch': '0.8789'} |
|
44%|████▍ | 225/512 [25:07<34:25, 7.20s/it]
44%|████▍ | 226/512 [25:14<33:32, 7.04s/it]
{'loss': '0.0006536', 'grad_norm': '0.07998', 'learning_rate': '6.746e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.49', 'tokens/train_per_sec_per_gpu': '34.67', 'tokens/total': 6834976, 'tokens/trainable': 103332, 'epoch': '0.8828'} |
|
44%|████▍ | 226/512 [25:14<33:32, 7.04s/it]
44%|████▍ | 227/512 [25:20<32:55, 6.93s/it]
{'loss': '0.0005653', 'grad_norm': '0.07349', 'learning_rate': '6.718e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.3', 'tokens/total': 6865408, 'tokens/trainable': 103790, 'epoch': '0.8867'} |
|
44%|████▍ | 227/512 [25:20<32:55, 6.93s/it]
45%|████▍ | 228/512 [25:27<32:24, 6.85s/it]
{'loss': '0.0001189', 'grad_norm': '0.01614', 'learning_rate': '6.69e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '34.46', 'tokens/total': 6895664, 'tokens/trainable': 104246, 'epoch': '0.8906'} |
|
45%|████▍ | 228/512 [25:27<32:24, 6.85s/it]
45%|████▍ | 229/512 [25:34<32:02, 6.79s/it]
{'loss': '0.0114', 'grad_norm': '0.4913', 'learning_rate': '6.662e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '37.82', 'tokens/total': 6926016, 'tokens/trainable': 104707, 'epoch': '0.8945'} |
|
45%|████▍ | 229/512 [25:34<32:02, 6.79s/it]
45%|████▍ | 230/512 [25:40<31:45, 6.76s/it]
{'loss': '0.0004522', 'grad_norm': '0.04466', 'learning_rate': '6.634e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '37.02', 'tokens/total': 6956384, 'tokens/trainable': 105167, 'epoch': '0.8984'} |
|
45%|████▍ | 230/512 [25:40<31:45, 6.76s/it]
45%|████▌ | 231/512 [25:47<31:26, 6.71s/it]
{'loss': '7.42e-05', 'grad_norm': '0.004377', 'learning_rate': '6.606e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.3', 'tokens/total': 6986768, 'tokens/trainable': 105576, 'epoch': '0.9023'} |
|
45%|████▌ | 231/512 [25:47<31:26, 6.71s/it]
45%|████▌ | 232/512 [25:54<31:12, 6.69s/it]
{'loss': '0.001178', 'grad_norm': '0.09353', 'learning_rate': '6.578e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '36.98', 'tokens/total': 7017232, 'tokens/trainable': 106043, 'epoch': '0.9062'} |
|
45%|████▌ | 232/512 [25:54<31:12, 6.69s/it]
46%|████▌ | 233/512 [26:00<31:01, 6.67s/it]
{'loss': '8.011e-05', 'grad_norm': '0.005993', 'learning_rate': '6.55e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.38', 'tokens/total': 7047568, 'tokens/trainable': 106506, 'epoch': '0.9102'} |
|
46%|████▌ | 233/512 [26:00<31:01, 6.67s/it]
46%|████▌ | 234/512 [26:07<30:50, 6.66s/it]
{'loss': '0.005336', 'grad_norm': '0.3207', 'learning_rate': '6.522e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '34.32', 'tokens/total': 7077760, 'tokens/trainable': 106951, 'epoch': '0.9141'} |
|
46%|████▌ | 234/512 [26:07<30:50, 6.66s/it]
46%|████▌ | 235/512 [26:14<30:37, 6.63s/it]
{'loss': '0.0007566', 'grad_norm': '0.08276', 'learning_rate': '6.493e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '30.85', 'tokens/total': 7107760, 'tokens/trainable': 107382, 'epoch': '0.918'} |
|
46%|████▌ | 235/512 [26:14<30:37, 6.63s/it]
46%|████▌ | 236/512 [26:20<30:33, 6.64s/it]
{'loss': '0.0008389', 'grad_norm': '0.07326', 'learning_rate': '6.465e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33', 'tokens/total': 7138144, 'tokens/trainable': 107814, 'epoch': '0.9219'} |
|
46%|████▌ | 236/512 [26:20<30:33, 6.64s/it]
46%|████▋ | 237/512 [26:27<30:30, 6.66s/it]
{'loss': '0.003933', 'grad_norm': '0.3893', 'learning_rate': '6.437e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.28', 'tokens/total': 7168352, 'tokens/trainable': 108274, 'epoch': '0.9258'} |
|
46%|████▋ | 237/512 [26:27<30:30, 6.66s/it]
46%|████▋ | 238/512 [26:34<30:25, 6.66s/it]
{'loss': '0.009527', 'grad_norm': '0.643', 'learning_rate': '6.408e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '29.23', 'tokens/total': 7198624, 'tokens/trainable': 108714, 'epoch': '0.9297'} |
|
46%|████▋ | 238/512 [26:34<30:25, 6.66s/it]
47%|████▋ | 239/512 [26:40<30:19, 6.66s/it]
{'loss': '0.0005601', 'grad_norm': '0.09937', 'learning_rate': '6.38e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '36.92', 'tokens/total': 7229216, 'tokens/trainable': 109220, 'epoch': '0.9336'} |
|
47%|████▋ | 239/512 [26:40<30:19, 6.66s/it]
47%|████▋ | 240/512 [26:47<30:11, 6.66s/it]
{'loss': '0.0003478', 'grad_norm': '0.042', 'learning_rate': '6.351e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '32.21', 'tokens/total': 7259648, 'tokens/trainable': 109672, 'epoch': '0.9375'} |
|
47%|████▋ | 240/512 [26:47<30:11, 6.66s/it]
47%|████▋ | 241/512 [26:54<30:04, 6.66s/it]
{'loss': '0.009971', 'grad_norm': '0.9473', 'learning_rate': '6.323e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.32', 'tokens/total': 7289824, 'tokens/trainable': 110135, 'epoch': '0.9414'} |
|
47%|████▋ | 241/512 [26:54<30:04, 6.66s/it]
47%|████▋ | 242/512 [27:00<29:59, 6.66s/it]
{'loss': '0.001862', 'grad_norm': '0.1647', 'learning_rate': '6.294e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '30.72', 'tokens/total': 7320288, 'tokens/trainable': 110563, 'epoch': '0.9453'} |
|
47%|████▋ | 242/512 [27:00<29:59, 6.66s/it]
47%|████▋ | 243/512 [27:07<29:52, 6.66s/it]
{'loss': '0.006691', 'grad_norm': '0.4174', 'learning_rate': '6.266e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '37.22', 'tokens/total': 7350736, 'tokens/trainable': 111049, 'epoch': '0.9492'} |
|
47%|████▋ | 243/512 [27:07<29:52, 6.66s/it]
48%|████▊ | 244/512 [27:14<29:55, 6.70s/it]
{'loss': '0.007665', 'grad_norm': '0.6252', 'learning_rate': '6.237e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.47', 'tokens/total': 7381280, 'tokens/trainable': 111495, 'epoch': '0.9531'} |
|
48%|████▊ | 244/512 [27:14<29:55, 6.70s/it]
48%|████▊ | 245/512 [27:20<29:47, 6.69s/it]
{'loss': '0.005139', 'grad_norm': '0.268', 'learning_rate': '6.208e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '31.9', 'tokens/total': 7411632, 'tokens/trainable': 111968, 'epoch': '0.957'} |
|
48%|████▊ | 245/512 [27:20<29:47, 6.69s/it]
48%|████▊ | 246/512 [27:27<29:34, 6.67s/it]
{'loss': '0.0002723', 'grad_norm': '0.01653', 'learning_rate': '6.18e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '36.59', 'tokens/total': 7441904, 'tokens/trainable': 112416, 'epoch': '0.9609'} |
|
48%|████▊ | 246/512 [27:27<29:34, 6.67s/it]
48%|████▊ | 247/512 [27:34<29:32, 6.69s/it]
{'loss': '0.0002142', 'grad_norm': '0.008991', 'learning_rate': '6.151e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '34.19', 'tokens/total': 7472368, 'tokens/trainable': 112882, 'epoch': '0.9648'} |
|
48%|████▊ | 247/512 [27:34<29:32, 6.69s/it]
48%|████▊ | 248/512 [27:40<29:24, 6.68s/it]
{'loss': '0.01342', 'grad_norm': '0.5434', 'learning_rate': '6.122e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '40.29', 'tokens/total': 7502656, 'tokens/trainable': 113390, 'epoch': '0.9688'} |
|
48%|████▊ | 248/512 [27:40<29:24, 6.68s/it]
49%|████▊ | 249/512 [27:47<29:16, 6.68s/it]
{'loss': '0.004542', 'grad_norm': '0.5201', 'learning_rate': '6.093e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '37.97', 'tokens/total': 7532800, 'tokens/trainable': 113904, 'epoch': '0.9727'} |
|
49%|████▊ | 249/512 [27:47<29:16, 6.68s/it]
49%|████▉ | 250/512 [27:54<29:10, 6.68s/it]
{'loss': '0.001719', 'grad_norm': '0.07659', 'learning_rate': '6.065e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '34.08', 'tokens/total': 7563264, 'tokens/trainable': 114351, 'epoch': '0.9766'} |
|
49%|████▉ | 250/512 [27:54<29:10, 6.68s/it]
49%|████▉ | 251/512 [28:00<29:05, 6.69s/it]
{'loss': '0.005591', 'grad_norm': '0.301', 'learning_rate': '6.036e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '35.41', 'tokens/total': 7593840, 'tokens/trainable': 114842, 'epoch': '0.9805'} |
|
49%|████▉ | 251/512 [28:00<29:05, 6.69s/it]
49%|████▉ | 252/512 [28:07<28:55, 6.67s/it]
{'loss': '0.003722', 'grad_norm': '0.3811', 'learning_rate': '6.007e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '29.06', 'tokens/total': 7624416, 'tokens/trainable': 115263, 'epoch': '0.9844'} |
|
49%|████▉ | 252/512 [28:07<28:55, 6.67s/it]
49%|████▉ | 253/512 [28:14<28:47, 6.67s/it]
{'loss': '0.005348', 'grad_norm': '1.165', 'learning_rate': '5.978e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '32.88', 'tokens/total': 7654688, 'tokens/trainable': 115703, 'epoch': '0.9883'} |
|
49%|████▉ | 253/512 [28:14<28:47, 6.67s/it]
50%|████▉ | 254/512 [28:20<28:37, 6.66s/it]
{'loss': '0.001302', 'grad_norm': '0.1629', 'learning_rate': '5.949e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '33.34', 'tokens/total': 7685088, 'tokens/trainable': 116157, 'epoch': '0.9922'} |
|
50%|████▉ | 254/512 [28:20<28:37, 6.66s/it]
50%|████▉ | 255/512 [28:27<28:30, 6.66s/it]
{'loss': '0.00702', 'grad_norm': '0.4163', 'learning_rate': '5.92e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '32.03', 'tokens/total': 7715552, 'tokens/trainable': 116567, 'epoch': '0.9961'} |
|
50%|████▉ | 255/512 [28:27<28:30, 6.66s/it]
50%|█████ | 256/512 [28:34<28:35, 6.70s/it]
{'loss': '0.004288', 'grad_norm': '0.1324', 'learning_rate': '5.891e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '31.31', 'tokens/total': 7745872, 'tokens/trainable': 117030, 'epoch': '1'} |
|
50%|█████ | 256/512 [28:34<28:35, 6.70s/it][2026-08-18 14:59:54,872] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-256 |
|
50%|█████ | 257/512 [28:43<31:15, 7.35s/it]
{'loss': '0.0005614', 'grad_norm': '0.06508', 'learning_rate': '5.862e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.23', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 7776208, 'tokens/trainable': 117517, 'epoch': '1.004'} |
|
50%|█████ | 257/512 [28:43<31:15, 7.35s/it]
50%|█████ | 258/512 [28:49<30:16, 7.15s/it]
{'loss': '0.0003597', 'grad_norm': '0.02545', 'learning_rate': '5.834e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.62', 'tokens/train_per_sec_per_gpu': '38.46', 'tokens/total': 7806480, 'tokens/trainable': 118015, 'epoch': '1.008'} |
|
50%|█████ | 258/512 [28:49<30:16, 7.15s/it]
51%|█████ | 259/512 [28:56<29:32, 7.01s/it]
{'loss': '0.003857', 'grad_norm': '0.1873', 'learning_rate': '5.805e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.62', 'tokens/train_per_sec_per_gpu': '36.49', 'tokens/total': 7836800, 'tokens/trainable': 118520, 'epoch': '1.012'} |
|
51%|█████ | 259/512 [28:56<29:32, 7.01s/it]
51%|█████ | 260/512 [29:03<29:00, 6.91s/it]
{'loss': '0.006049', 'grad_norm': '0.3347', 'learning_rate': '5.776e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '36.42', 'tokens/total': 7867040, 'tokens/trainable': 118992, 'epoch': '1.016'} |
|
51%|█████ | 260/512 [29:03<29:00, 6.91s/it]
51%|█████ | 261/512 [29:09<28:33, 6.83s/it]
{'loss': '0.005414', 'grad_norm': '0.286', 'learning_rate': '5.747e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '34.03', 'tokens/total': 7897408, 'tokens/trainable': 119438, 'epoch': '1.02'} |
|
51%|█████ | 261/512 [29:09<28:33, 6.83s/it]
51%|█████ | 262/512 [29:16<28:10, 6.76s/it]
{'loss': '0.0003921', 'grad_norm': '0.02921', 'learning_rate': '5.718e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '32.4', 'tokens/total': 7927648, 'tokens/trainable': 119881, 'epoch': '1.023'} |
|
51%|█████ | 262/512 [29:16<28:10, 6.76s/it]
51%|█████▏ | 263/512 [29:23<27:51, 6.71s/it]
{'loss': '0.0001376', 'grad_norm': '0.007834', 'learning_rate': '5.689e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '28.04', 'tokens/total': 7957968, 'tokens/trainable': 120285, 'epoch': '1.027'} |
|
51%|█████▏ | 263/512 [29:23<27:51, 6.71s/it]
52%|█████▏ | 264/512 [29:29<27:36, 6.68s/it]
{'loss': '0.0008238', 'grad_norm': '0.05843', 'learning_rate': '5.66e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.65', 'memory/max_allocated (GiB)': '33.65', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '33.35', 'tokens/total': 7987888, 'tokens/trainable': 120755, 'epoch': '1.031'} |
|
52%|█████▏ | 264/512 [29:29<27:36, 6.68s/it]
52%|█████▏ | 265/512 [29:36<27:25, 6.66s/it]
{'loss': '0.0002014', 'grad_norm': '0.02505', 'learning_rate': '5.631e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '35.82', 'tokens/total': 8018048, 'tokens/trainable': 121194, 'epoch': '1.035'} |
|
52%|█████▏ | 265/512 [29:36<27:25, 6.66s/it]
52%|█████▏ | 266/512 [29:42<27:17, 6.66s/it]
{'loss': '0.004579', 'grad_norm': '0.1903', 'learning_rate': '5.602e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '36.87', 'tokens/total': 8048448, 'tokens/trainable': 121670, 'epoch': '1.039'} |
|
52%|█████▏ | 266/512 [29:42<27:17, 6.66s/it]
52%|█████▏ | 267/512 [29:49<27:14, 6.67s/it]
{'loss': '0.0007128', 'grad_norm': '0.06157', 'learning_rate': '5.573e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '33.86', 'tokens/total': 8078848, 'tokens/trainable': 122142, 'epoch': '1.043'} |
|
52%|█████▏ | 267/512 [29:49<27:14, 6.67s/it]
52%|█████▏ | 268/512 [29:56<27:11, 6.69s/it]
{'loss': '0.001657', 'grad_norm': '0.2028', 'learning_rate': '5.544e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '32.57', 'tokens/total': 8109360, 'tokens/trainable': 122585, 'epoch': '1.047'} |
|
52%|█████▏ | 268/512 [29:56<27:11, 6.69s/it]
53%|█████▎ | 269/512 [30:03<27:04, 6.68s/it]
{'loss': '0.002813', 'grad_norm': '0.9749', 'learning_rate': '5.515e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '39.11', 'tokens/total': 8139888, 'tokens/trainable': 123091, 'epoch': '1.051'} |
|
53%|█████▎ | 269/512 [30:03<27:04, 6.68s/it]
53%|█████▎ | 270/512 [30:09<27:03, 6.71s/it]
{'loss': '0.009424', 'grad_norm': '0.2498', 'learning_rate': '5.485e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '31.2', 'tokens/total': 8170480, 'tokens/trainable': 123505, 'epoch': '1.055'} |
|
53%|█████▎ | 270/512 [30:09<27:03, 6.71s/it]
53%|█████▎ | 271/512 [30:15<26:17, 6.54s/it]
{'loss': '0.005858', 'grad_norm': '0.3943', 'learning_rate': '5.456e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.41', 'memory/max_allocated (GiB)': '33.41', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '35.71', 'tokens/total': 8198848, 'tokens/trainable': 123953, 'epoch': '1.059'} |
|
53%|█████▎ | 271/512 [30:15<26:17, 6.54s/it]
53%|█████▎ | 272/512 [30:22<26:20, 6.58s/it]
{'loss': '0.002318', 'grad_norm': '0.1328', 'learning_rate': '5.427e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '28.92', 'tokens/total': 8229200, 'tokens/trainable': 124400, 'epoch': '1.062'} |
|
53%|█████▎ | 272/512 [30:22<26:20, 6.58s/it]
53%|█████▎ | 273/512 [30:29<26:18, 6.60s/it]
{'loss': '0.006285', 'grad_norm': '0.2422', 'learning_rate': '5.398e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '35.34', 'tokens/total': 8259536, 'tokens/trainable': 124870, 'epoch': '1.066'} |
|
53%|█████▎ | 273/512 [30:29<26:18, 6.60s/it]
54%|█████▎ | 274/512 [30:35<26:13, 6.61s/it]
{'loss': '0.00336', 'grad_norm': '0.1999', 'learning_rate': '5.369e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '32.31', 'tokens/total': 8289904, 'tokens/trainable': 125333, 'epoch': '1.07'} |
|
54%|█████▎ | 274/512 [30:35<26:13, 6.61s/it]
54%|█████▎ | 275/512 [30:42<26:11, 6.63s/it]
{'loss': '0.0003117', 'grad_norm': '0.04859', 'learning_rate': '5.34e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '36.81', 'tokens/total': 8320176, 'tokens/trainable': 125834, 'epoch': '1.074'} |
|
54%|█████▎ | 275/512 [30:42<26:11, 6.63s/it]
54%|█████▍ | 276/512 [30:49<26:07, 6.64s/it]
{'loss': '4.315e-05', 'grad_norm': '0.002488', 'learning_rate': '5.311e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '38.29', 'tokens/total': 8350832, 'tokens/trainable': 126316, 'epoch': '1.078'} |
|
54%|█████▍ | 276/512 [30:49<26:07, 6.64s/it]
54%|█████▍ | 277/512 [30:55<26:03, 6.65s/it]
{'loss': '0.0005475', 'grad_norm': '0.05523', 'learning_rate': '5.282e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '35.51', 'tokens/total': 8380976, 'tokens/trainable': 126798, 'epoch': '1.082'} |
|
54%|█████▍ | 277/512 [30:55<26:03, 6.65s/it]
54%|█████▍ | 278/512 [31:02<25:48, 6.62s/it]
{'loss': '0.0002244', 'grad_norm': '0.02387', 'learning_rate': '5.253e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '29.84', 'tokens/total': 8411072, 'tokens/trainable': 127223, 'epoch': '1.086'} |
|
54%|█████▍ | 278/512 [31:02<25:48, 6.62s/it]
54%|█████▍ | 279/512 [31:08<25:31, 6.57s/it]
{'loss': '0.00281', 'grad_norm': '0.236', 'learning_rate': '5.224e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.66', 'memory/max_allocated (GiB)': '33.66', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '36.51', 'tokens/total': 8440736, 'tokens/trainable': 127672, 'epoch': '1.09'} |
|
54%|█████▍ | 279/512 [31:08<25:31, 6.57s/it]
55%|█████▍ | 280/512 [31:15<25:25, 6.58s/it]
{'loss': '0.001704', 'grad_norm': '0.141', 'learning_rate': '5.195e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.69', 'memory/max_allocated (GiB)': '33.69', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '32.36', 'tokens/total': 8470944, 'tokens/trainable': 128116, 'epoch': '1.094'} |
|
55%|█████▍ | 280/512 [31:15<25:25, 6.58s/it]
55%|█████▍ | 281/512 [31:22<25:25, 6.60s/it]
{'loss': '0.001691', 'grad_norm': '0.1573', 'learning_rate': '5.166e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '33.02', 'tokens/total': 8501104, 'tokens/trainable': 128589, 'epoch': '1.098'} |
|
55%|█████▍ | 281/512 [31:22<25:25, 6.60s/it]
55%|█████▌ | 282/512 [31:28<25:21, 6.61s/it]
{'loss': '0.00846', 'grad_norm': '0.3039', 'learning_rate': '5.138e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '34.06', 'tokens/total': 8531696, 'tokens/trainable': 129036, 'epoch': '1.102'} |
|
55%|█████▌ | 282/512 [31:28<25:21, 6.61s/it]
55%|█████▌ | 283/512 [31:35<25:18, 6.63s/it]
{'loss': '0.0001751', 'grad_norm': '0.02127', 'learning_rate': '5.109e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '35.4', 'tokens/total': 8561936, 'tokens/trainable': 129485, 'epoch': '1.105'} |
|
55%|█████▌ | 283/512 [31:35<25:18, 6.63s/it]
55%|█████▌ | 284/512 [31:42<25:13, 6.64s/it]
{'loss': '0.0004621', 'grad_norm': '0.03597', 'learning_rate': '5.08e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '29.75', 'tokens/total': 8592192, 'tokens/trainable': 129920, 'epoch': '1.109'} |
|
55%|█████▌ | 284/512 [31:42<25:13, 6.64s/it]
56%|█████▌ | 285/512 [31:48<25:10, 6.66s/it]
{'loss': '0.0007257', 'grad_norm': '0.05809', 'learning_rate': '5.051e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '36.29', 'tokens/total': 8622432, 'tokens/trainable': 130417, 'epoch': '1.113'} |
|
56%|█████▌ | 285/512 [31:48<25:10, 6.66s/it]
56%|█████▌ | 286/512 [31:55<25:05, 6.66s/it]
{'loss': '4.458e-05', 'grad_norm': '0.003009', 'learning_rate': '5.022e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '36.09', 'tokens/total': 8652768, 'tokens/trainable': 130903, 'epoch': '1.117'} |
|
56%|█████▌ | 286/512 [31:55<25:05, 6.66s/it]
56%|█████▌ | 287/512 [32:02<24:57, 6.66s/it]
{'loss': '0.001948', 'grad_norm': '0.2359', 'learning_rate': '4.993e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '35.72', 'tokens/total': 8683296, 'tokens/trainable': 131343, 'epoch': '1.121'} |
|
56%|█████▌ | 287/512 [32:02<24:57, 6.66s/it]
56%|█████▋ | 288/512 [32:08<24:49, 6.65s/it]
{'loss': '0.0006973', 'grad_norm': '0.06522', 'learning_rate': '4.964e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '32.41', 'tokens/total': 8713504, 'tokens/trainable': 131763, 'epoch': '1.125'} |
|
56%|█████▋ | 288/512 [32:08<24:49, 6.65s/it][2026-08-18 15:03:29,354] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-288 |
|
56%|█████▋ | 289/512 [32:17<26:57, 7.25s/it]
{'loss': '2.373e-05', 'grad_norm': '0.001396', 'learning_rate': '4.935e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.98', 'tokens/train_per_sec_per_gpu': '34.28', 'tokens/total': 8743888, 'tokens/trainable': 132229, 'epoch': '1.129'} |
|
56%|█████▋ | 289/512 [32:17<26:57, 7.25s/it]
57%|█████▋ | 290/512 [32:24<26:09, 7.07s/it]
{'loss': '8.758e-06', 'grad_norm': '0.0002614', 'learning_rate': '4.907e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.54', 'tokens/train_per_sec_per_gpu': '35.66', 'tokens/total': 8774336, 'tokens/trainable': 132678, 'epoch': '1.133'} |
|
57%|█████▋ | 290/512 [32:24<26:09, 7.07s/it]
57%|█████▋ | 291/512 [32:30<25:36, 6.95s/it]
{'loss': '0.002511', 'grad_norm': '0.2678', 'learning_rate': '4.878e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '29.52', 'tokens/total': 8804816, 'tokens/trainable': 133136, 'epoch': '1.137'} |
|
57%|█████▋ | 291/512 [32:30<25:36, 6.95s/it]
57%|█████▋ | 292/512 [32:37<25:11, 6.87s/it]
{'loss': '4.595e-05', 'grad_norm': '0.005347', 'learning_rate': '4.849e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '35.51', 'tokens/total': 8835024, 'tokens/trainable': 133608, 'epoch': '1.141'} |
|
57%|█████▋ | 292/512 [32:37<25:11, 6.87s/it]
57%|█████▋ | 293/512 [32:44<24:50, 6.80s/it]
{'loss': '0.003174', 'grad_norm': '0.3414', 'learning_rate': '4.82e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '40.6', 'tokens/total': 8865408, 'tokens/trainable': 134108, 'epoch': '1.145'} |
|
57%|█████▋ | 293/512 [32:44<24:50, 6.80s/it]
57%|█████▋ | 294/512 [32:50<24:35, 6.77s/it]
{'loss': '4.493e-05', 'grad_norm': '0.02141', 'learning_rate': '4.792e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '30.82', 'tokens/total': 8895824, 'tokens/trainable': 134541, 'epoch': '1.148'} |
|
57%|█████▋ | 294/512 [32:50<24:35, 6.77s/it]
58%|█████▊ | 295/512 [32:57<24:21, 6.74s/it]
{'loss': '2.889e-05', 'grad_norm': '0.003497', 'learning_rate': '4.763e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '29.58', 'tokens/total': 8926336, 'tokens/trainable': 134966, 'epoch': '1.152'} |
|
58%|█████▊ | 295/512 [32:57<24:21, 6.74s/it]
58%|█████▊ | 296/512 [33:04<24:07, 6.70s/it]
{'loss': '0.0003167', 'grad_norm': '0.1155', 'learning_rate': '4.734e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '33.68', 'tokens/total': 8956656, 'tokens/trainable': 135402, 'epoch': '1.156'} |
|
58%|█████▊ | 296/512 [33:04<24:07, 6.70s/it]
58%|█████▊ | 297/512 [33:10<23:51, 6.66s/it]
{'loss': '0.00615', 'grad_norm': '0.3867', 'learning_rate': '4.706e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '30.21', 'tokens/total': 8986752, 'tokens/trainable': 135804, 'epoch': '1.16'} |
|
58%|█████▊ | 297/512 [33:10<23:51, 6.66s/it]
58%|█████▊ | 298/512 [33:17<23:39, 6.63s/it]
{'loss': '4.449e-05', 'grad_norm': '0.003796', 'learning_rate': '4.677e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '31.32', 'tokens/total': 9017024, 'tokens/trainable': 136231, 'epoch': '1.164'} |
|
58%|█████▊ | 298/512 [33:17<23:39, 6.63s/it]
58%|█████▊ | 299/512 [33:23<23:34, 6.64s/it]
{'loss': '1.627e-05', 'grad_norm': '0.001018', 'learning_rate': '4.649e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.55', 'tokens/train_per_sec_per_gpu': '34.4', 'tokens/total': 9047424, 'tokens/trainable': 136705, 'epoch': '1.168'} |
|
58%|█████▊ | 299/512 [33:23<23:34, 6.64s/it]
59%|█████▊ | 300/512 [33:30<23:25, 6.63s/it]
{'loss': '0.0002693', 'grad_norm': '0.03619', 'learning_rate': '4.62e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 9077648, 'tokens/trainable': 137178, 'epoch': '1.172'} |
|
59%|█████▊ | 300/512 [33:30<23:25, 6.63s/it]
59%|█████▉ | 301/512 [33:37<23:21, 6.64s/it]
{'loss': '0.0007543', 'grad_norm': '0.1538', 'learning_rate': '4.592e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '34.54', 'tokens/total': 9108032, 'tokens/trainable': 137643, 'epoch': '1.176'} |
|
59%|█████▉ | 301/512 [33:37<23:21, 6.64s/it]
59%|█████▉ | 302/512 [33:43<23:14, 6.64s/it]
{'loss': '0.002032', 'grad_norm': '0.2317', 'learning_rate': '4.563e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '32.07', 'tokens/total': 9138240, 'tokens/trainable': 138078, 'epoch': '1.18'} |
|
59%|█████▉ | 302/512 [33:43<23:14, 6.64s/it]
59%|█████▉ | 303/512 [33:50<23:05, 6.63s/it]
{'loss': '0.01421', 'grad_norm': '1.085', 'learning_rate': '4.535e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '35.27', 'tokens/total': 9168352, 'tokens/trainable': 138515, 'epoch': '1.184'} |
|
59%|█████▉ | 303/512 [33:50<23:05, 6.63s/it]
59%|█████▉ | 304/512 [33:57<23:02, 6.65s/it]
{'loss': '0.005428', 'grad_norm': '0.4308', 'learning_rate': '4.507e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '35.29', 'tokens/total': 9198864, 'tokens/trainable': 138948, 'epoch': '1.188'} |
|
59%|█████▉ | 304/512 [33:57<23:02, 6.65s/it]
60%|█████▉ | 305/512 [34:03<22:56, 6.65s/it]
{'loss': '0.008615', 'grad_norm': '0.575', 'learning_rate': '4.478e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '38.3', 'tokens/total': 9229104, 'tokens/trainable': 139408, 'epoch': '1.191'} |
|
60%|█████▉ | 305/512 [34:03<22:56, 6.65s/it]
60%|█████▉ | 306/512 [34:09<22:18, 6.50s/it]
{'loss': '1.476e-05', 'grad_norm': '0.0007629', 'learning_rate': '4.45e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.38', 'memory/max_allocated (GiB)': '33.38', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '33.57', 'tokens/total': 9257344, 'tokens/trainable': 139840, 'epoch': '1.195'} |
|
60%|█████▉ | 306/512 [34:09<22:18, 6.50s/it]
60%|█████▉ | 307/512 [34:16<22:21, 6.54s/it]
{'loss': '0.00831', 'grad_norm': '0.604', 'learning_rate': '4.422e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '33.1', 'tokens/total': 9287520, 'tokens/trainable': 140281, 'epoch': '1.199'} |
|
60%|█████▉ | 307/512 [34:16<22:21, 6.54s/it]
60%|██████ | 308/512 [34:23<22:23, 6.59s/it]
{'loss': '0.006753', 'grad_norm': '1.484', 'learning_rate': '4.394e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '37.42', 'tokens/total': 9317888, 'tokens/trainable': 140776, 'epoch': '1.203'} |
|
60%|██████ | 308/512 [34:23<22:23, 6.59s/it]
60%|██████ | 309/512 [34:29<22:20, 6.61s/it]
{'loss': '0.001527', 'grad_norm': '0.2579', 'learning_rate': '4.366e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.72', 'tokens/train_per_sec_per_gpu': '37', 'tokens/total': 9348256, 'tokens/trainable': 141228, 'epoch': '1.207'} |
|
60%|██████ | 309/512 [34:29<22:20, 6.61s/it]
61%|██████ | 310/512 [34:36<22:13, 6.60s/it]
{'loss': '0.0004693', 'grad_norm': '0.07557', 'learning_rate': '4.338e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.74', 'tokens/train_per_sec_per_gpu': '37.6', 'tokens/total': 9378560, 'tokens/trainable': 141679, 'epoch': '1.211'} |
|
61%|██████ | 310/512 [34:36<22:13, 6.60s/it]
61%|██████ | 311/512 [34:43<22:05, 6.60s/it]
{'loss': '0.001339', 'grad_norm': '0.1331', 'learning_rate': '4.31e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.74', 'tokens/train_per_sec_per_gpu': '32.53', 'tokens/total': 9408848, 'tokens/trainable': 142122, 'epoch': '1.215'} |
|
61%|██████ | 311/512 [34:43<22:05, 6.60s/it]
61%|██████ | 312/512 [34:49<22:01, 6.61s/it]
{'loss': '3.625e-05', 'grad_norm': '0.002221', 'learning_rate': '4.282e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.2', 'tokens/total': 9439344, 'tokens/trainable': 142561, 'epoch': '1.219'} |
|
61%|██████ | 312/512 [34:49<22:01, 6.61s/it]
61%|██████ | 313/512 [34:56<21:58, 6.63s/it]
{'loss': '0.006627', 'grad_norm': '0.6299', 'learning_rate': '4.254e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.82', 'tokens/total': 9469792, 'tokens/trainable': 143046, 'epoch': '1.223'} |
|
61%|██████ | 313/512 [34:56<21:58, 6.63s/it]
61%|██████▏ | 314/512 [35:02<21:52, 6.63s/it]
{'loss': '0.003692', 'grad_norm': '0.211', 'learning_rate': '4.226e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.15', 'tokens/total': 9500128, 'tokens/trainable': 143493, 'epoch': '1.227'} |
|
61%|██████▏ | 314/512 [35:02<21:52, 6.63s/it]
62%|██████▏ | 315/512 [35:09<21:45, 6.63s/it]
{'loss': '0.001636', 'grad_norm': '0.2086', 'learning_rate': '4.198e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.38', 'tokens/total': 9530480, 'tokens/trainable': 143956, 'epoch': '1.23'} |
|
62%|██████▏ | 315/512 [35:09<21:45, 6.63s/it]
62%|██████▏ | 316/512 [35:16<21:40, 6.63s/it]
{'loss': '0.01269', 'grad_norm': '0.6389', 'learning_rate': '4.17e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '34.13', 'tokens/total': 9560912, 'tokens/trainable': 144411, 'epoch': '1.234'} |
|
62%|██████▏ | 316/512 [35:16<21:40, 6.63s/it]
62%|██████▏ | 317/512 [35:22<21:35, 6.64s/it]
{'loss': '0.0009713', 'grad_norm': '0.08197', 'learning_rate': '4.143e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '38.88', 'tokens/total': 9591344, 'tokens/trainable': 144892, 'epoch': '1.238'} |
|
62%|██████▏ | 317/512 [35:22<21:35, 6.64s/it]
62%|██████▏ | 318/512 [35:29<21:26, 6.63s/it]
{'loss': '0.001025', 'grad_norm': '0.08259', 'learning_rate': '4.115e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '37.95', 'tokens/train_per_sec_per_gpu': '32.83', 'tokens/total': 9621392, 'tokens/trainable': 145339, 'epoch': '1.242'} |
|
62%|██████▏ | 318/512 [35:29<21:26, 6.63s/it]
62%|██████▏ | 319/512 [35:36<21:24, 6.65s/it]
{'loss': '0.001894', 'grad_norm': '0.122', 'learning_rate': '4.087e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '37.95', 'tokens/train_per_sec_per_gpu': '36.57', 'tokens/total': 9651728, 'tokens/trainable': 145811, 'epoch': '1.246'} |
|
62%|██████▏ | 319/512 [35:36<21:24, 6.65s/it]
62%|██████▎ | 320/512 [35:42<21:20, 6.67s/it]
{'loss': '0.001203', 'grad_norm': '0.07121', 'learning_rate': '4.06e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '37.95', 'tokens/train_per_sec_per_gpu': '38.93', 'tokens/total': 9682224, 'tokens/trainable': 146314, 'epoch': '1.25'} |
|
62%|██████▎ | 320/512 [35:42<21:20, 6.67s/it][2026-08-18 15:07:03,489] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-320 |
|
63%|██████▎ | 321/512 [35:51<22:59, 7.22s/it]
{'loss': '0.0002382', 'grad_norm': '0.01913', 'learning_rate': '4.032e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '37.95', 'tokens/train_per_sec_per_gpu': '31.93', 'tokens/total': 9712656, 'tokens/trainable': 146781, 'epoch': '1.254'} |
|
63%|██████▎ | 321/512 [35:51<22:59, 7.22s/it]
63%|██████▎ | 322/512 [35:58<22:18, 7.04s/it]
{'loss': '0.0002096', 'grad_norm': '0.01715', 'learning_rate': '4.005e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '34.89', 'tokens/train_per_sec_per_gpu': '35.66', 'tokens/total': 9743088, 'tokens/trainable': 147225, 'epoch': '1.258'} |
|
63%|██████▎ | 322/512 [35:58<22:18, 7.04s/it]
63%|██████▎ | 323/512 [36:04<21:52, 6.95s/it]
{'loss': '0.002077', 'grad_norm': '0.1548', 'learning_rate': '3.978e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.17', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 9773808, 'tokens/trainable': 147673, 'epoch': '1.262'} |
|
63%|██████▎ | 323/512 [36:04<21:52, 6.95s/it]
63%|██████▎ | 324/512 [36:11<21:23, 6.83s/it]
{'loss': '0.0008681', 'grad_norm': '0.07501', 'learning_rate': '3.95e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '33.4', 'tokens/total': 9804016, 'tokens/trainable': 148103, 'epoch': '1.266'} |
|
63%|██████▎ | 324/512 [36:11<21:23, 6.83s/it]
63%|██████▎ | 325/512 [36:17<21:05, 6.77s/it]
{'loss': '0.0009622', 'grad_norm': '0.096', 'learning_rate': '3.923e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '31.9', 'tokens/total': 9834592, 'tokens/trainable': 148535, 'epoch': '1.27'} |
|
63%|██████▎ | 325/512 [36:17<21:05, 6.77s/it]
64%|██████▎ | 326/512 [36:24<20:51, 6.73s/it]
{'loss': '0.002263', 'grad_norm': '0.1281', 'learning_rate': '3.896e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '34.29', 'tokens/total': 9864848, 'tokens/trainable': 148999, 'epoch': '1.273'} |
|
64%|██████▎ | 326/512 [36:24<20:51, 6.73s/it]
64%|██████▍ | 327/512 [36:31<20:39, 6.70s/it]
{'loss': '0.01927', 'grad_norm': '0.6245', 'learning_rate': '3.869e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '31.85', 'tokens/total': 9895168, 'tokens/trainable': 149431, 'epoch': '1.277'} |
|
64%|██████▍ | 327/512 [36:31<20:39, 6.70s/it]
64%|██████▍ | 328/512 [36:37<20:27, 6.67s/it]
{'loss': '0.005126', 'grad_norm': '0.2624', 'learning_rate': '3.842e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '34.34', 'tokens/total': 9925696, 'tokens/trainable': 149886, 'epoch': '1.281'} |
|
64%|██████▍ | 328/512 [36:37<20:27, 6.67s/it]
64%|██████▍ | 329/512 [36:44<20:13, 6.63s/it]
{'loss': '0.003234', 'grad_norm': '0.373', 'learning_rate': '3.815e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '31.24', 'tokens/total': 9955648, 'tokens/trainable': 150288, 'epoch': '1.285'} |
|
64%|██████▍ | 329/512 [36:44<20:13, 6.63s/it]
64%|██████▍ | 330/512 [36:51<20:07, 6.64s/it]
{'loss': '0.0003214', 'grad_norm': '0.02653', 'learning_rate': '3.788e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '34.14', 'tokens/total': 9985856, 'tokens/trainable': 150733, 'epoch': '1.289'} |
|
64%|██████▍ | 330/512 [36:51<20:07, 6.64s/it]
65%|██████▍ | 331/512 [36:57<20:01, 6.64s/it]
{'loss': '0.001038', 'grad_norm': '0.08096', 'learning_rate': '3.761e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '37.02', 'tokens/total': 10016208, 'tokens/trainable': 151207, 'epoch': '1.293'} |
|
65%|██████▍ | 331/512 [36:57<20:01, 6.64s/it]
65%|██████▍ | 332/512 [37:04<19:56, 6.65s/it]
{'loss': '0.001181', 'grad_norm': '0.2568', 'learning_rate': '3.734e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '37.85', 'tokens/total': 10046512, 'tokens/trainable': 151694, 'epoch': '1.297'} |
|
65%|██████▍ | 332/512 [37:04<19:56, 6.65s/it]
65%|██████▌ | 333/512 [37:11<19:53, 6.67s/it]
{'loss': '0.000878', 'grad_norm': '0.04275', 'learning_rate': '3.708e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '38.61', 'tokens/total': 10076944, 'tokens/trainable': 152188, 'epoch': '1.301'} |
|
65%|██████▌ | 333/512 [37:11<19:53, 6.67s/it]
65%|██████▌ | 334/512 [37:17<19:43, 6.65s/it]
{'loss': '4.24e-05', 'grad_norm': '0.02138', 'learning_rate': '3.681e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '30.86', 'tokens/total': 10107296, 'tokens/trainable': 152596, 'epoch': '1.305'} |
|
65%|██████▌ | 334/512 [37:17<19:43, 6.65s/it]
65%|██████▌ | 335/512 [37:24<19:36, 6.65s/it]
{'loss': '0.0002038', 'grad_norm': '0.0136', 'learning_rate': '3.655e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '36.75', 'tokens/total': 10137568, 'tokens/trainable': 153052, 'epoch': '1.309'} |
|
65%|██████▌ | 335/512 [37:24<19:36, 6.65s/it]
66%|██████▌ | 336/512 [37:30<19:31, 6.66s/it]
{'loss': '0.0004199', 'grad_norm': '0.03614', 'learning_rate': '3.628e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '37.17', 'tokens/train_per_sec_per_gpu': '30.42', 'tokens/total': 10167952, 'tokens/trainable': 153491, 'epoch': '1.312'} |
|
66%|██████▌ | 336/512 [37:30<19:31, 6.66s/it]
66%|██████▌ | 337/512 [37:37<19:26, 6.67s/it]
{'loss': '0.00481', 'grad_norm': '0.1898', 'learning_rate': '3.602e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '40.62', 'tokens/total': 10198512, 'tokens/trainable': 153983, 'epoch': '1.316'} |
|
66%|██████▌ | 337/512 [37:37<19:26, 6.67s/it]
66%|██████▌ | 338/512 [37:44<19:20, 6.67s/it]
{'loss': '0.001726', 'grad_norm': '0.1318', 'learning_rate': '3.576e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '36.91', 'tokens/total': 10228752, 'tokens/trainable': 154464, 'epoch': '1.32'} |
|
66%|██████▌ | 338/512 [37:44<19:20, 6.67s/it]
66%|██████▌ | 339/512 [37:50<19:11, 6.66s/it]
{'loss': '7.29e-05', 'grad_norm': '0.007963', 'learning_rate': '3.549e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '36.05', 'tokens/total': 10259104, 'tokens/trainable': 154924, 'epoch': '1.324'} |
|
66%|██████▌ | 339/512 [37:50<19:11, 6.66s/it]
66%|██████▋ | 340/512 [37:57<19:02, 6.64s/it]
{'loss': '0.0002557', 'grad_norm': '0.01312', 'learning_rate': '3.523e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '30.99', 'tokens/total': 10289520, 'tokens/trainable': 155373, 'epoch': '1.328'} |
|
66%|██████▋ | 340/512 [37:57<19:02, 6.64s/it]
67%|██████▋ | 341/512 [38:04<18:58, 6.66s/it]
{'loss': '0.001129', 'grad_norm': '0.05724', 'learning_rate': '3.497e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '36.68', 'tokens/total': 10320016, 'tokens/trainable': 155873, 'epoch': '1.332'} |
|
67%|██████▋ | 341/512 [38:04<18:58, 6.66s/it]
67%|██████▋ | 342/512 [38:10<18:49, 6.64s/it]
{'loss': '0.000231', 'grad_norm': '0.01673', 'learning_rate': '3.471e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '30.57', 'tokens/total': 10350368, 'tokens/trainable': 156278, 'epoch': '1.336'} |
|
67%|██████▋ | 342/512 [38:10<18:49, 6.64s/it]
67%|██████▋ | 343/512 [38:17<18:43, 6.65s/it]
{'loss': '0.001225', 'grad_norm': '0.1632', 'learning_rate': '3.445e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '35.59', 'tokens/total': 10380624, 'tokens/trainable': 156733, 'epoch': '1.34'} |
|
67%|██████▋ | 343/512 [38:17<18:43, 6.65s/it]
67%|██████▋ | 344/512 [38:24<18:38, 6.66s/it]
{'loss': '0.0002068', 'grad_norm': '0.01341', 'learning_rate': '3.42e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '34.09', 'tokens/total': 10411024, 'tokens/trainable': 157203, 'epoch': '1.344'} |
|
67%|██████▋ | 344/512 [38:24<18:38, 6.66s/it]
67%|██████▋ | 345/512 [38:30<18:33, 6.67s/it]
{'loss': '0.001815', 'grad_norm': '0.1137', 'learning_rate': '3.394e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '38.04', 'tokens/total': 10441248, 'tokens/trainable': 157707, 'epoch': '1.348'} |
|
67%|██████▋ | 345/512 [38:30<18:33, 6.67s/it]
68%|██████▊ | 346/512 [38:37<18:30, 6.69s/it]
{'loss': '0.0003793', 'grad_norm': '0.03168', 'learning_rate': '3.368e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '35.18', 'tokens/total': 10471712, 'tokens/trainable': 158180, 'epoch': '1.352'} |
|
68%|██████▊ | 346/512 [38:37<18:30, 6.69s/it]
68%|██████▊ | 347/512 [38:44<18:23, 6.69s/it]
{'loss': '0.001218', 'grad_norm': '0.06486', 'learning_rate': '3.343e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.13', 'tokens/train_per_sec_per_gpu': '34.59', 'tokens/total': 10502288, 'tokens/trainable': 158646, 'epoch': '1.355'} |
|
68%|██████▊ | 347/512 [38:44<18:23, 6.69s/it]
68%|██████▊ | 348/512 [38:50<18:10, 6.65s/it]
{'loss': '0.003916', 'grad_norm': '0.2218', 'learning_rate': '3.317e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '35.84', 'tokens/total': 10532304, 'tokens/trainable': 159115, 'epoch': '1.359'} |
|
68%|██████▊ | 348/512 [38:50<18:10, 6.65s/it]
68%|██████▊ | 349/512 [38:57<18:01, 6.64s/it]
{'loss': '0.00193', 'grad_norm': '0.2674', 'learning_rate': '3.292e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '31.78', 'tokens/total': 10562608, 'tokens/trainable': 159538, 'epoch': '1.363'} |
|
68%|██████▊ | 349/512 [38:57<18:01, 6.64s/it]
68%|██████▊ | 350/512 [39:04<17:55, 6.64s/it]
{'loss': '0.0009783', 'grad_norm': '0.0556', 'learning_rate': '3.267e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '33.4', 'tokens/total': 10592992, 'tokens/trainable': 160016, 'epoch': '1.367'} |
|
68%|██████▊ | 350/512 [39:04<17:55, 6.64s/it]
69%|██████▊ | 351/512 [39:10<17:48, 6.64s/it]
{'loss': '0.0001372', 'grad_norm': '0.01259', 'learning_rate': '3.242e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '34.68', 'tokens/total': 10623328, 'tokens/trainable': 160475, 'epoch': '1.371'} |
|
69%|██████▊ | 351/512 [39:10<17:48, 6.64s/it]
69%|██████▉ | 352/512 [39:17<17:42, 6.64s/it]
{'loss': '0.0008305', 'grad_norm': '0.09844', 'learning_rate': '3.217e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '34.16', 'tokens/total': 10653632, 'tokens/trainable': 160952, 'epoch': '1.375'} |
|
69%|██████▉ | 352/512 [39:17<17:42, 6.64s/it][2026-08-18 15:10:37,987] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-352 |
|
69%|██████▉ | 353/512 [39:25<18:59, 7.17s/it]
{'loss': '0.000948', 'grad_norm': '0.1186', 'learning_rate': '3.192e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '38.4', 'tokens/train_per_sec_per_gpu': '33.06', 'tokens/total': 10684176, 'tokens/trainable': 161420, 'epoch': '1.379'} |
|
69%|██████▉ | 353/512 [39:25<18:59, 7.17s/it]
69%|██████▉ | 354/512 [39:32<18:24, 6.99s/it]
{'loss': '0.0005211', 'grad_norm': '0.05824', 'learning_rate': '3.167e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '34.31', 'tokens/total': 10714352, 'tokens/trainable': 161868, 'epoch': '1.383'} |
|
69%|██████▉ | 354/512 [39:32<18:24, 6.99s/it]
69%|██████▉ | 355/512 [39:39<18:00, 6.88s/it]
{'loss': '3.899e-05', 'grad_norm': '0.002648', 'learning_rate': '3.142e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '35.64', 'tokens/total': 10744464, 'tokens/trainable': 162318, 'epoch': '1.387'} |
|
69%|██████▉ | 355/512 [39:39<18:00, 6.88s/it]
70%|██████▉ | 356/512 [39:45<17:44, 6.82s/it]
{'loss': '2.845e-05', 'grad_norm': '0.001497', 'learning_rate': '3.117e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '31.69', 'tokens/total': 10774976, 'tokens/trainable': 162789, 'epoch': '1.391'} |
|
70%|██████▉ | 356/512 [39:45<17:44, 6.82s/it]
70%|██████▉ | 357/512 [39:52<17:30, 6.78s/it]
{'loss': '0.004984', 'grad_norm': '0.2008', 'learning_rate': '3.093e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '36.58', 'tokens/total': 10805600, 'tokens/trainable': 163254, 'epoch': '1.395'} |
|
70%|██████▉ | 357/512 [39:52<17:30, 6.78s/it]
70%|██████▉ | 358/512 [39:59<17:17, 6.74s/it]
{'loss': '1.922e-05', 'grad_norm': '0.001224', 'learning_rate': '3.068e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '34.68', 'tokens/total': 10835920, 'tokens/trainable': 163715, 'epoch': '1.398'} |
|
70%|██████▉ | 358/512 [39:59<17:17, 6.74s/it]
70%|███████ | 359/512 [40:05<17:07, 6.71s/it]
{'loss': '0.0005038', 'grad_norm': '0.04399', 'learning_rate': '3.044e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '34.67', 'tokens/total': 10866048, 'tokens/trainable': 164197, 'epoch': '1.402'} |
|
70%|███████ | 359/512 [40:05<17:07, 6.71s/it]
70%|███████ | 360/512 [40:12<16:57, 6.70s/it]
{'loss': '6.03e-05', 'grad_norm': '0.005501', 'learning_rate': '3.02e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '31.85', 'tokens/total': 10896288, 'tokens/trainable': 164661, 'epoch': '1.406'} |
|
70%|███████ | 360/512 [40:12<16:57, 6.70s/it]
71%|███████ | 361/512 [40:18<16:47, 6.67s/it]
{'loss': '0.0001087', 'grad_norm': '0.008387', 'learning_rate': '2.995e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 10926816, 'tokens/trainable': 165111, 'epoch': '1.41'} |
|
71%|███████ | 361/512 [40:18<16:47, 6.67s/it]
71%|███████ | 362/512 [40:25<16:15, 6.50s/it]
{'loss': '0.001125', 'grad_norm': '0.1003', 'learning_rate': '2.971e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.31', 'memory/max_allocated (GiB)': '33.31', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '30.65', 'tokens/total': 10954928, 'tokens/trainable': 165552, 'epoch': '1.414'} |
|
71%|███████ | 362/512 [40:25<16:15, 6.50s/it]
71%|███████ | 363/512 [40:31<16:15, 6.55s/it]
{'loss': '0.0001569', 'grad_norm': '0.01849', 'learning_rate': '2.947e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '35.49', 'tokens/total': 10985328, 'tokens/trainable': 166026, 'epoch': '1.418'} |
|
71%|███████ | 363/512 [40:31<16:15, 6.55s/it]
71%|███████ | 364/512 [40:38<16:13, 6.58s/it]
{'loss': '4.314e-05', 'grad_norm': '0.003734', 'learning_rate': '2.924e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '38.17', 'tokens/total': 11015664, 'tokens/trainable': 166486, 'epoch': '1.422'} |
|
71%|███████ | 364/512 [40:38<16:13, 6.58s/it]
71%|███████▏ | 365/512 [40:45<16:09, 6.60s/it]
{'loss': '7.389e-05', 'grad_norm': '0.007158', 'learning_rate': '2.9e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '36.65', 'tokens/total': 11046256, 'tokens/trainable': 166959, 'epoch': '1.426'} |
|
71%|███████▏ | 365/512 [40:45<16:09, 6.60s/it]
71%|███████▏ | 366/512 [40:51<16:01, 6.59s/it]
{'loss': '9.061e-05', 'grad_norm': '0.008675', 'learning_rate': '2.876e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.68', 'memory/max_allocated (GiB)': '33.68', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '33.94', 'tokens/total': 11076288, 'tokens/trainable': 167394, 'epoch': '1.43'} |
|
71%|███████▏ | 366/512 [40:51<16:01, 6.59s/it]
72%|███████▏ | 367/512 [40:58<15:55, 6.59s/it]
{'loss': '0.001297', 'grad_norm': '0.2448', 'learning_rate': '2.853e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '28.7', 'tokens/total': 11106416, 'tokens/trainable': 167832, 'epoch': '1.434'} |
|
72%|███████▏ | 367/512 [40:58<15:55, 6.59s/it]
72%|███████▏ | 368/512 [41:04<15:50, 6.60s/it]
{'loss': '4.491e-05', 'grad_norm': '0.005687', 'learning_rate': '2.829e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '31.11', 'tokens/total': 11136848, 'tokens/trainable': 168242, 'epoch': '1.438'} |
|
72%|███████▏ | 368/512 [41:04<15:50, 6.60s/it]
72%|███████▏ | 369/512 [41:11<15:44, 6.61s/it]
{'loss': '3.635e-05', 'grad_norm': '0.002911', 'learning_rate': '2.806e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.91', 'tokens/total': 11167376, 'tokens/trainable': 168715, 'epoch': '1.441'} |
|
72%|███████▏ | 369/512 [41:11<15:44, 6.61s/it]
72%|███████▏ | 370/512 [41:18<15:40, 6.62s/it]
{'loss': '0.001463', 'grad_norm': '0.1325', 'learning_rate': '2.783e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '31.96', 'tokens/total': 11197776, 'tokens/trainable': 169147, 'epoch': '1.445'} |
|
72%|███████▏ | 370/512 [41:18<15:40, 6.62s/it]
72%|███████▏ | 371/512 [41:24<15:35, 6.63s/it]
{'loss': '0.0003711', 'grad_norm': '0.06608', 'learning_rate': '2.76e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.31', 'tokens/total': 11227840, 'tokens/trainable': 169628, 'epoch': '1.449'} |
|
72%|███████▏ | 371/512 [41:24<15:35, 6.63s/it]
73%|███████▎ | 372/512 [41:31<15:30, 6.65s/it]
{'loss': '3.963e-05', 'grad_norm': '0.00427', 'learning_rate': '2.737e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.33', 'tokens/total': 11258176, 'tokens/trainable': 170085, 'epoch': '1.453'} |
|
73%|███████▎ | 372/512 [41:31<15:30, 6.65s/it]
73%|███████▎ | 373/512 [41:38<15:21, 6.63s/it]
{'loss': '8.674e-05', 'grad_norm': '0.009922', 'learning_rate': '2.714e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.77', 'tokens/total': 11288336, 'tokens/trainable': 170584, 'epoch': '1.457'} |
|
73%|███████▎ | 373/512 [41:38<15:21, 6.63s/it]
73%|███████▎ | 374/512 [41:44<15:15, 6.63s/it]
{'loss': '0.001597', 'grad_norm': '0.2059', 'learning_rate': '2.691e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.82', 'tokens/total': 11318848, 'tokens/trainable': 171058, 'epoch': '1.461'} |
|
73%|███████▎ | 374/512 [41:44<15:15, 6.63s/it]
73%|███████▎ | 375/512 [41:51<15:09, 6.64s/it]
{'loss': '3.412e-05', 'grad_norm': '0.002577', 'learning_rate': '2.668e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.08', 'tokens/total': 11349168, 'tokens/trainable': 171518, 'epoch': '1.465'} |
|
73%|███████▎ | 375/512 [41:51<15:09, 6.64s/it]
73%|███████▎ | 376/512 [41:57<15:01, 6.63s/it]
{'loss': '0.001489', 'grad_norm': '0.1485', 'learning_rate': '2.646e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '40.75', 'tokens/total': 11379328, 'tokens/trainable': 172007, 'epoch': '1.469'} |
|
73%|███████▎ | 376/512 [41:57<15:01, 6.63s/it]
74%|███████▎ | 377/512 [42:04<14:54, 6.62s/it]
{'loss': '0.0008696', 'grad_norm': '0.1735', 'learning_rate': '2.624e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 11409568, 'tokens/trainable': 172468, 'epoch': '1.473'} |
|
74%|███████▎ | 377/512 [42:04<14:54, 6.62s/it]
74%|███████▍ | 378/512 [42:11<14:48, 6.63s/it]
{'loss': '0.002175', 'grad_norm': '0.1359', 'learning_rate': '2.601e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.89', 'tokens/total': 11439968, 'tokens/trainable': 172923, 'epoch': '1.477'} |
|
74%|███████▍ | 378/512 [42:11<14:48, 6.63s/it]
74%|███████▍ | 379/512 [42:17<14:41, 6.63s/it]
{'loss': '0.01854', 'grad_norm': '0.9078', 'learning_rate': '2.579e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 11470496, 'tokens/trainable': 173370, 'epoch': '1.48'} |
|
74%|███████▍ | 379/512 [42:17<14:41, 6.63s/it]
74%|███████▍ | 380/512 [42:24<14:36, 6.64s/it]
{'loss': '5.761e-05', 'grad_norm': '0.01038', 'learning_rate': '2.557e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.34', 'tokens/total': 11500720, 'tokens/trainable': 173837, 'epoch': '1.484'} |
|
74%|███████▍ | 380/512 [42:24<14:36, 6.64s/it]
74%|███████▍ | 381/512 [42:31<14:29, 6.64s/it]
{'loss': '1.338e-05', 'grad_norm': '0.0007154', 'learning_rate': '2.535e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.75', 'tokens/total': 11531280, 'tokens/trainable': 174294, 'epoch': '1.488'} |
|
74%|███████▍ | 381/512 [42:31<14:29, 6.64s/it]
75%|███████▍ | 382/512 [42:37<14:21, 6.62s/it]
{'loss': '2.482e-05', 'grad_norm': '0.001728', 'learning_rate': '2.513e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '30.98', 'tokens/total': 11561264, 'tokens/trainable': 174701, 'epoch': '1.492'} |
|
75%|███████▍ | 382/512 [42:37<14:21, 6.62s/it]
75%|███████▍ | 383/512 [42:44<14:14, 6.62s/it]
{'loss': '0.0002584', 'grad_norm': '0.08112', 'learning_rate': '2.492e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.3', 'tokens/total': 11591616, 'tokens/trainable': 175131, 'epoch': '1.496'} |
|
75%|███████▍ | 383/512 [42:44<14:14, 6.62s/it]
75%|███████▌ | 384/512 [42:50<14:08, 6.63s/it]
{'loss': '0.0002893', 'grad_norm': '0.0206', 'learning_rate': '2.47e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.19', 'tokens/total': 11622000, 'tokens/trainable': 175586, 'epoch': '1.5'} |
|
75%|███████▌ | 384/512 [42:50<14:08, 6.63s/it][2026-08-18 15:14:11,511] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-384 |
|
75%|███████▌ | 385/512 [42:59<15:11, 7.18s/it]
{'loss': '0.002987', 'grad_norm': '0.3424', 'learning_rate': '2.449e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.3', 'tokens/total': 11652352, 'tokens/trainable': 176026, 'epoch': '1.504'} |
|
75%|███████▌ | 385/512 [42:59<15:11, 7.18s/it]
75%|███████▌ | 386/512 [43:06<14:45, 7.03s/it]
{'loss': '5.986e-05', 'grad_norm': '0.005428', 'learning_rate': '2.428e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '34.98', 'tokens/train_per_sec_per_gpu': '34.05', 'tokens/total': 11682736, 'tokens/trainable': 176512, 'epoch': '1.508'} |
|
75%|███████▌ | 386/512 [43:06<14:45, 7.03s/it]
76%|███████▌ | 387/512 [43:12<14:24, 6.92s/it]
{'loss': '0.001088', 'grad_norm': '0.1145', 'learning_rate': '2.406e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '34.98', 'tokens/train_per_sec_per_gpu': '30.63', 'tokens/total': 11713136, 'tokens/trainable': 176943, 'epoch': '1.512'} |
|
76%|███████▌ | 387/512 [43:12<14:24, 6.92s/it]
76%|███████▌ | 388/512 [43:19<14:06, 6.82s/it]
{'loss': '0.0001353', 'grad_norm': '0.01624', 'learning_rate': '2.385e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.23', 'tokens/train_per_sec_per_gpu': '32.24', 'tokens/total': 11743808, 'tokens/trainable': 177370, 'epoch': '1.516'} |
|
76%|███████▌ | 388/512 [43:19<14:06, 6.82s/it]
76%|███████▌ | 389/512 [43:26<13:53, 6.77s/it]
{'loss': '7.231e-05', 'grad_norm': '0.03344', 'learning_rate': '2.365e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.58', 'tokens/train_per_sec_per_gpu': '35.32', 'tokens/total': 11774144, 'tokens/trainable': 177851, 'epoch': '1.52'} |
|
76%|███████▌ | 389/512 [43:26<13:53, 6.77s/it]
76%|███████▌ | 390/512 [43:32<13:41, 6.73s/it]
{'loss': '5.419e-05', 'grad_norm': '0.004582', 'learning_rate': '2.344e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 11804272, 'tokens/trainable': 178310, 'epoch': '1.523'} |
|
76%|███████▌ | 390/512 [43:32<13:41, 6.73s/it]
76%|███████▋ | 391/512 [43:38<13:13, 6.56s/it]
{'loss': '3.318e-05', 'grad_norm': '0.003119', 'learning_rate': '2.323e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '37.07', 'tokens/total': 11832432, 'tokens/trainable': 178736, 'epoch': '1.527'} |
|
76%|███████▋ | 391/512 [43:38<13:13, 6.56s/it]
77%|███████▋ | 392/512 [43:45<13:13, 6.61s/it]
{'loss': '0.002361', 'grad_norm': '0.1388', 'learning_rate': '2.303e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '36.41', 'tokens/total': 11862960, 'tokens/trainable': 179233, 'epoch': '1.531'} |
|
77%|███████▋ | 392/512 [43:45<13:13, 6.61s/it]
77%|███████▋ | 393/512 [43:52<13:09, 6.63s/it]
{'loss': '0.00362', 'grad_norm': '0.3595', 'learning_rate': '2.282e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '38.74', 'tokens/total': 11893232, 'tokens/trainable': 179730, 'epoch': '1.535'} |
|
77%|███████▋ | 393/512 [43:52<13:09, 6.63s/it]
77%|███████▋ | 394/512 [43:58<13:04, 6.65s/it]
{'loss': '0.0002577', 'grad_norm': '0.04762', 'learning_rate': '2.262e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.05', 'tokens/total': 11923664, 'tokens/trainable': 180227, 'epoch': '1.539'} |
|
77%|███████▋ | 394/512 [43:58<13:04, 6.65s/it]
77%|███████▋ | 395/512 [44:05<12:58, 6.66s/it]
{'loss': '0.0004314', 'grad_norm': '0.05118', 'learning_rate': '2.242e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.1', 'tokens/total': 11954032, 'tokens/trainable': 180687, 'epoch': '1.543'} |
|
77%|███████▋ | 395/512 [44:05<12:58, 6.66s/it]
77%|███████▋ | 396/512 [44:12<12:51, 6.65s/it]
{'loss': '0.0005177', 'grad_norm': '0.06999', 'learning_rate': '2.222e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.12', 'tokens/total': 11984512, 'tokens/trainable': 181141, 'epoch': '1.547'} |
|
77%|███████▋ | 396/512 [44:12<12:51, 6.65s/it]
78%|███████▊ | 397/512 [44:18<12:47, 6.67s/it]
{'loss': '0.0007535', 'grad_norm': '0.2004', 'learning_rate': '2.202e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.2', 'tokens/total': 12014880, 'tokens/trainable': 181648, 'epoch': '1.551'} |
|
78%|███████▊ | 397/512 [44:18<12:47, 6.67s/it]
78%|███████▊ | 398/512 [44:25<12:37, 6.65s/it]
{'loss': '0.001293', 'grad_norm': '0.1333', 'learning_rate': '2.183e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.13', 'tokens/total': 12044928, 'tokens/trainable': 182109, 'epoch': '1.555'} |
|
78%|███████▊ | 398/512 [44:25<12:37, 6.65s/it]
78%|███████▊ | 399/512 [44:32<12:31, 6.65s/it]
{'loss': '0.01236', 'grad_norm': '0.7181', 'learning_rate': '2.163e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.78', 'tokens/total': 12075344, 'tokens/trainable': 182559, 'epoch': '1.559'} |
|
78%|███████▊ | 399/512 [44:32<12:31, 6.65s/it]
78%|███████▊ | 400/512 [44:38<12:22, 6.63s/it]
{'loss': '2.153e-05', 'grad_norm': '0.001306', 'learning_rate': '2.144e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.46', 'tokens/total': 12105616, 'tokens/trainable': 182985, 'epoch': '1.562'} |
|
78%|███████▊ | 400/512 [44:38<12:22, 6.63s/it]
78%|███████▊ | 401/512 [44:45<12:15, 6.63s/it]
{'loss': '8.926e-05', 'grad_norm': '0.01069', 'learning_rate': '2.124e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.9', 'tokens/total': 12136080, 'tokens/trainable': 183440, 'epoch': '1.566'} |
|
78%|███████▊ | 401/512 [44:45<12:15, 6.63s/it]
79%|███████▊ | 402/512 [44:52<12:09, 6.63s/it]
{'loss': '0.0002033', 'grad_norm': '0.01728', 'learning_rate': '2.105e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.74', 'tokens/total': 12166240, 'tokens/trainable': 183917, 'epoch': '1.57'} |
|
79%|███████▊ | 402/512 [44:52<12:09, 6.63s/it]
79%|███████▊ | 403/512 [44:58<12:02, 6.63s/it]
{'loss': '0.0005283', 'grad_norm': '0.08476', 'learning_rate': '2.086e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '30.56', 'tokens/total': 12196544, 'tokens/trainable': 184348, 'epoch': '1.574'} |
|
79%|███████▊ | 403/512 [44:58<12:02, 6.63s/it]
79%|███████▉ | 404/512 [45:05<11:55, 6.63s/it]
{'loss': '5.411e-05', 'grad_norm': '0.002903', 'learning_rate': '2.067e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.72', 'tokens/total': 12226784, 'tokens/trainable': 184786, 'epoch': '1.578'} |
|
79%|███████▉ | 404/512 [45:05<11:55, 6.63s/it]
79%|███████▉ | 405/512 [45:11<11:50, 6.64s/it]
{'loss': '0.0002449', 'grad_norm': '0.04438', 'learning_rate': '2.049e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.09', 'tokens/total': 12257264, 'tokens/trainable': 185249, 'epoch': '1.582'} |
|
79%|███████▉ | 405/512 [45:11<11:50, 6.64s/it]
79%|███████▉ | 406/512 [45:18<11:44, 6.64s/it]
{'loss': '0.0004027', 'grad_norm': '0.03067', 'learning_rate': '2.03e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.71', 'tokens/total': 12287584, 'tokens/trainable': 185680, 'epoch': '1.586'} |
|
79%|███████▉ | 406/512 [45:18<11:44, 6.64s/it]
79%|███████▉ | 407/512 [45:25<11:36, 6.64s/it]
{'loss': '7.472e-05', 'grad_norm': '0.009147', 'learning_rate': '2.012e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.87', 'tokens/total': 12317984, 'tokens/trainable': 186100, 'epoch': '1.59'} |
|
79%|███████▉ | 407/512 [45:25<11:36, 6.64s/it]
80%|███████▉ | 408/512 [45:31<11:29, 6.63s/it]
{'loss': '0.001381', 'grad_norm': '0.1184', 'learning_rate': '1.993e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '40.07', 'tokens/total': 12348144, 'tokens/trainable': 186565, 'epoch': '1.594'} |
|
80%|███████▉ | 408/512 [45:31<11:29, 6.63s/it]
80%|███████▉ | 409/512 [45:38<11:22, 6.63s/it]
{'loss': '0.00254', 'grad_norm': '0.2681', 'learning_rate': '1.975e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.07', 'tokens/total': 12378544, 'tokens/trainable': 187017, 'epoch': '1.598'} |
|
80%|███████▉ | 409/512 [45:38<11:22, 6.63s/it]
80%|████████ | 410/512 [45:45<11:17, 6.64s/it]
{'loss': '6.725e-05', 'grad_norm': '0.004361', 'learning_rate': '1.957e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.19', 'tokens/total': 12408784, 'tokens/trainable': 187468, 'epoch': '1.602'} |
|
80%|████████ | 410/512 [45:45<11:17, 6.64s/it]
80%|████████ | 411/512 [45:51<11:09, 6.63s/it]
{'loss': '1.57e-05', 'grad_norm': '0.0006756', 'learning_rate': '1.94e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37.85', 'tokens/total': 12438832, 'tokens/trainable': 187961, 'epoch': '1.605'} |
|
80%|████████ | 411/512 [45:51<11:09, 6.63s/it]
80%|████████ | 412/512 [45:58<11:03, 6.63s/it]
{'loss': '1.328e-05', 'grad_norm': '0.0005687', 'learning_rate': '1.922e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.18', 'tokens/total': 12469296, 'tokens/trainable': 188447, 'epoch': '1.609'} |
|
80%|████████ | 412/512 [45:58<11:03, 6.63s/it]
81%|████████ | 413/512 [46:04<10:55, 6.62s/it]
{'loss': '0.0001658', 'grad_norm': '0.01321', 'learning_rate': '1.904e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.05', 'tokens/train_per_sec_per_gpu': '36.21', 'tokens/total': 12499392, 'tokens/trainable': 188903, 'epoch': '1.613'} |
|
81%|████████ | 413/512 [46:04<10:55, 6.62s/it]
81%|████████ | 414/512 [46:11<10:52, 6.66s/it]
{'loss': '0.0004871', 'grad_norm': '0.04096', 'learning_rate': '1.887e-05', 'ppl': '1', 'memory/max_active (GiB)': '34', 'memory/max_allocated (GiB)': '34', 'memory/device_reserved (GiB)': '38.05', 'tokens/train_per_sec_per_gpu': '36.92', 'tokens/total': 12529952, 'tokens/trainable': 189398, 'epoch': '1.617'} |
|
81%|████████ | 414/512 [46:11<10:52, 6.66s/it]
81%|████████ | 415/512 [46:18<10:45, 6.66s/it]
{'loss': '5.711e-05', 'grad_norm': '0.005268', 'learning_rate': '1.87e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '38.05', 'tokens/train_per_sec_per_gpu': '34.43', 'tokens/total': 12560240, 'tokens/trainable': 189850, 'epoch': '1.621'} |
|
81%|████████ | 415/512 [46:18<10:45, 6.66s/it]
81%|████████▏ | 416/512 [46:24<10:38, 6.66s/it]
{'loss': '6.629e-05', 'grad_norm': '0.006016', 'learning_rate': '1.853e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.05', 'tokens/train_per_sec_per_gpu': '32.92', 'tokens/total': 12590736, 'tokens/trainable': 190306, 'epoch': '1.625'} |
|
81%|████████▏ | 416/512 [46:24<10:38, 6.66s/it][2026-08-18 15:17:45,547] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-416 |
|
81%|████████▏ | 417/512 [46:33<11:25, 7.21s/it]
{'loss': '0.0005899', 'grad_norm': '0.06729', 'learning_rate': '1.836e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '38.05', 'tokens/train_per_sec_per_gpu': '34.36', 'tokens/total': 12621168, 'tokens/trainable': 190746, 'epoch': '1.629'} |
|
81%|████████▏ | 417/512 [46:33<11:25, 7.21s/it]
82%|████████▏ | 418/512 [46:40<11:00, 7.03s/it]
{'loss': '5.955e-05', 'grad_norm': '0.003185', 'learning_rate': '1.819e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '31.39', 'tokens/total': 12651456, 'tokens/trainable': 191156, 'epoch': '1.633'} |
|
82%|████████▏ | 418/512 [46:40<11:00, 7.03s/it]
82%|████████▏ | 419/512 [46:46<10:41, 6.90s/it]
{'loss': '7.596e-05', 'grad_norm': '0.005559', 'learning_rate': '1.802e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '29.67', 'tokens/total': 12681760, 'tokens/trainable': 191573, 'epoch': '1.637'} |
|
82%|████████▏ | 419/512 [46:46<10:41, 6.90s/it]
82%|████████▏ | 420/512 [46:53<10:27, 6.82s/it]
{'loss': '0.0001359', 'grad_norm': '0.01467', 'learning_rate': '1.786e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '33.01', 'tokens/total': 12712144, 'tokens/trainable': 192014, 'epoch': '1.641'} |
|
82%|████████▏ | 420/512 [46:53<10:27, 6.82s/it]
82%|████████▏ | 421/512 [46:59<10:15, 6.76s/it]
{'loss': '0.001797', 'grad_norm': '0.392', 'learning_rate': '1.77e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '33.08', 'tokens/total': 12742400, 'tokens/trainable': 192453, 'epoch': '1.645'} |
|
82%|████████▏ | 421/512 [46:59<10:15, 6.76s/it]
82%|████████▏ | 422/512 [47:06<10:03, 6.71s/it]
{'loss': '8.881e-05', 'grad_norm': '0.01342', 'learning_rate': '1.753e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '33.45', 'tokens/total': 12772288, 'tokens/trainable': 192886, 'epoch': '1.648'} |
|
82%|████████▏ | 422/512 [47:06<10:03, 6.71s/it]
83%|████████▎ | 423/512 [47:13<09:57, 6.71s/it]
{'loss': '0.000102', 'grad_norm': '0.009214', 'learning_rate': '1.737e-05', 'ppl': '1', 'memory/max_active (GiB)': '34.03', 'memory/max_allocated (GiB)': '34.03', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '30.11', 'tokens/total': 12802880, 'tokens/trainable': 193323, 'epoch': '1.652'} |
|
83%|████████▎ | 423/512 [47:13<09:57, 6.71s/it]
83%|████████▎ | 424/512 [47:20<09:51, 6.72s/it]
{'loss': '9.561e-05', 'grad_norm': '0.008044', 'learning_rate': '1.722e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '32.88', 'tokens/total': 12833456, 'tokens/trainable': 193753, 'epoch': '1.656'} |
|
83%|████████▎ | 424/512 [47:20<09:51, 6.72s/it]
83%|████████▎ | 425/512 [47:26<09:42, 6.70s/it]
{'loss': '4.567e-05', 'grad_norm': '0.002295', 'learning_rate': '1.706e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '36.61', 'tokens/total': 12863952, 'tokens/trainable': 194240, 'epoch': '1.66'} |
|
83%|████████▎ | 425/512 [47:26<09:42, 6.70s/it]
83%|████████▎ | 426/512 [47:33<09:34, 6.68s/it]
{'loss': '0.003879', 'grad_norm': '0.2227', 'learning_rate': '1.69e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '35.77', 'tokens/total': 12894352, 'tokens/trainable': 194680, 'epoch': '1.664'} |
|
83%|████████▎ | 426/512 [47:33<09:34, 6.68s/it]
83%|████████▎ | 427/512 [47:39<09:27, 6.67s/it]
{'loss': '4.946e-05', 'grad_norm': '0.003741', 'learning_rate': '1.675e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '39.06', 'tokens/total': 12924688, 'tokens/trainable': 195186, 'epoch': '1.668'} |
|
83%|████████▎ | 427/512 [47:39<09:27, 6.67s/it]
84%|████████▎ | 428/512 [47:46<09:21, 6.68s/it]
{'loss': '0.0001484', 'grad_norm': '0.01903', 'learning_rate': '1.66e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '36.87', 'tokens/total': 12955072, 'tokens/trainable': 195693, 'epoch': '1.672'} |
|
84%|████████▎ | 428/512 [47:46<09:21, 6.68s/it]
84%|████████▍ | 429/512 [47:53<09:12, 6.66s/it]
{'loss': '8.986e-05', 'grad_norm': '0.009708', 'learning_rate': '1.645e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.05', 'tokens/train_per_sec_per_gpu': '37.21', 'tokens/total': 12985328, 'tokens/trainable': 196130, 'epoch': '1.676'} |
|
84%|████████▍ | 429/512 [47:53<09:12, 6.66s/it]
84%|████████▍ | 430/512 [47:59<09:04, 6.64s/it]
{'loss': '0.0001104', 'grad_norm': '0.01202', 'learning_rate': '1.63e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '29.77', 'tokens/total': 13015600, 'tokens/trainable': 196530, 'epoch': '1.68'} |
|
84%|████████▍ | 430/512 [47:59<09:04, 6.64s/it]
84%|████████▍ | 431/512 [48:06<08:56, 6.63s/it]
{'loss': '0.000595', 'grad_norm': '0.0743', 'learning_rate': '1.615e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '30.15', 'tokens/total': 13045728, 'tokens/trainable': 196974, 'epoch': '1.684'} |
|
84%|████████▍ | 431/512 [48:06<08:56, 6.63s/it]
84%|████████▍ | 432/512 [48:13<08:50, 6.63s/it]
{'loss': '0.001181', 'grad_norm': '0.1797', 'learning_rate': '1.6e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '35.48', 'tokens/total': 13075920, 'tokens/trainable': 197454, 'epoch': '1.688'} |
|
84%|████████▍ | 432/512 [48:13<08:50, 6.63s/it]
85%|████████▍ | 433/512 [48:19<08:44, 6.64s/it]
{'loss': '1.079e-05', 'grad_norm': '0.0005297', 'learning_rate': '1.586e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '31.63', 'tokens/total': 13106128, 'tokens/trainable': 197902, 'epoch': '1.691'} |
|
85%|████████▍ | 433/512 [48:19<08:44, 6.64s/it]
85%|████████▍ | 434/512 [48:26<08:36, 6.62s/it]
{'loss': '5.284e-05', 'grad_norm': '0.004878', 'learning_rate': '1.572e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '36.24', 'tokens/total': 13136368, 'tokens/trainable': 198374, 'epoch': '1.695'} |
|
85%|████████▍ | 434/512 [48:26<08:36, 6.62s/it]
85%|████████▍ | 435/512 [48:32<08:30, 6.63s/it]
{'loss': '7.287e-05', 'grad_norm': '0.01216', 'learning_rate': '1.558e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '37.86', 'tokens/total': 13166704, 'tokens/trainable': 198846, 'epoch': '1.699'} |
|
85%|████████▍ | 435/512 [48:32<08:30, 6.63s/it]
85%|████████▌ | 436/512 [48:39<08:24, 6.64s/it]
{'loss': '0.001207', 'grad_norm': '0.1016', 'learning_rate': '1.544e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '38.04', 'tokens/total': 13197344, 'tokens/trainable': 199320, 'epoch': '1.703'} |
|
85%|████████▌ | 436/512 [48:39<08:24, 6.64s/it]
85%|████████▌ | 437/512 [48:46<08:17, 6.64s/it]
{'loss': '2.548e-05', 'grad_norm': '0.002813', 'learning_rate': '1.53e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.06', 'tokens/train_per_sec_per_gpu': '37.71', 'tokens/total': 13227664, 'tokens/trainable': 199777, 'epoch': '1.707'} |
|
85%|████████▌ | 437/512 [48:46<08:17, 6.64s/it]
86%|████████▌ | 438/512 [48:52<08:09, 6.61s/it]
{'loss': '5.002e-05', 'grad_norm': '0.00345', 'learning_rate': '1.516e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '31.13', 'tokens/total': 13257632, 'tokens/trainable': 200194, 'epoch': '1.711'} |
|
86%|████████▌ | 438/512 [48:52<08:09, 6.61s/it]
86%|████████▌ | 439/512 [48:59<08:05, 6.65s/it]
{'loss': '7.197e-06', 'grad_norm': '0.0002853', 'learning_rate': '1.503e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '34.1', 'tokens/total': 13288240, 'tokens/trainable': 200663, 'epoch': '1.715'} |
|
86%|████████▌ | 439/512 [48:59<08:05, 6.65s/it]
86%|████████▌ | 440/512 [49:06<08:01, 6.68s/it]
{'loss': '4.477e-05', 'grad_norm': '0.002178', 'learning_rate': '1.49e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '34.06', 'tokens/total': 13318992, 'tokens/trainable': 201121, 'epoch': '1.719'} |
|
86%|████████▌ | 440/512 [49:06<08:01, 6.68s/it]
86%|████████▌ | 441/512 [49:12<07:52, 6.66s/it]
{'loss': '5.032e-05', 'grad_norm': '0.003926', 'learning_rate': '1.477e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '40.1', 'tokens/total': 13349392, 'tokens/trainable': 201598, 'epoch': '1.723'} |
|
86%|████████▌ | 441/512 [49:12<07:52, 6.66s/it]
86%|████████▋ | 442/512 [49:19<07:46, 6.67s/it]
{'loss': '9.367e-05', 'grad_norm': '0.007513', 'learning_rate': '1.464e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '35.22', 'tokens/total': 13379648, 'tokens/trainable': 202068, 'epoch': '1.727'} |
|
86%|████████▋ | 442/512 [49:19<07:46, 6.67s/it]
87%|████████▋ | 443/512 [49:26<07:38, 6.64s/it]
{'loss': '6.84e-05', 'grad_norm': '0.006775', 'learning_rate': '1.451e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '35.79', 'tokens/total': 13409936, 'tokens/trainable': 202524, 'epoch': '1.73'} |
|
87%|████████▋ | 443/512 [49:26<07:38, 6.64s/it]
87%|████████▋ | 444/512 [49:32<07:31, 6.64s/it]
{'loss': '0.0004944', 'grad_norm': '0.1046', 'learning_rate': '1.438e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '36.62', 'tokens/total': 13440288, 'tokens/trainable': 202996, 'epoch': '1.734'} |
|
87%|████████▋ | 444/512 [49:32<07:31, 6.64s/it]
87%|████████▋ | 445/512 [49:39<07:16, 6.51s/it]
{'loss': '0.0006291', 'grad_norm': '0.06457', 'learning_rate': '1.426e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '39.56', 'tokens/total': 13468880, 'tokens/trainable': 203470, 'epoch': '1.738'} |
|
87%|████████▋ | 445/512 [49:39<07:16, 6.51s/it]
87%|████████▋ | 446/512 [49:45<07:12, 6.55s/it]
{'loss': '0.0002773', 'grad_norm': '0.02827', 'learning_rate': '1.414e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '35.54', 'tokens/total': 13499184, 'tokens/trainable': 203919, 'epoch': '1.742'} |
|
87%|████████▋ | 446/512 [49:45<07:12, 6.55s/it]
87%|████████▋ | 447/512 [49:52<07:07, 6.58s/it]
{'loss': '6.543e-05', 'grad_norm': '0.006509', 'learning_rate': '1.402e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '33.36', 'tokens/total': 13529632, 'tokens/trainable': 204376, 'epoch': '1.746'} |
|
87%|████████▋ | 447/512 [49:52<07:07, 6.58s/it]
88%|████████▊ | 448/512 [49:59<07:03, 6.61s/it]
{'loss': '0.0001307', 'grad_norm': '0.01417', 'learning_rate': '1.39e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '32.03', 'tokens/total': 13560080, 'tokens/trainable': 204821, 'epoch': '1.75'} |
|
88%|████████▊ | 448/512 [49:59<07:03, 6.61s/it][2026-08-18 15:21:19,574] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-448 |
|
88%|████████▊ | 449/512 [50:07<07:32, 7.19s/it]
{'loss': '6.887e-06', 'grad_norm': '0.0002402', 'learning_rate': '1.378e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '33.78', 'tokens/total': 13590496, 'tokens/trainable': 205271, 'epoch': '1.754'} |
|
88%|████████▊ | 449/512 [50:07<07:32, 7.19s/it]
88%|████████▊ | 450/512 [50:14<07:15, 7.02s/it]
{'loss': '1.354e-05', 'grad_norm': '0.0006253', 'learning_rate': '1.367e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '34.92', 'tokens/train_per_sec_per_gpu': '31.12', 'tokens/total': 13620960, 'tokens/trainable': 205723, 'epoch': '1.758'} |
|
88%|████████▊ | 450/512 [50:14<07:15, 7.02s/it]
88%|████████▊ | 451/512 [50:20<07:01, 6.91s/it]
{'loss': '0.0001233', 'grad_norm': '0.01594', 'learning_rate': '1.355e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.04', 'tokens/train_per_sec_per_gpu': '32.7', 'tokens/total': 13651344, 'tokens/trainable': 206169, 'epoch': '1.762'} |
|
88%|████████▊ | 451/512 [50:20<07:01, 6.91s/it]
88%|████████▊ | 452/512 [50:27<06:50, 6.85s/it]
{'loss': '0.002173', 'grad_norm': '0.1871', 'learning_rate': '1.344e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.04', 'tokens/train_per_sec_per_gpu': '33.02', 'tokens/total': 13681792, 'tokens/trainable': 206592, 'epoch': '1.766'} |
|
88%|████████▊ | 452/512 [50:27<06:50, 6.85s/it]
88%|████████▊ | 453/512 [50:34<06:40, 6.78s/it]
{'loss': '8.33e-05', 'grad_norm': '0.007497', 'learning_rate': '1.333e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '33.31', 'tokens/total': 13712256, 'tokens/trainable': 207055, 'epoch': '1.77'} |
|
88%|████████▊ | 453/512 [50:34<06:40, 6.78s/it]
89%|████████▊ | 454/512 [50:40<06:29, 6.71s/it]
{'loss': '0.0001119', 'grad_norm': '0.02134', 'learning_rate': '1.322e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '34.87', 'tokens/total': 13742128, 'tokens/trainable': 207494, 'epoch': '1.773'} |
|
89%|████████▊ | 454/512 [50:40<06:29, 6.71s/it]
89%|████████▉ | 455/512 [50:47<06:20, 6.67s/it]
{'loss': '3.2e-05', 'grad_norm': '0.0028', 'learning_rate': '1.311e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '33.26', 'tokens/total': 13772160, 'tokens/trainable': 207956, 'epoch': '1.777'} |
|
89%|████████▉ | 455/512 [50:47<06:20, 6.67s/it]
89%|████████▉ | 456/512 [50:53<06:12, 6.66s/it]
{'loss': '0.0001453', 'grad_norm': '0.02571', 'learning_rate': '1.301e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '35.79', 'tokens/total': 13802448, 'tokens/trainable': 208408, 'epoch': '1.781'} |
|
89%|████████▉ | 456/512 [50:53<06:12, 6.66s/it]
89%|████████▉ | 457/512 [51:00<06:05, 6.65s/it]
{'loss': '0.009408', 'grad_norm': '0.5833', 'learning_rate': '1.29e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '36.32', 'tokens/total': 13832880, 'tokens/trainable': 208889, 'epoch': '1.785'} |
|
89%|████████▉ | 457/512 [51:00<06:05, 6.65s/it]
89%|████████▉ | 458/512 [51:07<05:58, 6.64s/it]
{'loss': '4.802e-05', 'grad_norm': '0.003063', 'learning_rate': '1.28e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.64', 'tokens/train_per_sec_per_gpu': '34.18', 'tokens/total': 13862880, 'tokens/trainable': 209336, 'epoch': '1.789'} |
|
89%|████████▉ | 458/512 [51:07<05:58, 6.64s/it]
90%|████████▉ | 459/512 [51:13<05:52, 6.65s/it]
{'loss': '8.897e-05', 'grad_norm': '0.01603', 'learning_rate': '1.27e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.33', 'tokens/total': 13893440, 'tokens/trainable': 209797, 'epoch': '1.793'} |
|
90%|████████▉ | 459/512 [51:13<05:52, 6.65s/it]
90%|████████▉ | 460/512 [51:20<05:45, 6.65s/it]
{'loss': '0.000399', 'grad_norm': '0.1747', 'learning_rate': '1.26e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '36.31', 'tokens/total': 13923824, 'tokens/trainable': 210240, 'epoch': '1.797'} |
|
90%|████████▉ | 460/512 [51:20<05:45, 6.65s/it]
90%|█████████ | 461/512 [51:27<05:39, 6.65s/it]
{'loss': '0.00195', 'grad_norm': '0.2413', 'learning_rate': '1.251e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '35.16', 'tokens/total': 13954208, 'tokens/trainable': 210702, 'epoch': '1.801'} |
|
90%|█████████ | 461/512 [51:27<05:39, 6.65s/it]
90%|█████████ | 462/512 [51:33<05:32, 6.65s/it]
{'loss': '0.0007608', 'grad_norm': '0.07432', 'learning_rate': '1.241e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '31.73', 'tokens/total': 13984784, 'tokens/trainable': 211140, 'epoch': '1.805'} |
|
90%|█████████ | 462/512 [51:33<05:32, 6.65s/it]
90%|█████████ | 463/512 [51:40<05:25, 6.64s/it]
{'loss': '3.745e-05', 'grad_norm': '0.002309', 'learning_rate': '1.232e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '32.88', 'tokens/total': 14014896, 'tokens/trainable': 211575, 'epoch': '1.809'} |
|
90%|█████████ | 463/512 [51:40<05:25, 6.64s/it]
91%|█████████ | 464/512 [51:47<05:18, 6.64s/it]
{'loss': '0.0004949', 'grad_norm': '0.07452', 'learning_rate': '1.223e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '39.34', 'tokens/total': 14045312, 'tokens/trainable': 212083, 'epoch': '1.812'} |
|
91%|█████████ | 464/512 [51:47<05:18, 6.64s/it]
91%|█████████ | 465/512 [51:53<05:12, 6.66s/it]
{'loss': '4.363e-05', 'grad_norm': '0.009561', 'learning_rate': '1.214e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '36.91', 'tokens/total': 14075840, 'tokens/trainable': 212558, 'epoch': '1.816'} |
|
91%|█████████ | 465/512 [51:53<05:12, 6.66s/it]
91%|█████████ | 466/512 [52:00<05:05, 6.64s/it]
{'loss': '0.0002267', 'grad_norm': '0.05597', 'learning_rate': '1.205e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.97', 'tokens/train_per_sec_per_gpu': '33.13', 'tokens/total': 14106016, 'tokens/trainable': 213014, 'epoch': '1.82'} |
|
91%|█████████ | 466/512 [52:00<05:05, 6.64s/it]
91%|█████████ | 467/512 [52:07<04:59, 6.65s/it]
{'loss': '0.0003227', 'grad_norm': '0.04577', 'learning_rate': '1.197e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '37.99', 'tokens/train_per_sec_per_gpu': '34.26', 'tokens/total': 14136512, 'tokens/trainable': 213465, 'epoch': '1.824'} |
|
91%|█████████ | 467/512 [52:07<04:59, 6.65s/it]
91%|█████████▏| 468/512 [52:13<04:52, 6.65s/it]
{'loss': '1.573e-05', 'grad_norm': '0.001274', 'learning_rate': '1.188e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '36.93', 'tokens/total': 14166832, 'tokens/trainable': 213940, 'epoch': '1.828'} |
|
91%|█████████▏| 468/512 [52:13<04:52, 6.65s/it]
92%|█████████▏| 469/512 [52:20<04:46, 6.67s/it]
{'loss': '1.218e-05', 'grad_norm': '0.00117', 'learning_rate': '1.18e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '35.31', 'tokens/total': 14197360, 'tokens/trainable': 214415, 'epoch': '1.832'} |
|
92%|█████████▏| 469/512 [52:20<04:46, 6.67s/it]
92%|█████████▏| 470/512 [52:27<04:40, 6.67s/it]
{'loss': '8.308e-06', 'grad_norm': '0.0005075', 'learning_rate': '1.172e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '34.52', 'tokens/total': 14227600, 'tokens/trainable': 214880, 'epoch': '1.836'} |
|
92%|█████████▏| 470/512 [52:27<04:40, 6.67s/it]
92%|█████████▏| 471/512 [52:33<04:32, 6.65s/it]
{'loss': '0.000172', 'grad_norm': '0.01797', 'learning_rate': '1.164e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '33.7', 'tokens/total': 14257888, 'tokens/trainable': 215334, 'epoch': '1.84'} |
|
92%|█████████▏| 471/512 [52:33<04:32, 6.65s/it]
92%|█████████▏| 472/512 [52:40<04:26, 6.67s/it]
{'loss': '6.079e-06', 'grad_norm': '0.0002969', 'learning_rate': '1.156e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '35.06', 'tokens/total': 14288208, 'tokens/trainable': 215822, 'epoch': '1.844'} |
|
92%|█████████▏| 472/512 [52:40<04:26, 6.67s/it]
92%|█████████▏| 473/512 [52:47<04:20, 6.67s/it]
{'loss': '0.0007842', 'grad_norm': '0.06467', 'learning_rate': '1.149e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '30.87', 'tokens/total': 14318448, 'tokens/trainable': 216239, 'epoch': '1.848'} |
|
92%|█████████▏| 473/512 [52:47<04:20, 6.67s/it]
93%|█████████▎| 474/512 [52:53<04:13, 6.66s/it]
{'loss': '0.002163', 'grad_norm': '0.1723', 'learning_rate': '1.142e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '33.49', 'tokens/total': 14348912, 'tokens/trainable': 216691, 'epoch': '1.852'} |
|
93%|█████████▎| 474/512 [52:53<04:13, 6.66s/it]
93%|█████████▎| 475/512 [53:00<04:05, 6.65s/it]
{'loss': '0.001815', 'grad_norm': '0.1359', 'learning_rate': '1.135e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '33.38', 'tokens/total': 14379440, 'tokens/trainable': 217162, 'epoch': '1.855'} |
|
93%|█████████▎| 475/512 [53:00<04:05, 6.65s/it]
93%|█████████▎| 476/512 [53:06<03:59, 6.66s/it]
{'loss': '4.464e-05', 'grad_norm': '0.003589', 'learning_rate': '1.128e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '36.12', 'tokens/total': 14409840, 'tokens/trainable': 217638, 'epoch': '1.859'} |
|
93%|█████████▎| 476/512 [53:07<03:59, 6.66s/it]
93%|█████████▎| 477/512 [53:13<03:53, 6.66s/it]
{'loss': '0.0001891', 'grad_norm': '0.03571', 'learning_rate': '1.121e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '33.05', 'tokens/total': 14440256, 'tokens/trainable': 218053, 'epoch': '1.863'} |
|
93%|█████████▎| 477/512 [53:13<03:53, 6.66s/it]
93%|█████████▎| 478/512 [53:20<03:46, 6.65s/it]
{'loss': '1.546e-05', 'grad_norm': '0.001079', 'learning_rate': '1.114e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '34.38', 'tokens/total': 14470704, 'tokens/trainable': 218503, 'epoch': '1.867'} |
|
93%|█████████▎| 478/512 [53:20<03:46, 6.65s/it]
94%|█████████▎| 479/512 [53:26<03:39, 6.65s/it]
{'loss': '8.504e-05', 'grad_norm': '0.009676', 'learning_rate': '1.108e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '36.45', 'tokens/total': 14501072, 'tokens/trainable': 218964, 'epoch': '1.871'} |
|
94%|█████████▎| 479/512 [53:26<03:39, 6.65s/it]
94%|█████████▍| 480/512 [53:33<03:32, 6.64s/it]
{'loss': '3.897e-05', 'grad_norm': '0.003448', 'learning_rate': '1.102e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '35.75', 'tokens/total': 14531200, 'tokens/trainable': 219412, 'epoch': '1.875'} |
|
94%|█████████▍| 480/512 [53:33<03:32, 6.64s/it][2026-08-18 15:24:54,091] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-480 |
|
94%|█████████▍| 481/512 [53:41<03:41, 7.15s/it]
{'loss': '0.0001407', 'grad_norm': '0.01793', 'learning_rate': '1.096e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '38.63', 'tokens/train_per_sec_per_gpu': '38.5', 'tokens/total': 14561152, 'tokens/trainable': 219886, 'epoch': '1.879'} |
|
94%|█████████▍| 481/512 [53:41<03:41, 7.15s/it]
94%|█████████▍| 482/512 [53:48<03:29, 6.99s/it]
{'loss': '2.85e-05', 'grad_norm': '0.002653', 'learning_rate': '1.09e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '37.77', 'tokens/train_per_sec_per_gpu': '34.77', 'tokens/total': 14591264, 'tokens/trainable': 220343, 'epoch': '1.883'} |
|
94%|█████████▍| 482/512 [53:48<03:29, 6.99s/it]
94%|█████████▍| 483/512 [53:55<03:19, 6.89s/it]
{'loss': '0.003052', 'grad_norm': '0.1956', 'learning_rate': '1.084e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '37.77', 'tokens/train_per_sec_per_gpu': '35.49', 'tokens/total': 14621568, 'tokens/trainable': 220825, 'epoch': '1.887'} |
|
94%|█████████▍| 483/512 [53:55<03:19, 6.89s/it]
95%|█████████▍| 484/512 [54:01<03:10, 6.82s/it]
{'loss': '2.215e-05', 'grad_norm': '0.001849', 'learning_rate': '1.079e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.77', 'tokens/train_per_sec_per_gpu': '32.75', 'tokens/total': 14652064, 'tokens/trainable': 221282, 'epoch': '1.891'} |
|
95%|█████████▍| 484/512 [54:01<03:10, 6.82s/it]
95%|█████████▍| 485/512 [54:08<03:02, 6.77s/it]
{'loss': '8.448e-06', 'grad_norm': '0.0004012', 'learning_rate': '1.073e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '38.11', 'tokens/total': 14682608, 'tokens/trainable': 221780, 'epoch': '1.895'} |
|
95%|█████████▍| 485/512 [54:08<03:02, 6.77s/it]
95%|█████████▍| 486/512 [54:15<02:55, 6.75s/it]
{'loss': '1.871e-05', 'grad_norm': '0.0009792', 'learning_rate': '1.068e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '31.78', 'tokens/total': 14713168, 'tokens/trainable': 222210, 'epoch': '1.898'} |
|
95%|█████████▍| 486/512 [54:15<02:55, 6.75s/it]
95%|█████████▌| 487/512 [54:21<02:47, 6.71s/it]
{'loss': '6.944e-05', 'grad_norm': '0.005298', 'learning_rate': '1.063e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '29.58', 'tokens/total': 14743536, 'tokens/trainable': 222646, 'epoch': '1.902'} |
|
95%|█████████▌| 487/512 [54:21<02:47, 6.71s/it]
95%|█████████▌| 488/512 [54:28<02:40, 6.69s/it]
{'loss': '0.001105', 'grad_norm': '0.07348', 'learning_rate': '1.058e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '33.38', 'tokens/total': 14773728, 'tokens/trainable': 223066, 'epoch': '1.906'} |
|
95%|█████████▌| 488/512 [54:28<02:40, 6.69s/it]
96%|█████████▌| 489/512 [54:35<02:33, 6.66s/it]
{'loss': '0.0001521', 'grad_norm': '0.01697', 'learning_rate': '1.054e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.49', 'tokens/total': 14803744, 'tokens/trainable': 223512, 'epoch': '1.91'} |
|
96%|█████████▌| 489/512 [54:35<02:33, 6.66s/it]
96%|█████████▌| 490/512 [54:41<02:23, 6.51s/it]
{'loss': '9.547e-05', 'grad_norm': '0.006854', 'learning_rate': '1.049e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '39.3', 'tokens/total': 14832112, 'tokens/trainable': 223956, 'epoch': '1.914'} |
|
96%|█████████▌| 490/512 [54:41<02:23, 6.51s/it]
96%|█████████▌| 491/512 [54:47<02:14, 6.40s/it]
{'loss': '2.966e-05', 'grad_norm': '0.003061', 'learning_rate': '1.045e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.44', 'memory/max_allocated (GiB)': '33.44', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.66', 'tokens/total': 14860560, 'tokens/trainable': 224416, 'epoch': '1.918'} |
|
96%|█████████▌| 491/512 [54:47<02:14, 6.40s/it]
96%|█████████▌| 492/512 [54:53<02:09, 6.47s/it]
{'loss': '0.0006922', 'grad_norm': '0.09637', 'learning_rate': '1.041e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.38', 'tokens/total': 14890848, 'tokens/trainable': 224883, 'epoch': '1.922'} |
|
96%|█████████▌| 492/512 [54:53<02:09, 6.47s/it]
96%|█████████▋| 493/512 [55:00<02:03, 6.51s/it]
{'loss': '2.717e-05', 'grad_norm': '0.001974', 'learning_rate': '1.037e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '35.76', 'tokens/total': 14921008, 'tokens/trainable': 225324, 'epoch': '1.926'} |
|
96%|█████████▋| 493/512 [55:00<02:03, 6.51s/it]
96%|█████████▋| 494/512 [55:07<01:57, 6.54s/it]
{'loss': '0.000147', 'grad_norm': '0.02106', 'learning_rate': '1.034e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '35.16', 'tokens/total': 14951360, 'tokens/trainable': 225783, 'epoch': '1.93'} |
|
96%|█████████▋| 494/512 [55:07<01:57, 6.54s/it]
97%|█████████▋| 495/512 [55:13<01:52, 6.59s/it]
{'loss': '0.005182', 'grad_norm': '0.3205', 'learning_rate': '1.03e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.58', 'tokens/total': 14981552, 'tokens/trainable': 226281, 'epoch': '1.934'} |
|
97%|█████████▋| 495/512 [55:13<01:52, 6.59s/it]
97%|█████████▋| 496/512 [55:20<01:45, 6.58s/it]
{'loss': '1.262e-05', 'grad_norm': '0.0008805', 'learning_rate': '1.027e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.89', 'tokens/total': 15011584, 'tokens/trainable': 226735, 'epoch': '1.938'} |
|
97%|█████████▋| 496/512 [55:20<01:45, 6.58s/it]
97%|█████████▋| 497/512 [55:27<01:39, 6.60s/it]
{'loss': '0.003941', 'grad_norm': '0.2586', 'learning_rate': '1.024e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.57', 'tokens/total': 15042016, 'tokens/trainable': 227198, 'epoch': '1.941'} |
|
97%|█████████▋| 497/512 [55:27<01:39, 6.60s/it]
97%|█████████▋| 498/512 [55:33<01:32, 6.59s/it]
{'loss': '9.387e-06', 'grad_norm': '0.0003626', 'learning_rate': '1.021e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.68', 'memory/max_allocated (GiB)': '33.68', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '31.28', 'tokens/total': 15071984, 'tokens/trainable': 227623, 'epoch': '1.945'} |
|
97%|█████████▋| 498/512 [55:33<01:32, 6.59s/it]
97%|█████████▋| 499/512 [55:40<01:25, 6.60s/it]
{'loss': '0.003037', 'grad_norm': '0.2358', 'learning_rate': '1.018e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '32.39', 'tokens/total': 15102192, 'tokens/trainable': 228074, 'epoch': '1.949'} |
|
97%|█████████▋| 499/512 [55:40<01:25, 6.60s/it]
98%|█████████▊| 500/512 [55:46<01:19, 6.61s/it]
{'loss': '9.965e-05', 'grad_norm': '0.008456', 'learning_rate': '1.016e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '35.88', 'tokens/total': 15132560, 'tokens/trainable': 228541, 'epoch': '1.953'} |
|
98%|█████████▊| 500/512 [55:46<01:19, 6.61s/it]
98%|█████████▊| 501/512 [55:53<01:12, 6.63s/it]
{'loss': '2.887e-05', 'grad_norm': '0.005541', 'learning_rate': '1.013e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '38.03', 'tokens/total': 15163152, 'tokens/trainable': 229031, 'epoch': '1.957'} |
|
98%|█████████▊| 501/512 [55:53<01:12, 6.63s/it]
98%|█████████▊| 502/512 [56:00<01:06, 6.63s/it]
{'loss': '8.65e-05', 'grad_norm': '0.01309', 'learning_rate': '1.011e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '39.03', 'tokens/total': 15193456, 'tokens/trainable': 229512, 'epoch': '1.961'} |
|
98%|█████████▊| 502/512 [56:00<01:06, 6.63s/it]
98%|█████████▊| 503/512 [56:06<00:59, 6.62s/it]
{'loss': '0.001191', 'grad_norm': '0.1275', 'learning_rate': '1.009e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.49', 'tokens/total': 15223680, 'tokens/trainable': 229989, 'epoch': '1.965'} |
|
98%|█████████▊| 503/512 [56:06<00:59, 6.62s/it]
98%|█████████▊| 504/512 [56:13<00:53, 6.63s/it]
{'loss': '7.171e-05', 'grad_norm': '0.007072', 'learning_rate': '1.008e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.03', 'tokens/total': 15254336, 'tokens/trainable': 230447, 'epoch': '1.969'} |
|
98%|█████████▊| 504/512 [56:13<00:53, 6.63s/it]
99%|█████████▊| 505/512 [56:20<00:46, 6.63s/it]
{'loss': '0.0001278', 'grad_norm': '0.01566', 'learning_rate': '1.006e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.52', 'tokens/total': 15284496, 'tokens/trainable': 230879, 'epoch': '1.973'} |
|
99%|█████████▊| 505/512 [56:20<00:46, 6.63s/it]
99%|█████████▉| 506/512 [56:26<00:39, 6.64s/it]
{'loss': '1.993e-05', 'grad_norm': '0.002059', 'learning_rate': '1.005e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.33', 'tokens/total': 15314976, 'tokens/trainable': 231348, 'epoch': '1.977'} |
|
99%|█████████▉| 506/512 [56:26<00:39, 6.64s/it]
99%|█████████▉| 507/512 [56:33<00:33, 6.64s/it]
{'loss': '6.569e-05', 'grad_norm': '0.007667', 'learning_rate': '1.003e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '35.53', 'tokens/total': 15345280, 'tokens/trainable': 231819, 'epoch': '1.98'} |
|
99%|█████████▉| 507/512 [56:33<00:33, 6.64s/it]
99%|█████████▉| 508/512 [56:39<00:26, 6.63s/it]
{'loss': '2.466e-05', 'grad_norm': '0.001949', 'learning_rate': '1.002e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.77', 'tokens/total': 15375424, 'tokens/trainable': 232269, 'epoch': '1.984'} |
|
99%|█████████▉| 508/512 [56:39<00:26, 6.63s/it]
99%|█████████▉| 509/512 [56:46<00:19, 6.62s/it]
{'loss': '5.146e-05', 'grad_norm': '0.005513', 'learning_rate': '1.001e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '34.34', 'tokens/total': 15405872, 'tokens/trainable': 232716, 'epoch': '1.988'} |
|
99%|█████████▉| 509/512 [56:46<00:19, 6.62s/it]
100%|█████████▉| 510/512 [56:53<00:13, 6.63s/it]
{'loss': '8.855e-06', 'grad_norm': '0.0006265', 'learning_rate': '1.001e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.34', 'tokens/total': 15436128, 'tokens/trainable': 233178, 'epoch': '1.992'} |
|
100%|█████████▉| 510/512 [56:53<00:13, 6.63s/it]
100%|█████████▉| 511/512 [56:59<00:06, 6.64s/it]
{'loss': '0.0002487', 'grad_norm': '0.04183', 'learning_rate': '1e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '36.15', 'tokens/total': 15466464, 'tokens/trainable': 233647, 'epoch': '1.996'} |
|
100%|█████████▉| 511/512 [56:59<00:06, 6.64s/it]
100%|██████████| 512/512 [57:06<00:00, 6.66s/it]
{'loss': '0.0003747', 'grad_norm': '0.04211', 'learning_rate': '1e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.67', 'memory/max_allocated (GiB)': '33.67', 'memory/device_reserved (GiB)': '38.44', 'tokens/train_per_sec_per_gpu': '29.59', 'tokens/total': 15496368, 'tokens/trainable': 234060, 'epoch': '2'} |
|
100%|██████████| 512/512 [57:06<00:00, 6.66s/it][2026-08-18 15:28:27,173] [INFO] [axolotl.core.trainers.base._save:828] [PID:12143] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-512 |
|
{'train_runtime': '3428', 'train_samples_per_second': '4.779', 'train_steps_per_second': '0.149', 'train_loss': '0.009169', 'memory/max_active (GiB)': '24.25', 'memory/max_allocated (GiB)': '24.25', 'memory/device_reserved (GiB)': '38.44', 'epoch': '2', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 512/512 [57:08<00:00, 6.66s/it]
100%|██████████| 512/512 [57:08<00:00, 6.70s/it] |
| [2026-08-18 15:28:28,875] [INFO] [axolotl.train.save_trained_model:267] [PID:12143] Training completed! Saving trained model to /workspace/wave/training/checkpoints. |
| [2026-08-18 15:28:29,160] [INFO] [axolotl.train.save_trained_model:388] [PID:12143] Model successfully saved to /workspace/wave/training/checkpoints |
|
|
